// following functions are shamelessly copied from LLVM.
bool isPotentiallyReachableFromMany(SmallVectorImpl<BasicBlock *> &Worklist, BasicBlock *StopBB) {
// Limit the number of blocks we visit. The goal is to avoid run-away compile
// times on large CFGs without hampering sensible code. Arbitrarily chosen.
unsigned Limit = 32;
SmallSet<const BasicBlock*, 64> Visited;
do {
BasicBlock *BB = Worklist.pop_back_val();
if (!Visited.insert(BB).second)
continue;
if (BB == StopBB)
return true;
if (!--Limit) {
// We haven't been able to prove it one way or the other. Conservatively
// answer true -- that there is potentially a path.
return true;
}
Worklist.append(succ_begin(BB), succ_end(BB));
} while (!Worklist.empty());
// We have exhausted all possible paths and are certain that 'To' can not be
// reached from 'From'.
return false;
}
/// Emit a check for whether a non-native function call produced an
/// error.
///
/// \c results should be left with only values that match the formal
/// direct results of the function.
void
SILGenFunction::emitForeignErrorCheck(SILLocation loc,
SmallVectorImpl<ManagedValue> &results,
ManagedValue errorSlot,
bool suppressErrorCheck,
const ForeignErrorConvention &foreignError) {
// All of this is autogenerated.
loc.markAutoGenerated();
switch (foreignError.getKind()) {
case ForeignErrorConvention::ZeroPreservedResult:
assert(results.size() == 1);
emitResultIsZeroErrorCheck(*this, loc, results[0], errorSlot,
suppressErrorCheck,
/*zeroIsError*/ true);
return;
case ForeignErrorConvention::ZeroResult:
assert(results.size() == 1);
emitResultIsZeroErrorCheck(*this, loc, results.pop_back_val(),
errorSlot, suppressErrorCheck,
/*zeroIsError*/ true);
return;
case ForeignErrorConvention::NonZeroResult:
assert(results.size() == 1);
emitResultIsZeroErrorCheck(*this, loc, results.pop_back_val(),
errorSlot, suppressErrorCheck,
/*zeroIsError*/ false);
return;
case ForeignErrorConvention::NilResult:
assert(results.size() == 1);
results[0] = emitResultIsNilErrorCheck(*this, loc, results[0], errorSlot,
suppressErrorCheck);
return;
case ForeignErrorConvention::NonNilError:
// Leave the direct results alone.
emitErrorIsNonNilErrorCheck(*this, loc, errorSlot, suppressErrorCheck);
return;
}
llvm_unreachable("bad foreign error convention kind");
}
/// \brief Given a list of combinable load. Combine the maximum number of them.
bool LoadCombine::combineLoads(SmallVectorImpl<LoadPOPPair> &Loads) {
// Remove loads from the end while the size is not a power of 2.
unsigned TotalSize = 0;
for (const auto &L : Loads)
TotalSize += L.Load->getType()->getPrimitiveSizeInBits();
while (TotalSize != 0 && !isPowerOf2_32(TotalSize))
TotalSize -= Loads.pop_back_val().Load->getType()->getPrimitiveSizeInBits();
if (Loads.size() < 2)
return false;
DEBUG({
dbgs() << "***** Combining Loads ******\n";
for (const auto &L : Loads) {
dbgs() << L.POP.Offset << ": " << *L.Load << "\n";
}
});
static bool isPotentiallyReachableInner(SmallVectorImpl<BasicBlock *> &Worklist,
BasicBlock *StopBB,
const DominatorTree *DT,
const LoopInfo *LI) {
// When the stop block is unreachable, it's dominated from everywhere,
// regardless of whether there's a path between the two blocks.
if (DT && !DT->isReachableFromEntry(StopBB))
DT = 0;
// Limit the number of blocks we visit. The goal is to avoid run-away compile
// times on large CFGs without hampering sensible code. Arbitrarily chosen.
unsigned Limit = 32;
SmallSet<const BasicBlock*, 64> Visited;
do {
BasicBlock *BB = Worklist.pop_back_val();
if (!Visited.insert(BB))
continue;
if (BB == StopBB)
return true;
if (DT && DT->dominates(BB, StopBB))
return true;
if (LI && loopContainsBoth(LI, BB, StopBB))
return true;
if (!--Limit) {
// We haven't been able to prove it one way or the other. Conservatively
// answer true -- that there is potentially a path.
return true;
}
if (const Loop *Outer = LI ? getOutermostLoop(LI, BB) : 0) {
// All blocks in a single loop are reachable from all other blocks. From
// any of these blocks, we can skip directly to the exits of the loop,
// ignoring any other blocks inside the loop body.
Outer->getExitBlocks(Worklist);
} else {
for (succ_iterator I = succ_begin(BB), E = succ_end(BB); I != E; ++I)
Worklist.push_back(*I);
}
} while (!Worklist.empty());
// We have exhausted all possible paths and are certain that 'To' can not be
// reached from 'From'.
return false;
}
/// For every instruction from the worklist, check to see if it has any uses
/// that are outside the current loop. If so, insert LCSSA PHI nodes and
/// rewrite the uses.
bool llvm::formLCSSAForInstructions(SmallVectorImpl<Instruction *> &Worklist,
DominatorTree &DT, LoopInfo &LI) {
SmallVector<Use *, 16> UsesToRewrite;
SmallSetVector<PHINode *, 16> PHIsToRemove;
PredIteratorCache PredCache;
bool Changed = false;
// Cache the Loop ExitBlocks across this loop. We expect to get a lot of
// instructions within the same loops, computing the exit blocks is
// expensive, and we're not mutating the loop structure.
SmallDenseMap<Loop*, SmallVector<BasicBlock *,1>> LoopExitBlocks;
while (!Worklist.empty()) {
UsesToRewrite.clear();
Instruction *I = Worklist.pop_back_val();
BasicBlock *InstBB = I->getParent();
Loop *L = LI.getLoopFor(InstBB);
if (!LoopExitBlocks.count(L))
L->getExitBlocks(LoopExitBlocks[L]);
assert(LoopExitBlocks.count(L));
const SmallVectorImpl<BasicBlock *> &ExitBlocks = LoopExitBlocks[L];
if (ExitBlocks.empty())
continue;
// Tokens cannot be used in PHI nodes, so we skip over them.
// We can run into tokens which are live out of a loop with catchswitch
// instructions in Windows EH if the catchswitch has one catchpad which
// is inside the loop and another which is not.
if (I->getType()->isTokenTy())
continue;
for (Use &U : I->uses()) {
Instruction *User = cast<Instruction>(U.getUser());
BasicBlock *UserBB = User->getParent();
if (PHINode *PN = dyn_cast<PHINode>(User))
UserBB = PN->getIncomingBlock(U);
if (InstBB != UserBB && !L->contains(UserBB))
UsesToRewrite.push_back(&U);
}
// If there are no uses outside the loop, exit with no change.
if (UsesToRewrite.empty())
continue;
++NumLCSSA; // We are applying the transformation
// Invoke instructions are special in that their result value is not
// available along their unwind edge. The code below tests to see whether
// DomBB dominates the value, so adjust DomBB to the normal destination
// block, which is effectively where the value is first usable.
BasicBlock *DomBB = InstBB;
if (InvokeInst *Inv = dyn_cast<InvokeInst>(I))
DomBB = Inv->getNormalDest();
DomTreeNode *DomNode = DT.getNode(DomBB);
SmallVector<PHINode *, 16> AddedPHIs;
SmallVector<PHINode *, 8> PostProcessPHIs;
SmallVector<PHINode *, 4> InsertedPHIs;
SSAUpdater SSAUpdate(&InsertedPHIs);
SSAUpdate.Initialize(I->getType(), I->getName());
// Insert the LCSSA phi's into all of the exit blocks dominated by the
// value, and add them to the Phi's map.
for (BasicBlock *ExitBB : ExitBlocks) {
if (!DT.dominates(DomNode, DT.getNode(ExitBB)))
continue;
// If we already inserted something for this BB, don't reprocess it.
if (SSAUpdate.HasValueForBlock(ExitBB))
continue;
PHINode *PN = PHINode::Create(I->getType(), PredCache.size(ExitBB),
I->getName() + ".lcssa", &ExitBB->front());
// Add inputs from inside the loop for this PHI.
for (BasicBlock *Pred : PredCache.get(ExitBB)) {
PN->addIncoming(I, Pred);
// If the exit block has a predecessor not within the loop, arrange for
// the incoming value use corresponding to that predecessor to be
// rewritten in terms of a different LCSSA PHI.
if (!L->contains(Pred))
UsesToRewrite.push_back(
&PN->getOperandUse(PN->getOperandNumForIncomingValue(
PN->getNumIncomingValues() - 1)));
}
AddedPHIs.push_back(PN);
// Remember that this phi makes the value alive in this block.
SSAUpdate.AddAvailableValue(ExitBB, PN);
// LoopSimplify might fail to simplify some loops (e.g. when indirect
// branches are involved). In such situations, it might happen that an
// exit for Loop L1 is the header of a disjoint Loop L2. Thus, when we
// create PHIs in such an exit block, we are also inserting PHIs into L2's
// header. This could break LCSSA form for L2 because these inserted PHIs
// can also have uses outside of L2. Remember all PHIs in such situation
// as to revisit than later on. FIXME: Remove this if indirectbr support
// into LoopSimplify gets improved.
if (auto *OtherLoop = LI.getLoopFor(ExitBB))
if (!L->contains(OtherLoop))
PostProcessPHIs.push_back(PN);
}
// Rewrite all uses outside the loop in terms of the new PHIs we just
// inserted.
for (Use *UseToRewrite : UsesToRewrite) {
// If this use is in an exit block, rewrite to use the newly inserted PHI.
// This is required for correctness because SSAUpdate doesn't handle uses
// in the same block. It assumes the PHI we inserted is at the end of the
// block.
Instruction *User = cast<Instruction>(UseToRewrite->getUser());
BasicBlock *UserBB = User->getParent();
if (PHINode *PN = dyn_cast<PHINode>(User))
UserBB = PN->getIncomingBlock(*UseToRewrite);
if (isa<PHINode>(UserBB->begin()) && isExitBlock(UserBB, ExitBlocks)) {
// Tell the VHs that the uses changed. This updates SCEV's caches.
if (UseToRewrite->get()->hasValueHandle())
ValueHandleBase::ValueIsRAUWd(*UseToRewrite, &UserBB->front());
UseToRewrite->set(&UserBB->front());
continue;
}
// Otherwise, do full PHI insertion.
SSAUpdate.RewriteUse(*UseToRewrite);
}
// SSAUpdater might have inserted phi-nodes inside other loops. We'll need
// to post-process them to keep LCSSA form.
for (PHINode *InsertedPN : InsertedPHIs) {
if (auto *OtherLoop = LI.getLoopFor(InsertedPN->getParent()))
if (!L->contains(OtherLoop))
PostProcessPHIs.push_back(InsertedPN);
}
// Post process PHI instructions that were inserted into another disjoint
// loop and update their exits properly.
for (auto *PostProcessPN : PostProcessPHIs) {
if (PostProcessPN->use_empty())
continue;
// Reprocess each PHI instruction.
Worklist.push_back(PostProcessPN);
}
// Keep track of PHI nodes that we want to remove because they did not have
// any uses rewritten.
for (PHINode *PN : AddedPHIs)
if (PN->use_empty())
PHIsToRemove.insert(PN);
Changed = true;
}
// Remove PHI nodes that did not have any uses rewritten.
for (PHINode *PN : PHIsToRemove) {
assert (PN->use_empty() && "Trying to remove a phi with uses.");
PN->eraseFromParent();
}
return Changed;
}
bool llvm::isPotentiallyReachableFromMany(
SmallVectorImpl<BasicBlock *> &Worklist, BasicBlock *StopBB,
const SmallPtrSetImpl<BasicBlock *> *ExclusionSet, const DominatorTree *DT,
const LoopInfo *LI) {
// When the stop block is unreachable, it's dominated from everywhere,
// regardless of whether there's a path between the two blocks.
if (DT && !DT->isReachableFromEntry(StopBB))
DT = nullptr;
// We can't skip directly from a block that dominates the stop block if the
// exclusion block is potentially in between.
if (ExclusionSet && !ExclusionSet->empty())
DT = nullptr;
// Normally any block in a loop is reachable from any other block in a loop,
// however excluded blocks might partition the body of a loop to make that
// untrue.
SmallPtrSet<const Loop *, 8> LoopsWithHoles;
if (LI && ExclusionSet) {
for (auto BB : *ExclusionSet) {
if (const Loop *L = getOutermostLoop(LI, BB))
LoopsWithHoles.insert(L);
}
}
const Loop *StopLoop = LI ? getOutermostLoop(LI, StopBB) : nullptr;
// Limit the number of blocks we visit. The goal is to avoid run-away compile
// times on large CFGs without hampering sensible code. Arbitrarily chosen.
unsigned Limit = 32;
SmallPtrSet<const BasicBlock*, 32> Visited;
do {
BasicBlock *BB = Worklist.pop_back_val();
if (!Visited.insert(BB).second)
continue;
if (BB == StopBB)
return true;
if (ExclusionSet && ExclusionSet->count(BB))
continue;
if (DT && DT->dominates(BB, StopBB))
return true;
const Loop *Outer = nullptr;
if (LI) {
Outer = getOutermostLoop(LI, BB);
// If we're in a loop with a hole, not all blocks in the loop are
// reachable from all other blocks. That implies we can't simply jump to
// the loop's exit blocks, as that exit might need to pass through an
// excluded block. Clear Outer so we process BB's successors.
if (LoopsWithHoles.count(Outer))
Outer = nullptr;
if (StopLoop && Outer == StopLoop)
return true;
}
if (!--Limit) {
// We haven't been able to prove it one way or the other. Conservatively
// answer true -- that there is potentially a path.
return true;
}
if (Outer) {
// All blocks in a single loop are reachable from all other blocks. From
// any of these blocks, we can skip directly to the exits of the loop,
// ignoring any other blocks inside the loop body.
Outer->getExitBlocks(Worklist);
} else {
Worklist.append(succ_begin(BB), succ_end(BB));
}
} while (!Worklist.empty());
// We have exhausted all possible paths and are certain that 'To' can not be
// reached from 'From'.
return false;
}
void LiveRangeEdit::eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead,
ArrayRef<unsigned> RegsBeingSpilled) {
ToShrinkSet ToShrink;
for (;;) {
// Erase all dead defs.
while (!Dead.empty())
eliminateDeadDef(Dead.pop_back_val(), ToShrink);
if (ToShrink.empty())
break;
// Shrink just one live interval. Then delete new dead defs.
LiveInterval *LI = ToShrink.back();
ToShrink.pop_back();
if (foldAsLoad(LI, Dead))
continue;
if (TheDelegate)
TheDelegate->LRE_WillShrinkVirtReg(LI->reg);
if (!LIS.shrinkToUses(LI, &Dead))
continue;
// Don't create new intervals for a register being spilled.
// The new intervals would have to be spilled anyway so its not worth it.
// Also they currently aren't spilled so creating them and not spilling
// them results in incorrect code.
bool BeingSpilled = false;
for (unsigned i = 0, e = RegsBeingSpilled.size(); i != e; ++i) {
if (LI->reg == RegsBeingSpilled[i]) {
BeingSpilled = true;
break;
}
}
if (BeingSpilled) continue;
// LI may have been separated, create new intervals.
LI->RenumberValues();
ConnectedVNInfoEqClasses ConEQ(LIS);
unsigned NumComp = ConEQ.Classify(LI);
if (NumComp <= 1)
continue;
++NumFracRanges;
bool IsOriginal = VRM && VRM->getOriginal(LI->reg) == LI->reg;
DEBUG(dbgs() << NumComp << " components: " << *LI << '\n');
SmallVector<LiveInterval*, 8> Dups(1, LI);
for (unsigned i = 1; i != NumComp; ++i) {
Dups.push_back(&createEmptyIntervalFrom(LI->reg));
// If LI is an original interval that hasn't been split yet, make the new
// intervals their own originals instead of referring to LI. The original
// interval must contain all the split products, and LI doesn't.
if (IsOriginal)
VRM->setIsSplitFromReg(Dups.back()->reg, 0);
if (TheDelegate)
TheDelegate->LRE_DidCloneVirtReg(Dups.back()->reg, LI->reg);
}
ConEQ.Distribute(&Dups[0], MRI);
DEBUG({
for (unsigned i = 0; i != NumComp; ++i)
dbgs() << '\t' << *Dups[i] << '\n';
});
}
static bool isPotentiallyReachableInnerNotViaDef(const BasicBlock *DefBB, SmallVectorImpl<BasicBlock *> &Worklist, const BasicBlock *SpBB,
std::set<BasicBlock*>& EndBlocks,
const DominatorTree *DT,
const LoopInfo *LI) {
// we can not conservatively include the use when timeout
SmallSet<const BasicBlock*, 64> Visited;
do {
// This is a depth-first search for a path not via def
// return true whenever the first valid path is found
BasicBlock *BB = Worklist.pop_back_val();
// if has already visited
if (!Visited.insert(BB))
continue;
// Here we have 3 cases:
// 1. If BB == SpBB, we just start the search from safepoint, and def should before safepoint in the same block, and BB == DefBB does not mean we reach the def in the search path.
// 2. If DefBB == StopBB, it infers that use is a phi otherwise this case would be returned in a fastpath in isPotentiallyReachableNotViaDef().
// and if use is a phi and in the same block with def, we reach use before def.
// However, this case could not happen because if use is phi, it will stop at one of its incoming block and return true.
// 3. If only BB == DefBB, we reach the def in the search path and this path should be discarded.
if (BB == DefBB && BB != SpBB)
continue;
if (EndBlocks.find(BB) != EndBlocks.end())
return true;
#ifdef OPTIMIZED_LIVENESS_ANALYSIS
// If curent basicblock, the defblock, and any incoming block of PhiNode are not within a loop,
// we can strip the loop and only add the loop exsit basicblock into the worklist to make the search shorter
if (LI && (DefBB == NULL || !loopContainsBoth(LI, BB, DefBB))) {
bool contains = false;
for (std::set<BasicBlock*>::iterator itr = EndBlocks.begin(), end = EndBlocks.end(); itr != end; itr++) {
if (loopContainsBoth(LI, BB, *itr)) {
contains = true;
break;
}
}
if (!contains) {
if (const Loop *Outer = getOutermostLoop(LI, BB)) {
// All blocks in a single loop are reachable from all other blocks. From
// any of these blocks, we can skip directly to the exits of the loop,
// ignoring any other blocks inside the loop body.
Outer->getExitBlocks(Worklist);
}
} else {
Worklist.append(succ_begin(BB), succ_end(BB));
}
} else {
Worklist.append(succ_begin(BB), succ_end(BB));
}
#else
Worklist.append(succ_begin(BB), succ_end(BB));
#endif
} while (!Worklist.empty());
// We have exhausted all possible paths and are certain that 'To' can not be
// reached from 'From'.
return false;
}
void LiveRangeEdit::eliminateDeadDefs(SmallVectorImpl<MachineInstr*> &Dead,
ArrayRef<unsigned> RegsBeingSpilled) {
SetVector<LiveInterval*,
SmallVector<LiveInterval*, 8>,
SmallPtrSet<LiveInterval*, 8> > ToShrink;
for (;;) {
// Erase all dead defs.
while (!Dead.empty()) {
MachineInstr *MI = Dead.pop_back_val();
assert(MI->allDefsAreDead() && "Def isn't really dead");
SlotIndex Idx = LIS.getInstructionIndex(MI).getRegSlot();
// Never delete inline asm.
if (MI->isInlineAsm()) {
DEBUG(dbgs() << "Won't delete: " << Idx << '\t' << *MI);
continue;
}
// Use the same criteria as DeadMachineInstructionElim.
bool SawStore = false;
if (!MI->isSafeToMove(&TII, 0, SawStore)) {
DEBUG(dbgs() << "Can't delete: " << Idx << '\t' << *MI);
continue;
}
DEBUG(dbgs() << "Deleting dead def " << Idx << '\t' << *MI);
// Check for live intervals that may shrink
for (MachineInstr::mop_iterator MOI = MI->operands_begin(),
MOE = MI->operands_end(); MOI != MOE; ++MOI) {
if (!MOI->isReg())
continue;
unsigned Reg = MOI->getReg();
if (!TargetRegisterInfo::isVirtualRegister(Reg))
continue;
LiveInterval &LI = LIS.getInterval(Reg);
// Shrink read registers, unless it is likely to be expensive and
// unlikely to change anything. We typically don't want to shrink the
// PIC base register that has lots of uses everywhere.
// Always shrink COPY uses that probably come from live range splitting.
if (MI->readsVirtualRegister(Reg) &&
(MI->isCopy() || MOI->isDef() || MRI.hasOneNonDBGUse(Reg) ||
LI.killedAt(Idx)))
ToShrink.insert(&LI);
// Remove defined value.
if (MOI->isDef()) {
if (VNInfo *VNI = LI.getVNInfoAt(Idx)) {
if (TheDelegate)
TheDelegate->LRE_WillShrinkVirtReg(LI.reg);
LI.removeValNo(VNI);
if (LI.empty()) {
ToShrink.remove(&LI);
eraseVirtReg(Reg);
}
}
}
}
if (TheDelegate)
TheDelegate->LRE_WillEraseInstruction(MI);
LIS.RemoveMachineInstrFromMaps(MI);
MI->eraseFromParent();
++NumDCEDeleted;
}
if (ToShrink.empty())
break;
// Shrink just one live interval. Then delete new dead defs.
LiveInterval *LI = ToShrink.back();
ToShrink.pop_back();
if (foldAsLoad(LI, Dead))
continue;
if (TheDelegate)
TheDelegate->LRE_WillShrinkVirtReg(LI->reg);
if (!LIS.shrinkToUses(LI, &Dead))
continue;
// Don't create new intervals for a register being spilled.
// The new intervals would have to be spilled anyway so its not worth it.
// Also they currently aren't spilled so creating them and not spilling
// them results in incorrect code.
bool BeingSpilled = false;
for (unsigned i = 0, e = RegsBeingSpilled.size(); i != e; ++i) {
if (LI->reg == RegsBeingSpilled[i]) {
BeingSpilled = true;
break;
}
}
if (BeingSpilled) continue;
// LI may have been separated, create new intervals.
LI->RenumberValues(LIS);
ConnectedVNInfoEqClasses ConEQ(LIS);
unsigned NumComp = ConEQ.Classify(LI);
if (NumComp <= 1)
continue;
++NumFracRanges;
bool IsOriginal = VRM && VRM->getOriginal(LI->reg) == LI->reg;
DEBUG(dbgs() << NumComp << " components: " << *LI << '\n');
SmallVector<LiveInterval*, 8> Dups(1, LI);
for (unsigned i = 1; i != NumComp; ++i) {
Dups.push_back(&createFrom(LI->reg));
// If LI is an original interval that hasn't been split yet, make the new
// intervals their own originals instead of referring to LI. The original
// interval must contain all the split products, and LI doesn't.
if (IsOriginal)
VRM->setIsSplitFromReg(Dups.back()->reg, 0);
if (TheDelegate)
TheDelegate->LRE_DidCloneVirtReg(Dups.back()->reg, LI->reg);
}
ConEQ.Distribute(&Dups[0], MRI);
DEBUG({
for (unsigned i = 0; i != NumComp; ++i)
dbgs() << '\t' << *Dups[i] << '\n';
});
}
