static bool checkEFLAGSLive(MachineInstr *MI) {
if (MI->killsRegister(X86::EFLAGS))
return false;
// The EFLAGS operand of MI might be missing a kill marker.
// Figure out whether EFLAGS operand should LIVE after MI instruction.
MachineBasicBlock *BB = MI->getParent();
MachineBasicBlock::iterator ItrMI = MI;
// Scan forward through BB for a use/def of EFLAGS.
for (auto I = std::next(ItrMI), E = BB->end(); I != E; ++I) {
if (I->readsRegister(X86::EFLAGS))
return true;
if (I->definesRegister(X86::EFLAGS))
return false;
}
// We hit the end of the block, check whether EFLAGS is live into a successor.
for (auto I = BB->succ_begin(), E = BB->succ_end(); I != E; ++I) {
if ((*I)->isLiveIn(X86::EFLAGS))
return true;
}
return false;
}
/// Return true if \p MBB has one successor immediately following, and is its
/// only predecessor
static bool hasTrivialSuccessor(const MachineBasicBlock &MBB) {
if (MBB.succ_size() != 1)
return false;
const MachineBasicBlock *Succ = *MBB.succ_begin();
return (Succ->pred_size() == 1) && MBB.isLayoutSuccessor(Succ);
}
void LazyLiveness::computeBackedgeChain(MachineFunction& mf,
MachineBasicBlock* MBB) {
SparseBitVector<128> tmp = rv[MBB];
tmp.set(preorder[MBB]);
tmp &= backedge_source;
calculated.set(preorder[MBB]);
for (SparseBitVector<128>::iterator I = tmp.begin(); I != tmp.end(); ++I) {
assert(rev_preorder.size() > *I && "Unknown block!");
MachineBasicBlock* SrcMBB = rev_preorder[*I];
for (MachineBasicBlock::succ_iterator SI = SrcMBB->succ_begin(),
SE = SrcMBB->succ_end(); SI != SE; ++SI) {
MachineBasicBlock* TgtMBB = *SI;
if (backedges.count(std::make_pair(SrcMBB, TgtMBB)) &&
!rv[MBB].test(preorder[TgtMBB])) {
if (!calculated.test(preorder[TgtMBB]))
computeBackedgeChain(mf, TgtMBB);
tv[MBB].set(preorder[TgtMBB]);
SparseBitVector<128> right = tv[TgtMBB];
tv[MBB] |= right;
}
}
tv[MBB].reset(preorder[MBB]);
}
}
void LiveIntervals::pruneValue(LiveRange &LR, SlotIndex Kill,
SmallVectorImpl<SlotIndex> *EndPoints) {
LiveQueryResult LRQ = LR.Query(Kill);
VNInfo *VNI = LRQ.valueOutOrDead();
if (!VNI)
return;
MachineBasicBlock *KillMBB = Indexes->getMBBFromIndex(Kill);
SlotIndex MBBEnd = Indexes->getMBBEndIdx(KillMBB);
// If VNI isn't live out from KillMBB, the value is trivially pruned.
if (LRQ.endPoint() < MBBEnd) {
LR.removeSegment(Kill, LRQ.endPoint());
if (EndPoints) EndPoints->push_back(LRQ.endPoint());
return;
}
// VNI is live out of KillMBB.
LR.removeSegment(Kill, MBBEnd);
if (EndPoints) EndPoints->push_back(MBBEnd);
// Find all blocks that are reachable from KillMBB without leaving VNI's live
// range. It is possible that KillMBB itself is reachable, so start a DFS
// from each successor.
typedef SmallPtrSet<MachineBasicBlock*, 9> VisitedTy;
VisitedTy Visited;
for (MachineBasicBlock::succ_iterator
SuccI = KillMBB->succ_begin(), SuccE = KillMBB->succ_end();
SuccI != SuccE; ++SuccI) {
for (df_ext_iterator<MachineBasicBlock*, VisitedTy>
I = df_ext_begin(*SuccI, Visited), E = df_ext_end(*SuccI, Visited);
I != E;) {
MachineBasicBlock *MBB = *I;
// Check if VNI is live in to MBB.
SlotIndex MBBStart, MBBEnd;
std::tie(MBBStart, MBBEnd) = Indexes->getMBBRange(MBB);
LiveQueryResult LRQ = LR.Query(MBBStart);
if (LRQ.valueIn() != VNI) {
// This block isn't part of the VNI segment. Prune the search.
I.skipChildren();
continue;
}
// Prune the search if VNI is killed in MBB.
if (LRQ.endPoint() < MBBEnd) {
LR.removeSegment(MBBStart, LRQ.endPoint());
if (EndPoints) EndPoints->push_back(LRQ.endPoint());
I.skipChildren();
continue;
}
// VNI is live through MBB.
LR.removeSegment(MBBStart, MBBEnd);
if (EndPoints) EndPoints->push_back(MBBEnd);
++I;
}
}
}
void RegDefsUses::addLiveOut(const MachineBasicBlock &MBB,
const MachineBasicBlock &SuccBB) {
for (MachineBasicBlock::const_succ_iterator SI = MBB.succ_begin(),
SE = MBB.succ_end(); SI != SE; ++SI)
if (*SI != &SuccBB)
for (const auto &LI : (*SI)->liveins())
Uses.set(LI.PhysReg);
}
bool PTXInstrInfo::
IsAnySuccessorAlsoLayoutSuccessor(const MachineBasicBlock& MBB) {
for (MachineBasicBlock::const_succ_iterator
i = MBB.succ_begin(), e = MBB.succ_end(); i != e; ++i)
if (MBB.isLayoutSuccessor((const MachineBasicBlock*) &*i))
return true;
return false;
}
static bool isCPSRLiveout(MachineBasicBlock &MBB) {
for (MachineBasicBlock::succ_iterator I = MBB.succ_begin(),
E = MBB.succ_end(); I != E; ++I) {
if ((*I)->isLiveIn(ARM::CPSR))
return true;
}
return false;
}
void RegDefsUses::addLiveOut(const MachineBasicBlock &MBB,
const MachineBasicBlock &SuccBB) {
for (MachineBasicBlock::const_succ_iterator SI = MBB.succ_begin(),
SE = MBB.succ_end(); SI != SE; ++SI)
if (*SI != &SuccBB)
for (MachineBasicBlock::livein_iterator LI = (*SI)->livein_begin(),
LE = (*SI)->livein_end(); LI != LE; ++LI)
Uses.set(*LI);
}
void WebAssemblyExceptionInfo::discoverAndMapException(
WebAssemblyException *WE, const MachineDominatorTree &MDT,
const MachineDominanceFrontier &MDF) {
unsigned NumBlocks = 0;
unsigned NumSubExceptions = 0;
// Map blocks that belong to a catchpad / cleanuppad
MachineBasicBlock *EHPad = WE->getEHPad();
// We group catch & catch-all terminate pads together within an exception
if (WebAssembly::isCatchTerminatePad(*EHPad)) {
assert(EHPad->succ_size() == 1 &&
"Catch terminate pad has more than one successors");
changeExceptionFor(EHPad, WE);
changeExceptionFor(*(EHPad->succ_begin()), WE);
return;
}
SmallVector<MachineBasicBlock *, 8> WL;
WL.push_back(EHPad);
while (!WL.empty()) {
MachineBasicBlock *MBB = WL.pop_back_val();
// Find its outermost discovered exception. If this is a discovered block,
// check if it is already discovered to be a subexception of this exception.
WebAssemblyException *SubE = getOutermostException(MBB);
if (SubE) {
if (SubE != WE) {
// Discover a subexception of this exception.
SubE->setParentException(WE);
++NumSubExceptions;
NumBlocks += SubE->getBlocksVector().capacity();
// All blocks that belong to this subexception have been already
// discovered. Skip all of them. Add the subexception's landing pad's
// dominance frontier to the worklist.
for (auto &Frontier : MDF.find(SubE->getEHPad())->second)
if (MDT.dominates(EHPad, Frontier))
WL.push_back(Frontier);
}
continue;
}
// This is an undiscovered block. Map it to the current exception.
changeExceptionFor(MBB, WE);
++NumBlocks;
// Add successors dominated by the current BB to the worklist.
for (auto *Succ : MBB->successors())
if (MDT.dominates(EHPad, Succ))
WL.push_back(Succ);
}
WE->getSubExceptions().reserve(NumSubExceptions);
WE->reserveBlocks(NumBlocks);
}
static bool
bothUsedInPHI(const MachineBasicBlock &A,
SmallPtrSet<MachineBasicBlock*, 8> SuccsB) {
for (MachineBasicBlock::const_succ_iterator SI = A.succ_begin(),
SE = A.succ_end(); SI != SE; ++SI) {
MachineBasicBlock *BB = *SI;
if (SuccsB.count(BB) && !BB->empty() && BB->begin()->isPHI())
return true;
}
return false;
}
void MIRPrinter::convert(ModuleSlotTracker &MST,
yaml::MachineBasicBlock &YamlMBB,
const MachineBasicBlock &MBB) {
assert(MBB.getNumber() >= 0 && "Invalid MBB number");
YamlMBB.ID = (unsigned)MBB.getNumber();
if (const auto *BB = MBB.getBasicBlock()) {
if (BB->hasName()) {
YamlMBB.Name.Value = BB->getName();
} else {
int Slot = MST.getLocalSlot(BB);
if (Slot == -1)
YamlMBB.IRBlock.Value = "<badref>";
else
YamlMBB.IRBlock.Value = (Twine("%ir-block.") + Twine(Slot)).str();
}
}
YamlMBB.Alignment = MBB.getAlignment();
YamlMBB.AddressTaken = MBB.hasAddressTaken();
YamlMBB.IsLandingPad = MBB.isLandingPad();
for (const auto *SuccMBB : MBB.successors()) {
std::string Str;
raw_string_ostream StrOS(Str);
MIPrinter(StrOS, MST, RegisterMaskIds, StackObjectOperandMapping)
.printMBBReference(*SuccMBB);
YamlMBB.Successors.push_back(StrOS.str());
}
if (MBB.hasSuccessorWeights()) {
for (auto I = MBB.succ_begin(), E = MBB.succ_end(); I != E; ++I)
YamlMBB.SuccessorWeights.push_back(
yaml::UnsignedValue(MBB.getSuccWeight(I)));
}
// Print the live in registers.
const auto *TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
assert(TRI && "Expected target register info");
for (auto I = MBB.livein_begin(), E = MBB.livein_end(); I != E; ++I) {
std::string Str;
raw_string_ostream StrOS(Str);
printReg(*I, StrOS, TRI);
YamlMBB.LiveIns.push_back(StrOS.str());
}
// Print the machine instructions.
YamlMBB.Instructions.reserve(MBB.size());
std::string Str;
for (const auto &MI : MBB) {
raw_string_ostream StrOS(Str);
MIPrinter(StrOS, MST, RegisterMaskIds, StackObjectOperandMapping).print(MI);
YamlMBB.Instructions.push_back(StrOS.str());
Str.clear();
}
}
/// isPostDominatedBy - Return true if A is post dominated by B.
static bool isPostDominatedBy(MachineBasicBlock *A, MachineBasicBlock *B) {
// FIXME - Use real post dominator.
if (A->succ_size() != 2)
return false;
MachineBasicBlock::succ_iterator I = A->succ_begin();
if (B == *I)
++I;
MachineBasicBlock *OtherSuccBlock = *I;
if (OtherSuccBlock->succ_size() != 1 ||
*(OtherSuccBlock->succ_begin()) != B)
return false;
return true;
}
// Returns true if a branch over the block was inserted.
bool SIInsertSkips::skipMaskBranch(MachineInstr &MI,
MachineBasicBlock &SrcMBB) {
MachineBasicBlock *DestBB = MI.getOperand(0).getMBB();
if (!shouldSkip(**SrcMBB.succ_begin(), *DestBB))
return false;
const DebugLoc &DL = MI.getDebugLoc();
MachineBasicBlock::iterator InsPt = std::next(MI.getIterator());
BuildMI(SrcMBB, InsPt, DL, TII->get(AMDGPU::S_CBRANCH_EXECZ))
.addMBB(DestBB);
return true;
}
bool MIPrinter::canPredictSuccessors(const MachineBasicBlock &MBB) const {
SmallVector<MachineBasicBlock*,8> GuessedSuccs;
bool GuessedFallthrough;
guessSuccessors(MBB, GuessedSuccs, GuessedFallthrough);
if (GuessedFallthrough) {
const MachineFunction &MF = *MBB.getParent();
MachineFunction::const_iterator NextI = std::next(MBB.getIterator());
if (NextI != MF.end()) {
MachineBasicBlock *Next = const_cast<MachineBasicBlock*>(&*NextI);
if (!is_contained(GuessedSuccs, Next))
GuessedSuccs.push_back(Next);
}
}
if (GuessedSuccs.size() != MBB.succ_size())
return false;
return std::equal(MBB.succ_begin(), MBB.succ_end(), GuessedSuccs.begin());
}
void SPScope::buildfcfg(void) {
std::set<const MachineBasicBlock *> body(++Blocks.begin(), Blocks.end());
std::vector<Edge> outedges;
for (unsigned i=0; i<Blocks.size(); i++) {
MachineBasicBlock *MBB = Blocks[i];
if (HeaderMap.count(MBB)) {
const SPScope *subloop = HeaderMap[MBB];
// successors of the loop
outedges.insert(outedges.end(),
subloop->ExitEdges.begin(),
subloop->ExitEdges.end());
} else {
// simple block
for (MachineBasicBlock::succ_iterator si = MBB->succ_begin(),
se = MBB->succ_end(); si != se; ++si) {
outedges.push_back(std::make_pair(MBB, *si));
}
}
Node &n = FCFG.getNodeFor(MBB);
for (unsigned i=0; i<outedges.size(); i++) {
const MachineBasicBlock *succ = outedges[i].second;
if (body.count(succ)) {
Node &ns = FCFG.getNodeFor(succ);
n.connect(ns, outedges[i]);
} else {
if (succ != getHeader()) {
// record exit edges
FCFG.toexit(n, outedges[i]);
} else {
// we don't need back edges recorded
FCFG.toexit(n);
}
}
}
// special case: top-level loop has no exit/backedge
if (outedges.empty()) {
assert(isTopLevel());
FCFG.toexit(n);
}
outedges.clear();
}
}
/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
/// after it, replacing it with an unconditional branch to NewDest.
void
TargetInstrInfoImpl::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail,
MachineBasicBlock *NewDest) const {
MachineBasicBlock *MBB = Tail->getParent();
// Remove all the old successors of MBB from the CFG.
while (!MBB->succ_empty())
MBB->removeSuccessor(MBB->succ_begin());
// Remove all the dead instructions from the end of MBB.
MBB->erase(Tail, MBB->end());
// If MBB isn't immediately before MBB, insert a branch to it.
if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest))
InsertBranch(*MBB, NewDest, 0, SmallVector<MachineOperand, 0>(),
Tail->getDebugLoc());
MBB->addSuccessor(NewDest);
}
bool LiveVariables::isLiveOut(unsigned Reg, const MachineBasicBlock &MBB) {
LiveVariables::VarInfo &VI = getVarInfo(Reg);
// Loop over all of the successors of the basic block, checking to see if
// the value is either live in the block, or if it is killed in the block.
SmallVector<MachineBasicBlock*, 8> OpSuccBlocks;
for (MachineBasicBlock::const_succ_iterator SI = MBB.succ_begin(),
E = MBB.succ_end(); SI != E; ++SI) {
MachineBasicBlock *SuccMBB = *SI;
// Is it alive in this successor?
unsigned SuccIdx = SuccMBB->getNumber();
if (VI.AliveBlocks.test(SuccIdx))
return true;
OpSuccBlocks.push_back(SuccMBB);
}
// Check to see if this value is live because there is a use in a successor
// that kills it.
switch (OpSuccBlocks.size()) {
case 1: {
MachineBasicBlock *SuccMBB = OpSuccBlocks[0];
for (unsigned i = 0, e = VI.Kills.size(); i != e; ++i)
if (VI.Kills[i]->getParent() == SuccMBB)
return true;
break;
}
case 2: {
MachineBasicBlock *SuccMBB1 = OpSuccBlocks[0], *SuccMBB2 = OpSuccBlocks[1];
for (unsigned i = 0, e = VI.Kills.size(); i != e; ++i)
if (VI.Kills[i]->getParent() == SuccMBB1 ||
VI.Kills[i]->getParent() == SuccMBB2)
return true;
break;
}
default:
std::sort(OpSuccBlocks.begin(), OpSuccBlocks.end());
for (unsigned i = 0, e = VI.Kills.size(); i != e; ++i)
if (std::binary_search(OpSuccBlocks.begin(), OpSuccBlocks.end(),
VI.Kills[i]->getParent()))
return true;
}
return false;
}
bool SILowerControlFlowPass::shouldSkip(MachineBasicBlock *From,
MachineBasicBlock *To) {
unsigned NumInstr = 0;
for (MachineBasicBlock *MBB = From; MBB != To && !MBB->succ_empty();
MBB = *MBB->succ_begin()) {
for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end();
NumInstr < SkipThreshold && I != E; ++I) {
if (I->isBundle() || !I->isBundled())
if (++NumInstr >= SkipThreshold)
return true;
}
}
return false;
}
bool
TailDuplicatePass::canCompletelyDuplicateBB(MachineBasicBlock &BB) {
SmallPtrSet<MachineBasicBlock*, 8> Succs(BB.succ_begin(), BB.succ_end());
for (MachineBasicBlock::pred_iterator PI = BB.pred_begin(),
PE = BB.pred_end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (PredBB->succ_size() > 1)
return false;
MachineBasicBlock *PredTBB = NULL, *PredFBB = NULL;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
return false;
if (!PredCond.empty())
return false;
}
return true;
}
void LoopSplitter::dumpOddTerminators() {
for (MachineFunction::iterator bbItr = mf->begin(), bbEnd = mf->end();
bbItr != bbEnd; ++bbItr) {
MachineBasicBlock *mbb = &*bbItr;
MachineBasicBlock *a = 0, *b = 0;
SmallVector<MachineOperand, 4> c;
if (tii->AnalyzeBranch(*mbb, a, b, c)) {
dbgs() << "MBB#" << mbb->getNumber() << " has multiway terminator.\n";
dbgs() << " Terminators:\n";
for (MachineBasicBlock::iterator iItr = mbb->begin(), iEnd = mbb->end();
iItr != iEnd; ++iItr) {
MachineInstr *instr= &*iItr;
dbgs() << " " << *instr << "";
}
dbgs() << "\n Listed successors: [ ";
for (MachineBasicBlock::succ_iterator sItr = mbb->succ_begin(), sEnd = mbb->succ_end();
sItr != sEnd; ++sItr) {
MachineBasicBlock *succMBB = *sItr;
dbgs() << succMBB->getNumber() << " ";
}
dbgs() << "]\n\n";
}
}
}
void MIPrinter::print(const MachineBasicBlock &MBB) {
assert(MBB.getNumber() >= 0 && "Invalid MBB number");
OS << "bb." << MBB.getNumber();
bool HasAttributes = false;
if (const auto *BB = MBB.getBasicBlock()) {
if (BB->hasName()) {
OS << "." << BB->getName();
} else {
HasAttributes = true;
OS << " (";
int Slot = MST.getLocalSlot(BB);
if (Slot == -1)
OS << "<ir-block badref>";
else
OS << (Twine("%ir-block.") + Twine(Slot)).str();
}
}
if (MBB.hasAddressTaken()) {
OS << (HasAttributes ? ", " : " (");
OS << "address-taken";
HasAttributes = true;
}
if (MBB.isEHPad()) {
OS << (HasAttributes ? ", " : " (");
OS << "landing-pad";
HasAttributes = true;
}
if (MBB.getAlignment()) {
OS << (HasAttributes ? ", " : " (");
OS << "align " << MBB.getAlignment();
HasAttributes = true;
}
if (HasAttributes)
OS << ")";
OS << ":\n";
bool HasLineAttributes = false;
// Print the successors
if (!MBB.succ_empty()) {
OS.indent(2) << "successors: ";
for (auto I = MBB.succ_begin(), E = MBB.succ_end(); I != E; ++I) {
if (I != MBB.succ_begin())
OS << ", ";
printMBBReference(**I);
if (MBB.hasSuccessorWeights())
OS << '(' << MBB.getSuccWeight(I) << ')';
}
OS << "\n";
HasLineAttributes = true;
}
// Print the live in registers.
const auto *TRI = MBB.getParent()->getSubtarget().getRegisterInfo();
assert(TRI && "Expected target register info");
if (!MBB.livein_empty()) {
OS.indent(2) << "liveins: ";
bool First = true;
for (unsigned LI : MBB.liveins()) {
if (!First)
OS << ", ";
First = false;
printReg(LI, OS, TRI);
}
OS << "\n";
HasLineAttributes = true;
}
if (HasLineAttributes)
OS << "\n";
bool IsInBundle = false;
for (auto I = MBB.instr_begin(), E = MBB.instr_end(); I != E; ++I) {
const MachineInstr &MI = *I;
if (IsInBundle && !MI.isInsideBundle()) {
OS.indent(2) << "}\n";
IsInBundle = false;
}
OS.indent(IsInBundle ? 4 : 2);
print(MI);
if (!IsInBundle && MI.getFlag(MachineInstr::BundledSucc)) {
OS << " {";
IsInBundle = true;
}
OS << "\n";
}
if (IsInBundle)
OS.indent(2) << "}\n";
}
/// TailDuplicate - If it is profitable, duplicate TailBB's contents in each
/// of its predecessors.
bool
TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF,
SmallVector<MachineBasicBlock*, 8> &TDBBs,
SmallVector<MachineInstr*, 16> &Copies) {
if (!shouldTailDuplicate(MF, *TailBB))
return false;
DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
// Iterate through all the unique predecessors and tail-duplicate this
// block into them, if possible. Copying the list ahead of time also
// avoids trouble with the predecessor list reallocating.
bool Changed = false;
SmallSetVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
DenseSet<unsigned> UsedByPhi;
getRegsUsedByPHIs(*TailBB, &UsedByPhi);
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
assert(TailBB != PredBB &&
"Single-block loop should have been rejected earlier!");
// EH edges are ignored by AnalyzeBranch.
if (PredBB->succ_size() > 1)
continue;
MachineBasicBlock *PredTBB, *PredFBB;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
continue;
if (!PredCond.empty())
continue;
// Don't duplicate into a fall-through predecessor (at least for now).
if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
continue;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From Succ: " << *TailBB);
TDBBs.push_back(PredBB);
// Remove PredBB's unconditional branch.
TII->RemoveBranch(*PredBB);
// Clone the contents of TailBB into PredBB.
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
while (I != TailBB->end()) {
MachineInstr *MI = &*I;
++I;
if (MI->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true);
} else {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
DuplicateInstruction(MI, TailBB, PredBB, MF, LocalVRMap, UsedByPhi);
}
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
// Simplify
TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true);
NumInstrDups += TailBB->size() - 1; // subtract one for removed branch
// Update the CFG.
PredBB->removeSuccessor(PredBB->succ_begin());
assert(PredBB->succ_empty() &&
"TailDuplicate called on block with multiple successors!");
for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(),
E = TailBB->succ_end(); I != E; ++I)
PredBB->addSuccessor(*I);
Changed = true;
++NumTailDups;
}
// If TailBB was duplicated into all its predecessors except for the prior
// block, which falls through unconditionally, move the contents of this
// block into the prior block.
MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB));
MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0;
SmallVector<MachineOperand, 4> PriorCond;
// This has to check PrevBB->succ_size() because EH edges are ignored by
// AnalyzeBranch.
if (PrevBB->succ_size() == 1 &&
!TII->AnalyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond, true) &&
PriorCond.empty() && !PriorTBB && TailBB->pred_size() == 1 &&
!TailBB->hasAddressTaken()) {
DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
<< "From MBB: " << *TailBB);
if (PreRegAlloc) {
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos, UsedByPhi, true);
if (MI->getParent())
MI->eraseFromParent();
}
// Now copy the non-PHI instructions.
while (I != TailBB->end()) {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
MachineInstr *MI = &*I++;
DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap, UsedByPhi);
MI->eraseFromParent();
}
MachineBasicBlock::iterator Loc = PrevBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PrevBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first)
.addReg(CopyInfos[i].second));
}
} else {
// No PHIs to worry about, just splice the instructions over.
PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
}
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);
TDBBs.push_back(PrevBB);
Changed = true;
}
// If this is after register allocation, there are no phis to fix.
if (!PreRegAlloc)
return Changed;
// If we made no changes so far, we are safe.
if (!Changed)
return Changed;
// Handle the nasty case in that we duplicated a block that is part of a loop
// into some but not all of its predecessors. For example:
// 1 -> 2 <-> 3 |
// \ |
// \---> rest |
// if we duplicate 2 into 1 but not into 3, we end up with
// 12 -> 3 <-> 2 -> rest |
// \ / |
// \----->-----/ |
// If there was a "var = phi(1, 3)" in 2, it has to be ultimately replaced
// with a phi in 3 (which now dominates 2).
// What we do here is introduce a copy in 3 of the register defined by the
// phi, just like when we are duplicating 2 into 3, but we don't copy any
// real instructions or remove the 3 -> 2 edge from the phi in 2.
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (std::find(TDBBs.begin(), TDBBs.end(), PredBB) != TDBBs.end())
continue;
// EH edges
if (PredBB->succ_size() != 1)
continue;
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false);
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
}
return Changed;
}
/// TailDuplicateBlocks - Look for small blocks that are unconditionally
/// branched to and do not fall through. Tail-duplicate their instructions
/// into their predecessors to eliminate (dynamic) branches.
bool TailDuplicatePass::TailDuplicateBlocks(MachineFunction &MF) {
bool MadeChange = false;
if (PreRegAlloc && TailDupVerify) {
DEBUG(dbgs() << "\n*** Before tail-duplicating\n");
VerifyPHIs(MF, true);
}
SmallVector<MachineInstr*, 8> NewPHIs;
MachineSSAUpdater SSAUpdate(MF, &NewPHIs);
for (MachineFunction::iterator I = ++MF.begin(), E = MF.end(); I != E; ) {
MachineBasicBlock *MBB = I++;
if (NumTails == TailDupLimit)
break;
// Save the successors list.
SmallSetVector<MachineBasicBlock*, 8> Succs(MBB->succ_begin(),
MBB->succ_end());
SmallVector<MachineBasicBlock*, 8> TDBBs;
SmallVector<MachineInstr*, 16> Copies;
if (TailDuplicate(MBB, MF, TDBBs, Copies)) {
++NumTails;
// TailBB's immediate successors are now successors of those predecessors
// which duplicated TailBB. Add the predecessors as sources to the PHI
// instructions.
bool isDead = MBB->pred_empty();
if (PreRegAlloc)
UpdateSuccessorsPHIs(MBB, isDead, TDBBs, Succs);
// If it is dead, remove it.
if (isDead) {
NumInstrDups -= MBB->size();
RemoveDeadBlock(MBB);
++NumDeadBlocks;
}
// Update SSA form.
if (!SSAUpdateVRs.empty()) {
for (unsigned i = 0, e = SSAUpdateVRs.size(); i != e; ++i) {
unsigned VReg = SSAUpdateVRs[i];
SSAUpdate.Initialize(VReg);
// If the original definition is still around, add it as an available
// value.
MachineInstr *DefMI = MRI->getVRegDef(VReg);
MachineBasicBlock *DefBB = 0;
if (DefMI) {
DefBB = DefMI->getParent();
SSAUpdate.AddAvailableValue(DefBB, VReg);
}
// Add the new vregs as available values.
DenseMap<unsigned, AvailableValsTy>::iterator LI =
SSAUpdateVals.find(VReg);
for (unsigned j = 0, ee = LI->second.size(); j != ee; ++j) {
MachineBasicBlock *SrcBB = LI->second[j].first;
unsigned SrcReg = LI->second[j].second;
SSAUpdate.AddAvailableValue(SrcBB, SrcReg);
}
// Rewrite uses that are outside of the original def's block.
MachineRegisterInfo::use_iterator UI = MRI->use_begin(VReg);
while (UI != MRI->use_end()) {
MachineOperand &UseMO = UI.getOperand();
MachineInstr *UseMI = &*UI;
++UI;
if (UseMI->isDebugValue()) {
// SSAUpdate can replace the use with an undef. That creates
// a debug instruction that is a kill.
// FIXME: Should it SSAUpdate job to delete debug instructions
// instead of replacing the use with undef?
UseMI->eraseFromParent();
continue;
}
if (UseMI->getParent() == DefBB && !UseMI->isPHI())
continue;
SSAUpdate.RewriteUse(UseMO);
}
}
SSAUpdateVRs.clear();
SSAUpdateVals.clear();
}
// Eliminate some of the copies inserted by tail duplication to maintain
// SSA form.
for (unsigned i = 0, e = Copies.size(); i != e; ++i) {
MachineInstr *Copy = Copies[i];
if (!Copy->isCopy())
continue;
unsigned Dst = Copy->getOperand(0).getReg();
unsigned Src = Copy->getOperand(1).getReg();
MachineRegisterInfo::use_iterator UI = MRI->use_begin(Src);
if (++UI == MRI->use_end()) {
// Copy is the only use. Do trivial copy propagation here.
MRI->replaceRegWith(Dst, Src);
Copy->eraseFromParent();
}
}
if (PreRegAlloc && TailDupVerify)
VerifyPHIs(MF, false);
MadeChange = true;
}
}
NumAddedPHIs += NewPHIs.size();
return MadeChange;
}
bool HexagonNewValueJump::runOnMachineFunction(MachineFunction &MF) {
DEBUG(dbgs() << "********** Hexagon New Value Jump **********\n"
<< "********** Function: "
<< MF.getName() << "\n");
if (skipFunction(*MF.getFunction()))
return false;
// If we move NewValueJump before register allocation we'll need live variable
// analysis here too.
QII = static_cast<const HexagonInstrInfo *>(MF.getSubtarget().getInstrInfo());
QRI = static_cast<const HexagonRegisterInfo *>(
MF.getSubtarget().getRegisterInfo());
MBPI = &getAnalysis<MachineBranchProbabilityInfo>();
if (DisableNewValueJumps) {
return false;
}
int nvjCount = DbgNVJCount;
int nvjGenerated = 0;
// Loop through all the bb's of the function
for (MachineFunction::iterator MBBb = MF.begin(), MBBe = MF.end();
MBBb != MBBe; ++MBBb) {
MachineBasicBlock *MBB = &*MBBb;
DEBUG(dbgs() << "** dumping bb ** "
<< MBB->getNumber() << "\n");
DEBUG(MBB->dump());
DEBUG(dbgs() << "\n" << "********** dumping instr bottom up **********\n");
bool foundJump = false;
bool foundCompare = false;
bool invertPredicate = false;
unsigned predReg = 0; // predicate reg of the jump.
unsigned cmpReg1 = 0;
int cmpOp2 = 0;
bool MO1IsKill = false;
bool MO2IsKill = false;
MachineBasicBlock::iterator jmpPos;
MachineBasicBlock::iterator cmpPos;
MachineInstr *cmpInstr = nullptr, *jmpInstr = nullptr;
MachineBasicBlock *jmpTarget = nullptr;
bool afterRA = false;
bool isSecondOpReg = false;
bool isSecondOpNewified = false;
// Traverse the basic block - bottom up
for (MachineBasicBlock::iterator MII = MBB->end(), E = MBB->begin();
MII != E;) {
MachineInstr &MI = *--MII;
if (MI.isDebugValue()) {
continue;
}
if ((nvjCount == 0) || (nvjCount > -1 && nvjCount <= nvjGenerated))
break;
DEBUG(dbgs() << "Instr: "; MI.dump(); dbgs() << "\n");
if (!foundJump && (MI.getOpcode() == Hexagon::J2_jumpt ||
MI.getOpcode() == Hexagon::J2_jumpf ||
MI.getOpcode() == Hexagon::J2_jumptnewpt ||
MI.getOpcode() == Hexagon::J2_jumptnew ||
MI.getOpcode() == Hexagon::J2_jumpfnewpt ||
MI.getOpcode() == Hexagon::J2_jumpfnew)) {
// This is where you would insert your compare and
// instr that feeds compare
jmpPos = MII;
jmpInstr = &MI;
predReg = MI.getOperand(0).getReg();
afterRA = TargetRegisterInfo::isPhysicalRegister(predReg);
// If ifconverter had not messed up with the kill flags of the
// operands, the following check on the kill flag would suffice.
// if(!jmpInstr->getOperand(0).isKill()) break;
// This predicate register is live out out of BB
// this would only work if we can actually use Live
// variable analysis on phy regs - but LLVM does not
// provide LV analysis on phys regs.
//if(LVs.isLiveOut(predReg, *MBB)) break;
// Get all the successors of this block - which will always
// be 2. Check if the predicate register is live in in those
// successor. If yes, we can not delete the predicate -
// I am doing this only because LLVM does not provide LiveOut
// at the BB level.
bool predLive = false;
for (MachineBasicBlock::const_succ_iterator SI = MBB->succ_begin(),
SIE = MBB->succ_end(); SI != SIE; ++SI) {
MachineBasicBlock* succMBB = *SI;
if (succMBB->isLiveIn(predReg)) {
predLive = true;
}
}
if (predLive)
break;
if (!MI.getOperand(1).isMBB())
continue;
jmpTarget = MI.getOperand(1).getMBB();
foundJump = true;
if (MI.getOpcode() == Hexagon::J2_jumpf ||
MI.getOpcode() == Hexagon::J2_jumpfnewpt ||
MI.getOpcode() == Hexagon::J2_jumpfnew) {
invertPredicate = true;
}
continue;
}
// No new value jump if there is a barrier. A barrier has to be in its
// own packet. A barrier has zero operands. We conservatively bail out
// here if we see any instruction with zero operands.
if (foundJump && MI.getNumOperands() == 0)
break;
if (foundJump && !foundCompare && MI.getOperand(0).isReg() &&
MI.getOperand(0).getReg() == predReg) {
// Not all compares can be new value compare. Arch Spec: 7.6.1.1
if (isNewValueJumpCandidate(MI)) {
assert(
(MI.getDesc().isCompare()) &&
"Only compare instruction can be collapsed into New Value Jump");
isSecondOpReg = MI.getOperand(2).isReg();
if (!canCompareBeNewValueJump(QII, QRI, MII, predReg, isSecondOpReg,
afterRA, jmpPos, MF))
break;
cmpInstr = &MI;
cmpPos = MII;
foundCompare = true;
// We need cmpReg1 and cmpOp2(imm or reg) while building
// new value jump instruction.
cmpReg1 = MI.getOperand(1).getReg();
if (MI.getOperand(1).isKill())
MO1IsKill = true;
if (isSecondOpReg) {
cmpOp2 = MI.getOperand(2).getReg();
if (MI.getOperand(2).isKill())
MO2IsKill = true;
} else
cmpOp2 = MI.getOperand(2).getImm();
continue;
}
}
if (foundCompare && foundJump) {
// If "common" checks fail, bail out on this BB.
if (!commonChecksToProhibitNewValueJump(afterRA, MII))
break;
bool foundFeeder = false;
MachineBasicBlock::iterator feederPos = MII;
if (MI.getOperand(0).isReg() && MI.getOperand(0).isDef() &&
(MI.getOperand(0).getReg() == cmpReg1 ||
(isSecondOpReg &&
MI.getOperand(0).getReg() == (unsigned)cmpOp2))) {
unsigned feederReg = MI.getOperand(0).getReg();
// First try to see if we can get the feeder from the first operand
// of the compare. If we can not, and if secondOpReg is true
// (second operand of the compare is also register), try that one.
// TODO: Try to come up with some heuristic to figure out which
// feeder would benefit.
if (feederReg == cmpReg1) {
if (!canBeFeederToNewValueJump(QII, QRI, MII, jmpPos, cmpPos, MF)) {
if (!isSecondOpReg)
break;
else
continue;
} else
foundFeeder = true;
}
if (!foundFeeder &&
isSecondOpReg &&
feederReg == (unsigned) cmpOp2)
if (!canBeFeederToNewValueJump(QII, QRI, MII, jmpPos, cmpPos, MF))
break;
if (isSecondOpReg) {
// In case of CMPLT, or CMPLTU, or EQ with the second register
// to newify, swap the operands.
if (cmpInstr->getOpcode() == Hexagon::C2_cmpeq &&
feederReg == (unsigned) cmpOp2) {
unsigned tmp = cmpReg1;
bool tmpIsKill = MO1IsKill;
cmpReg1 = cmpOp2;
MO1IsKill = MO2IsKill;
cmpOp2 = tmp;
MO2IsKill = tmpIsKill;
}
// Now we have swapped the operands, all we need to check is,
// if the second operand (after swap) is the feeder.
// And if it is, make a note.
if (feederReg == (unsigned)cmpOp2)
isSecondOpNewified = true;
}
// Now that we are moving feeder close the jump,
// make sure we are respecting the kill values of
// the operands of the feeder.
bool updatedIsKill = false;
for (unsigned i = 0; i < MI.getNumOperands(); i++) {
MachineOperand &MO = MI.getOperand(i);
if (MO.isReg() && MO.isUse()) {
unsigned feederReg = MO.getReg();
for (MachineBasicBlock::iterator localII = feederPos,
end = jmpPos; localII != end; localII++) {
MachineInstr &localMI = *localII;
for (unsigned j = 0; j < localMI.getNumOperands(); j++) {
MachineOperand &localMO = localMI.getOperand(j);
if (localMO.isReg() && localMO.isUse() &&
localMO.isKill() && feederReg == localMO.getReg()) {
// We found that there is kill of a use register
// Set up a kill flag on the register
localMO.setIsKill(false);
MO.setIsKill();
updatedIsKill = true;
break;
}
}
if (updatedIsKill) break;
}
}
if (updatedIsKill) break;
}
MBB->splice(jmpPos, MI.getParent(), MI);
MBB->splice(jmpPos, MI.getParent(), cmpInstr);
DebugLoc dl = MI.getDebugLoc();
MachineInstr *NewMI;
assert((isNewValueJumpCandidate(*cmpInstr)) &&
"This compare is not a New Value Jump candidate.");
unsigned opc = getNewValueJumpOpcode(cmpInstr, cmpOp2,
isSecondOpNewified,
jmpTarget, MBPI);
if (invertPredicate)
opc = QII->getInvertedPredicatedOpcode(opc);
if (isSecondOpReg)
NewMI = BuildMI(*MBB, jmpPos, dl,
QII->get(opc))
.addReg(cmpReg1, getKillRegState(MO1IsKill))
.addReg(cmpOp2, getKillRegState(MO2IsKill))
.addMBB(jmpTarget);
else if ((cmpInstr->getOpcode() == Hexagon::C2_cmpeqi ||
cmpInstr->getOpcode() == Hexagon::C2_cmpgti) &&
cmpOp2 == -1 )
// Corresponding new-value compare jump instructions don't have the
// operand for -1 immediate value.
NewMI = BuildMI(*MBB, jmpPos, dl,
QII->get(opc))
.addReg(cmpReg1, getKillRegState(MO1IsKill))
.addMBB(jmpTarget);
else
NewMI = BuildMI(*MBB, jmpPos, dl,
QII->get(opc))
.addReg(cmpReg1, getKillRegState(MO1IsKill))
.addImm(cmpOp2)
.addMBB(jmpTarget);
assert(NewMI && "New Value Jump Instruction Not created!");
(void)NewMI;
if (cmpInstr->getOperand(0).isReg() &&
cmpInstr->getOperand(0).isKill())
cmpInstr->getOperand(0).setIsKill(false);
if (cmpInstr->getOperand(1).isReg() &&
cmpInstr->getOperand(1).isKill())
cmpInstr->getOperand(1).setIsKill(false);
cmpInstr->eraseFromParent();
jmpInstr->eraseFromParent();
++nvjGenerated;
++NumNVJGenerated;
break;
}
}
}
}
return true;
}
/// Splits a MachineBasicBlock to branch before \p SplitBefore. The original
/// branch is \p OrigBranch. The target of the new branch can either be the same
/// as the target of the original branch or the fallthrough successor of the
/// original block as determined by \p BranchToFallThrough. The branch
/// conditions will be inverted according to \p InvertNewBranch and
/// \p InvertOrigBranch. If an instruction that previously fed the branch is to
/// be deleted, it is provided in \p MIToDelete and \p NewCond will be used as
/// the branch condition. The branch probabilities will be set if the
/// MachineBranchProbabilityInfo isn't null.
static bool splitMBB(BlockSplitInfo &BSI) {
assert(BSI.allInstrsInSameMBB() &&
"All instructions must be in the same block.");
MachineBasicBlock *ThisMBB = BSI.OrigBranch->getParent();
MachineFunction *MF = ThisMBB->getParent();
MachineRegisterInfo *MRI = &MF->getRegInfo();
assert(MRI->isSSA() && "Can only do this while the function is in SSA form.");
if (ThisMBB->succ_size() != 2) {
LLVM_DEBUG(
dbgs() << "Don't know how to handle blocks that don't have exactly"
<< " two successors.\n");
return false;
}
const PPCInstrInfo *TII = MF->getSubtarget<PPCSubtarget>().getInstrInfo();
unsigned OrigBROpcode = BSI.OrigBranch->getOpcode();
unsigned InvertedOpcode =
OrigBROpcode == PPC::BC
? PPC::BCn
: OrigBROpcode == PPC::BCn
? PPC::BC
: OrigBROpcode == PPC::BCLR ? PPC::BCLRn : PPC::BCLR;
unsigned NewBROpcode = BSI.InvertNewBranch ? InvertedOpcode : OrigBROpcode;
MachineBasicBlock *OrigTarget = BSI.OrigBranch->getOperand(1).getMBB();
MachineBasicBlock *OrigFallThrough = OrigTarget == *ThisMBB->succ_begin()
? *ThisMBB->succ_rbegin()
: *ThisMBB->succ_begin();
MachineBasicBlock *NewBRTarget =
BSI.BranchToFallThrough ? OrigFallThrough : OrigTarget;
BranchProbability ProbToNewTarget =
!BSI.MBPI ? BranchProbability::getUnknown()
: BSI.MBPI->getEdgeProbability(ThisMBB, NewBRTarget);
// Create a new basic block.
MachineBasicBlock::iterator InsertPoint = BSI.SplitBefore;
const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
MachineFunction::iterator It = ThisMBB->getIterator();
MachineBasicBlock *NewMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MF->insert(++It, NewMBB);
// Move everything after SplitBefore into the new block.
NewMBB->splice(NewMBB->end(), ThisMBB, InsertPoint, ThisMBB->end());
NewMBB->transferSuccessors(ThisMBB);
// Add the two successors to ThisMBB. The probabilities come from the
// existing blocks if available.
ThisMBB->addSuccessor(NewBRTarget, ProbToNewTarget);
ThisMBB->addSuccessor(NewMBB, ProbToNewTarget.getCompl());
// Add the branches to ThisMBB.
BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(),
TII->get(NewBROpcode))
.addReg(BSI.SplitCond->getOperand(0).getReg())
.addMBB(NewBRTarget);
BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(),
TII->get(PPC::B))
.addMBB(NewMBB);
if (BSI.MIToDelete)
BSI.MIToDelete->eraseFromParent();
// Change the condition on the original branch and invert it if requested.
auto FirstTerminator = NewMBB->getFirstTerminator();
if (BSI.NewCond) {
assert(FirstTerminator->getOperand(0).isReg() &&
"Can't update condition of unconditional branch.");
FirstTerminator->getOperand(0).setReg(BSI.NewCond->getOperand(0).getReg());
}
if (BSI.InvertOrigBranch)
FirstTerminator->setDesc(TII->get(InvertedOpcode));
// If any of the PHIs in the successors of NewMBB reference values that
// now come from NewMBB, they need to be updated.
for (auto *Succ : NewMBB->successors()) {
updatePHIs(Succ, ThisMBB, NewMBB, MRI);
}
addIncomingValuesToPHIs(NewBRTarget, ThisMBB, NewMBB, MRI);
LLVM_DEBUG(dbgs() << "After splitting, ThisMBB:\n"; ThisMBB->dump());
LLVM_DEBUG(dbgs() << "NewMBB:\n"; NewMBB->dump());
LLVM_DEBUG(dbgs() << "New branch-to block:\n"; NewBRTarget->dump());
return true;
}
void StackColoring::calculateLocalLiveness() {
// Perform a standard reverse dataflow computation to solve for
// global liveness. The BEGIN set here is equivalent to KILL in the standard
// formulation, and END is equivalent to GEN. The result of this computation
// is a map from blocks to bitvectors where the bitvectors represent which
// allocas are live in/out of that block.
SmallPtrSet<MachineBasicBlock*, 8> BBSet(BasicBlockNumbering.begin(),
BasicBlockNumbering.end());
unsigned NumSSMIters = 0;
bool changed = true;
while (changed) {
changed = false;
++NumSSMIters;
SmallPtrSet<MachineBasicBlock*, 8> NextBBSet;
for (SmallVector<MachineBasicBlock*, 8>::iterator
PI = BasicBlockNumbering.begin(), PE = BasicBlockNumbering.end();
PI != PE; ++PI) {
MachineBasicBlock *BB = *PI;
if (!BBSet.count(BB)) continue;
BitVector LocalLiveIn;
BitVector LocalLiveOut;
// Forward propagation from begins to ends.
for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
PE = BB->pred_end(); PI != PE; ++PI)
LocalLiveIn |= BlockLiveness[*PI].LiveOut;
LocalLiveIn |= BlockLiveness[BB].End;
LocalLiveIn.reset(BlockLiveness[BB].Begin);
// Reverse propagation from ends to begins.
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
LocalLiveOut |= BlockLiveness[*SI].LiveIn;
LocalLiveOut |= BlockLiveness[BB].Begin;
LocalLiveOut.reset(BlockLiveness[BB].End);
LocalLiveIn |= LocalLiveOut;
LocalLiveOut |= LocalLiveIn;
// After adopting the live bits, we need to turn-off the bits which
// are de-activated in this block.
LocalLiveOut.reset(BlockLiveness[BB].End);
LocalLiveIn.reset(BlockLiveness[BB].Begin);
// If we have both BEGIN and END markers in the same basic block then
// we know that the BEGIN marker comes after the END, because we already
// handle the case where the BEGIN comes before the END when collecting
// the markers (and building the BEGIN/END vectore).
// Want to enable the LIVE_IN and LIVE_OUT of slots that have both
// BEGIN and END because it means that the value lives before and after
// this basic block.
BitVector LocalEndBegin = BlockLiveness[BB].End;
LocalEndBegin &= BlockLiveness[BB].Begin;
LocalLiveIn |= LocalEndBegin;
LocalLiveOut |= LocalEndBegin;
if (LocalLiveIn.test(BlockLiveness[BB].LiveIn)) {
changed = true;
BlockLiveness[BB].LiveIn |= LocalLiveIn;
for (MachineBasicBlock::pred_iterator PI = BB->pred_begin(),
PE = BB->pred_end(); PI != PE; ++PI)
NextBBSet.insert(*PI);
}
if (LocalLiveOut.test(BlockLiveness[BB].LiveOut)) {
changed = true;
BlockLiveness[BB].LiveOut |= LocalLiveOut;
for (MachineBasicBlock::succ_iterator SI = BB->succ_begin(),
SE = BB->succ_end(); SI != SE; ++SI)
NextBBSet.insert(*SI);
}
}
BBSet = NextBBSet;
}// while changed.
}
bool SIFixSGPRLiveRanges::runOnMachineFunction(MachineFunction &MF) {
MachineRegisterInfo &MRI = MF.getRegInfo();
const TargetInstrInfo *TII = MF.getSubtarget().getInstrInfo();
const SIRegisterInfo *TRI = static_cast<const SIRegisterInfo *>(
MF.getSubtarget().getRegisterInfo());
bool MadeChange = false;
MachinePostDominatorTree *PDT = &getAnalysis<MachinePostDominatorTree>();
std::vector<std::pair<unsigned, LiveRange *>> SGPRLiveRanges;
LiveIntervals *LIS = &getAnalysis<LiveIntervals>();
LiveVariables *LV = getAnalysisIfAvailable<LiveVariables>();
MachineBasicBlock *Entry = MF.begin();
// Use a depth first order so that in SSA, we encounter all defs before
// uses. Once the defs of the block have been found, attempt to insert
// SGPR_USE instructions in successor blocks if required.
for (MachineBasicBlock *MBB : depth_first(Entry)) {
for (const MachineInstr &MI : *MBB) {
for (const MachineOperand &MO : MI.defs()) {
if (MO.isImplicit())
continue;
unsigned Def = MO.getReg();
if (TargetRegisterInfo::isVirtualRegister(Def)) {
if (TRI->isSGPRClass(MRI.getRegClass(Def))) {
// Only consider defs that are live outs. We don't care about def /
// use within the same block.
LiveRange &LR = LIS->getInterval(Def);
if (LIS->isLiveOutOfMBB(LR, MBB))
SGPRLiveRanges.push_back(std::make_pair(Def, &LR));
}
} else if (TRI->isSGPRClass(TRI->getPhysRegClass(Def))) {
SGPRLiveRanges.push_back(std::make_pair(Def, &LIS->getRegUnit(Def)));
}
}
}
if (MBB->succ_size() < 2)
continue;
// We have structured control flow, so the number of successors should be
// two.
assert(MBB->succ_size() == 2);
MachineBasicBlock *SuccA = *MBB->succ_begin();
MachineBasicBlock *SuccB = *(++MBB->succ_begin());
MachineBasicBlock *NCD = PDT->findNearestCommonDominator(SuccA, SuccB);
if (!NCD)
continue;
MachineBasicBlock::iterator NCDTerm = NCD->getFirstTerminator();
if (NCDTerm != NCD->end() && NCDTerm->getOpcode() == AMDGPU::SI_ELSE) {
assert(NCD->succ_size() == 2);
// We want to make sure we insert the Use after the ENDIF, not after
// the ELSE.
NCD = PDT->findNearestCommonDominator(*NCD->succ_begin(),
*(++NCD->succ_begin()));
}
for (std::pair<unsigned, LiveRange*> RegLR : SGPRLiveRanges) {
unsigned Reg = RegLR.first;
LiveRange *LR = RegLR.second;
// FIXME: We could be smarter here. If the register is Live-In to one
// block, but the other doesn't have any SGPR defs, then there won't be a
// conflict. Also, if the branch condition is uniform then there will be
// no conflict.
bool LiveInToA = LIS->isLiveInToMBB(*LR, SuccA);
bool LiveInToB = LIS->isLiveInToMBB(*LR, SuccB);
if (!LiveInToA && !LiveInToB) {
DEBUG(dbgs() << PrintReg(Reg, TRI, 0)
<< " is live into neither successor\n");
continue;
}
if (LiveInToA && LiveInToB) {
DEBUG(dbgs() << PrintReg(Reg, TRI, 0)
<< " is live into both successors\n");
continue;
}
// This interval is live in to one successor, but not the other, so
// we need to update its range so it is live in to both.
DEBUG(dbgs() << "Possible SGPR conflict detected for "
<< PrintReg(Reg, TRI, 0) << " in " << *LR
<< " BB#" << SuccA->getNumber() << ", BB#"
<< SuccB->getNumber()
<< " with NCD = BB#" << NCD->getNumber() << '\n');
assert(TargetRegisterInfo::isVirtualRegister(Reg) &&
"Not expecting to extend live range of physreg");
// FIXME: Need to figure out how to update LiveRange here so this pass
// will be able to preserve LiveInterval analysis.
MachineInstr *NCDSGPRUse =
BuildMI(*NCD, NCD->getFirstNonPHI(), DebugLoc(),
TII->get(AMDGPU::SGPR_USE))
.addReg(Reg, RegState::Implicit);
MadeChange = true;
SlotIndex SI = LIS->InsertMachineInstrInMaps(NCDSGPRUse);
LIS->extendToIndices(*LR, SI.getRegSlot());
if (LV) {
// TODO: This won't work post-SSA
LV->HandleVirtRegUse(Reg, NCD, NCDSGPRUse);
}
DEBUG(NCDSGPRUse->dump());
}
}
return MadeChange;
}
bool UnreachableMachineBlockElim::runOnMachineFunction(MachineFunction &F) {
df_iterator_default_set<MachineBasicBlock*> Reachable;
bool ModifiedPHI = false;
MMI = getAnalysisIfAvailable<MachineModuleInfo>();
MachineDominatorTree *MDT = getAnalysisIfAvailable<MachineDominatorTree>();
MachineLoopInfo *MLI = getAnalysisIfAvailable<MachineLoopInfo>();
// Mark all reachable blocks.
for (MachineBasicBlock *BB : depth_first_ext(&F, Reachable))
(void)BB/* Mark all reachable blocks */;
// Loop over all dead blocks, remembering them and deleting all instructions
// in them.
std::vector<MachineBasicBlock*> DeadBlocks;
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = &*I;
// Test for deadness.
if (!Reachable.count(BB)) {
DeadBlocks.push_back(BB);
// Update dominator and loop info.
if (MLI) MLI->removeBlock(BB);
if (MDT && MDT->getNode(BB)) MDT->eraseNode(BB);
while (BB->succ_begin() != BB->succ_end()) {
MachineBasicBlock* succ = *BB->succ_begin();
MachineBasicBlock::iterator start = succ->begin();
while (start != succ->end() && start->isPHI()) {
for (unsigned i = start->getNumOperands() - 1; i >= 2; i-=2)
if (start->getOperand(i).isMBB() &&
start->getOperand(i).getMBB() == BB) {
start->RemoveOperand(i);
start->RemoveOperand(i-1);
}
start++;
}
BB->removeSuccessor(BB->succ_begin());
}
}
}
// Actually remove the blocks now.
for (unsigned i = 0, e = DeadBlocks.size(); i != e; ++i)
DeadBlocks[i]->eraseFromParent();
// Cleanup PHI nodes.
for (MachineFunction::iterator I = F.begin(), E = F.end(); I != E; ++I) {
MachineBasicBlock *BB = &*I;
// Prune unneeded PHI entries.
SmallPtrSet<MachineBasicBlock*, 8> preds(BB->pred_begin(),
BB->pred_end());
MachineBasicBlock::iterator phi = BB->begin();
while (phi != BB->end() && phi->isPHI()) {
for (unsigned i = phi->getNumOperands() - 1; i >= 2; i-=2)
if (!preds.count(phi->getOperand(i).getMBB())) {
phi->RemoveOperand(i);
phi->RemoveOperand(i-1);
ModifiedPHI = true;
}
if (phi->getNumOperands() == 3) {
const MachineOperand &Input = phi->getOperand(1);
const MachineOperand &Output = phi->getOperand(0);
unsigned InputReg = Input.getReg();
unsigned OutputReg = Output.getReg();
assert(Output.getSubReg() == 0 && "Cannot have output subregister");
ModifiedPHI = true;
if (InputReg != OutputReg) {
MachineRegisterInfo &MRI = F.getRegInfo();
unsigned InputSub = Input.getSubReg();
if (InputSub == 0 &&
MRI.constrainRegClass(InputReg, MRI.getRegClass(OutputReg))) {
MRI.replaceRegWith(OutputReg, InputReg);
} else {
// The input register to the PHI has a subregister or it can't be
// constrained to the proper register class:
// insert a COPY instead of simply replacing the output
// with the input.
const TargetInstrInfo *TII = F.getSubtarget().getInstrInfo();
BuildMI(*BB, BB->getFirstNonPHI(), phi->getDebugLoc(),
TII->get(TargetOpcode::COPY), OutputReg)
.addReg(InputReg, getRegState(Input), InputSub);
}
phi++->eraseFromParent();
}
continue;
}
++phi;
}
}
F.RenumberBlocks();
return (!DeadBlocks.empty() || ModifiedPHI);
}
// Return true if CC is live out of MBB.
static bool isCCLiveOut(MachineBasicBlock &MBB) {
for (auto SI = MBB.succ_begin(), SE = MBB.succ_end(); SI != SE; ++SI)
if ((*SI)->isLiveIn(SystemZ::CC))
return true;
return false;
}
bool
TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB,
SmallVectorImpl<MachineBasicBlock *> &TDBBs,
const DenseSet<unsigned> &UsedByPhi,
SmallVectorImpl<MachineInstr *> &Copies) {
SmallPtrSet<MachineBasicBlock*, 8> Succs(TailBB->succ_begin(),
TailBB->succ_end());
SmallVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
bool Changed = false;
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (PredBB->getLandingPadSuccessor())
continue;
if (bothUsedInPHI(*PredBB, Succs))
continue;
MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
continue;
Changed = true;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From simple Succ: " << *TailBB);
MachineBasicBlock *NewTarget = *TailBB->succ_begin();
MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(PredBB));
// Make PredFBB explicit.
if (PredCond.empty())
PredFBB = PredTBB;
// Make fall through explicit.
if (!PredTBB)
PredTBB = NextBB;
if (!PredFBB)
PredFBB = NextBB;
// Redirect
if (PredFBB == TailBB)
PredFBB = NewTarget;
if (PredTBB == TailBB)
PredTBB = NewTarget;
// Make the branch unconditional if possible
if (PredTBB == PredFBB) {
PredCond.clear();
PredFBB = nullptr;
}
// Avoid adding fall through branches.
if (PredFBB == NextBB)
PredFBB = nullptr;
if (PredTBB == NextBB && PredFBB == nullptr)
PredTBB = nullptr;
TII->RemoveBranch(*PredBB);
if (PredTBB)
TII->InsertBranch(*PredBB, PredTBB, PredFBB, PredCond, DebugLoc());
uint32_t Weight = MBPI->getEdgeWeight(PredBB, TailBB);
PredBB->removeSuccessor(TailBB);
unsigned NumSuccessors = PredBB->succ_size();
assert(NumSuccessors <= 1);
if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget)
PredBB->addSuccessor(NewTarget, Weight);
TDBBs.push_back(PredBB);
}
return Changed;
}
