/// Performs the actual liveness calculation for the function.
bool StackMapLiveness::calculateLiveness() {
bool HasChanged = false;
// For all basic blocks in the function.
for (MachineFunction::iterator MBBI = MF->begin(), MBBE = MF->end();
MBBI != MBBE; ++MBBI) {
DEBUG(dbgs() << "****** BB " << MBBI->getName() << " ******\n");
LiveRegs.init(TRI);
LiveRegs.addLiveOuts(MBBI);
bool HasStackMap = false;
// Reverse iterate over all instructions and add the current live register
// set to an instruction if we encounter a stackmap or patchpoint
// instruction.
for (MachineBasicBlock::reverse_iterator I = MBBI->rbegin(),
E = MBBI->rend(); I != E; ++I) {
int Opc = I->getOpcode();
if ((EnableStackMapLiveness && (Opc == TargetOpcode::STACKMAP)) ||
(EnablePatchPointLiveness && (Opc == TargetOpcode::PATCHPOINT))) {
addLiveOutSetToMI(*I);
HasChanged = true;
HasStackMap = true;
++NumStackMaps;
}
DEBUG(dbgs() << " " << LiveRegs << " " << *I);
LiveRegs.stepBackward(*I);
}
++NumBBsVisited;
if (!HasStackMap)
++NumBBsHaveNoStackmap;
}
return HasChanged;
}
unsigned MipsInstrInfo::removeBranch(MachineBasicBlock &MBB,
int *BytesRemoved) const {
assert(!BytesRemoved && "code size not handled");
MachineBasicBlock::reverse_iterator I = MBB.rbegin(), REnd = MBB.rend();
unsigned removed;
// Skip all the debug instructions.
while (I != REnd && I->isDebugValue())
++I;
if (I == REnd)
return 0;
MachineBasicBlock::iterator FirstBr = ++I.getReverse();
// Up to 2 branches are removed.
// Note that indirect branches are not removed.
for (removed = 0; I != REnd && removed < 2; ++I, ++removed)
if (!getAnalyzableBrOpc(I->getOpcode()))
break;
MBB.erase((--I).getReverse(), FirstBr);
return removed;
}
unsigned MipsInstrInfo::removeBranch(MachineBasicBlock &MBB,
int *BytesRemoved) const {
assert(!BytesRemoved && "code size not handled");
MachineBasicBlock::reverse_iterator I = MBB.rbegin(), REnd = MBB.rend();
unsigned removed = 0;
// Up to 2 branches are removed.
// Note that indirect branches are not removed.
while (I != REnd && removed < 2) {
// Skip past debug instructions.
if (I->isDebugValue()) {
++I;
continue;
}
if (!getAnalyzableBrOpc(I->getOpcode()))
break;
// Remove the branch.
I->eraseFromParent();
I = MBB.rbegin();
++removed;
}
return removed;
}
static MachineBasicBlock::reverse_iterator fixTerminators(
const SIInstrInfo &TII,
MachineBasicBlock &MBB) {
MachineBasicBlock::reverse_iterator I = MBB.rbegin(), E = MBB.rend();
for (; I != E; ++I) {
if (!I->isTerminator())
return I;
if (removeTerminatorBit(TII, *I))
return I;
}
return E;
}
unsigned MipsInstrInfo::RemoveBranch(MachineBasicBlock &MBB) const {
MachineBasicBlock::reverse_iterator I = MBB.rbegin(), REnd = MBB.rend();
MachineBasicBlock::reverse_iterator FirstBr;
unsigned removed;
// Skip all the debug instructions.
while (I != REnd && I->isDebugValue())
++I;
FirstBr = I;
// Up to 2 branches are removed.
// Note that indirect branches are not removed.
for (removed = 0; I != REnd && removed < 2; ++I, ++removed)
if (!getAnalyzableBrOpc(I->getOpcode()))
break;
MBB.erase(I.base(), FirstBr.base());
return removed;
}
MipsInstrInfo::BranchType MipsInstrInfo::AnalyzeBranch(
MachineBasicBlock &MBB, MachineBasicBlock *&TBB, MachineBasicBlock *&FBB,
SmallVectorImpl<MachineOperand> &Cond, bool AllowModify,
SmallVectorImpl<MachineInstr *> &BranchInstrs) const {
MachineBasicBlock::reverse_iterator I = MBB.rbegin(), REnd = MBB.rend();
// Skip all the debug instructions.
while (I != REnd && I->isDebugValue())
++I;
if (I == REnd || !isUnpredicatedTerminator(*I)) {
// This block ends with no branches (it just falls through to its succ).
// Leave TBB/FBB null.
TBB = FBB = nullptr;
return BT_NoBranch;
}
MachineInstr *LastInst = &*I;
unsigned LastOpc = LastInst->getOpcode();
BranchInstrs.push_back(LastInst);
// Not an analyzable branch (e.g., indirect jump).
if (!getAnalyzableBrOpc(LastOpc))
return LastInst->isIndirectBranch() ? BT_Indirect : BT_None;
// Get the second to last instruction in the block.
unsigned SecondLastOpc = 0;
MachineInstr *SecondLastInst = nullptr;
if (++I != REnd) {
SecondLastInst = &*I;
SecondLastOpc = getAnalyzableBrOpc(SecondLastInst->getOpcode());
// Not an analyzable branch (must be an indirect jump).
if (isUnpredicatedTerminator(*SecondLastInst) && !SecondLastOpc)
return BT_None;
}
// If there is only one terminator instruction, process it.
if (!SecondLastOpc) {
// Unconditional branch.
if (LastInst->isUnconditionalBranch()) {
TBB = LastInst->getOperand(0).getMBB();
return BT_Uncond;
}
// Conditional branch
AnalyzeCondBr(LastInst, LastOpc, TBB, Cond);
return BT_Cond;
}
// If we reached here, there are two branches.
// If there are three terminators, we don't know what sort of block this is.
if (++I != REnd && isUnpredicatedTerminator(*I))
return BT_None;
BranchInstrs.insert(BranchInstrs.begin(), SecondLastInst);
// If second to last instruction is an unconditional branch,
// analyze it and remove the last instruction.
if (SecondLastInst->isUnconditionalBranch()) {
// Return if the last instruction cannot be removed.
if (!AllowModify)
return BT_None;
TBB = SecondLastInst->getOperand(0).getMBB();
LastInst->eraseFromParent();
BranchInstrs.pop_back();
return BT_Uncond;
}
// Conditional branch followed by an unconditional branch.
// The last one must be unconditional.
if (!LastInst->isUnconditionalBranch())
return BT_None;
AnalyzeCondBr(SecondLastInst, SecondLastOpc, TBB, Cond);
FBB = LastInst->getOperand(0).getMBB();
return BT_CondUncond;
}
bool SIOptimizeExecMasking::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
const SISubtarget &ST = MF.getSubtarget<SISubtarget>();
const SIRegisterInfo *TRI = ST.getRegisterInfo();
const SIInstrInfo *TII = ST.getInstrInfo();
// Optimize sequences emitted for control flow lowering. They are originally
// emitted as the separate operations because spill code may need to be
// inserted for the saved copy of exec.
//
// x = copy exec
// z = s_<op>_b64 x, y
// exec = copy z
// =>
// x = s_<op>_saveexec_b64 y
//
for (MachineBasicBlock &MBB : MF) {
MachineBasicBlock::reverse_iterator I = fixTerminators(*TII, MBB);
MachineBasicBlock::reverse_iterator E = MBB.rend();
if (I == E)
continue;
unsigned CopyToExec = isCopyToExec(*I);
if (CopyToExec == AMDGPU::NoRegister)
continue;
// Scan backwards to find the def.
auto CopyToExecInst = &*I;
auto CopyFromExecInst = findExecCopy(*TII, MBB, I, CopyToExec);
if (CopyFromExecInst == E) {
auto PrepareExecInst = std::next(I);
if (PrepareExecInst == E)
continue;
// Fold exec = COPY (S_AND_B64 reg, exec) -> exec = S_AND_B64 reg, exec
if (CopyToExecInst->getOperand(1).isKill() &&
isLogicalOpOnExec(*PrepareExecInst) == CopyToExec) {
DEBUG(dbgs() << "Fold exec copy: " << *PrepareExecInst);
PrepareExecInst->getOperand(0).setReg(AMDGPU::EXEC);
PrepareExecInst->getOperand(0).setIsRenamable(false);
DEBUG(dbgs() << "into: " << *PrepareExecInst << '\n');
CopyToExecInst->eraseFromParent();
}
continue;
}
if (isLiveOut(MBB, CopyToExec)) {
// The copied register is live out and has a second use in another block.
DEBUG(dbgs() << "Exec copy source register is live out\n");
continue;
}
unsigned CopyFromExec = CopyFromExecInst->getOperand(0).getReg();
MachineInstr *SaveExecInst = nullptr;
SmallVector<MachineInstr *, 4> OtherUseInsts;
for (MachineBasicBlock::iterator J
= std::next(CopyFromExecInst->getIterator()), JE = I->getIterator();
J != JE; ++J) {
if (SaveExecInst && J->readsRegister(AMDGPU::EXEC, TRI)) {
DEBUG(dbgs() << "exec read prevents saveexec: " << *J << '\n');
// Make sure this is inserted after any VALU ops that may have been
// scheduled in between.
SaveExecInst = nullptr;
break;
}
bool ReadsCopyFromExec = J->readsRegister(CopyFromExec, TRI);
if (J->modifiesRegister(CopyToExec, TRI)) {
if (SaveExecInst) {
DEBUG(dbgs() << "Multiple instructions modify "
<< printReg(CopyToExec, TRI) << '\n');
SaveExecInst = nullptr;
break;
}
unsigned SaveExecOp = getSaveExecOp(J->getOpcode());
if (SaveExecOp == AMDGPU::INSTRUCTION_LIST_END)
break;
if (ReadsCopyFromExec) {
SaveExecInst = &*J;
DEBUG(dbgs() << "Found save exec op: " << *SaveExecInst << '\n');
continue;
} else {
DEBUG(dbgs() << "Instruction does not read exec copy: " << *J << '\n');
break;
}
} else if (ReadsCopyFromExec && !SaveExecInst) {
// Make sure no other instruction is trying to use this copy, before it
// will be rewritten by the saveexec, i.e. hasOneUse. There may have
// been another use, such as an inserted spill. For example:
//
// %sgpr0_sgpr1 = COPY %exec
// spill %sgpr0_sgpr1
// %sgpr2_sgpr3 = S_AND_B64 %sgpr0_sgpr1
//
DEBUG(dbgs() << "Found second use of save inst candidate: "
<< *J << '\n');
break;
}
if (SaveExecInst && J->readsRegister(CopyToExec, TRI)) {
assert(SaveExecInst != &*J);
OtherUseInsts.push_back(&*J);
}
}
if (!SaveExecInst)
continue;
DEBUG(dbgs() << "Insert save exec op: " << *SaveExecInst << '\n');
MachineOperand &Src0 = SaveExecInst->getOperand(1);
MachineOperand &Src1 = SaveExecInst->getOperand(2);
MachineOperand *OtherOp = nullptr;
if (Src0.isReg() && Src0.getReg() == CopyFromExec) {
OtherOp = &Src1;
} else if (Src1.isReg() && Src1.getReg() == CopyFromExec) {
if (!SaveExecInst->isCommutable())
break;
OtherOp = &Src0;
} else
llvm_unreachable("unexpected");
CopyFromExecInst->eraseFromParent();
auto InsPt = SaveExecInst->getIterator();
const DebugLoc &DL = SaveExecInst->getDebugLoc();
BuildMI(MBB, InsPt, DL, TII->get(getSaveExecOp(SaveExecInst->getOpcode())),
CopyFromExec)
.addReg(OtherOp->getReg());
SaveExecInst->eraseFromParent();
CopyToExecInst->eraseFromParent();
for (MachineInstr *OtherInst : OtherUseInsts) {
OtherInst->substituteRegister(CopyToExec, AMDGPU::EXEC,
AMDGPU::NoSubRegister, *TRI,
/*ClearIsRenamable=*/true);
}
}
return true;
}
bool CoffeeInstrInfo::AnalyzeBranch(MachineBasicBlock &MBB,MachineBasicBlock *&TBB,
MachineBasicBlock *&FBB,
SmallVectorImpl<MachineOperand> &Cond,
bool AllowModify) const {
MachineBasicBlock::reverse_iterator I = MBB.rbegin(), REnd = MBB.rend();
// Skip all the debug instructions.
while (I != REnd && I->isDebugValue())
++I;
if (I == REnd || !isUnpredicatedTerminator(&*I)) {
// If this block ends with no branches (it just falls through to its succ)
// just return false, leaving TBB/FBB null.
TBB = FBB = NULL;
return false;
}
MachineInstr *LastInst = &*I;
unsigned LastOpc = LastInst->getOpcode();
// Not an analyzable branch (must be an indirect jump).
if (!GetAnalyzableBrOpc(LastOpc))
return true;
// Get the second to last instruction in the block.
unsigned SecondLastOpc = 0;
MachineInstr *SecondLastInst = NULL;
if (++I != REnd) {
SecondLastInst = &*I;
SecondLastOpc = GetAnalyzableBrOpc(SecondLastInst->getOpcode());
// Not an analyzable branch (must be an indirect jump).
if (isUnpredicatedTerminator(SecondLastInst) && !SecondLastOpc)
return true;
}
// If there is only one terminator instruction, process it.
if (!SecondLastOpc) {
// Unconditional branch
if (LastOpc == UncondBrOpc) {
TBB = LastInst->getOperand(0).getMBB();
return false;
}
// Conditional branch
AnalyzeCondBr(LastInst, LastOpc, TBB, Cond);
return false;
}
// If we reached here, there are two branches.
// If there are three terminators, we don't know what sort of block this is.
if (++I != REnd && isUnpredicatedTerminator(&*I))
return true;
// If second to last instruction is an unconditional branch,
// analyze it and remove the last instruction.
if (SecondLastOpc == UncondBrOpc) {
// Return if the last instruction cannot be removed.
if (!AllowModify)
return true;
TBB = SecondLastInst->getOperand(0).getMBB();
LastInst->eraseFromParent();
return false;
}
// Conditional branch followed by an unconditional branch.
// The last one must be unconditional.
if (LastOpc != UncondBrOpc)
return true;
AnalyzeCondBr(SecondLastInst, SecondLastOpc, TBB, Cond);
FBB = LastInst->getOperand(0).getMBB();
return false;
}