// Try to fuse comparison instruction Compare into a later branch.
// Return true on success and if Compare is therefore redundant.
bool SystemZElimCompare::
fuseCompareAndBranch(MachineInstr *Compare,
SmallVectorImpl<MachineInstr *> &CCUsers) {
// See whether we have a comparison that can be fused.
unsigned FusedOpcode = TII->getCompareAndBranch(Compare->getOpcode(),
Compare);
if (!FusedOpcode)
return false;
// See whether we have a single branch with which to fuse.
if (CCUsers.size() != 1)
return false;
MachineInstr *Branch = CCUsers[0];
if (Branch->getOpcode() != SystemZ::BRC)
return false;
// Make sure that the operands are available at the branch.
unsigned SrcReg = Compare->getOperand(0).getReg();
unsigned SrcReg2 = (Compare->getOperand(1).isReg() ?
Compare->getOperand(1).getReg() : 0);
MachineBasicBlock::iterator MBBI = Compare, MBBE = Branch;
for (++MBBI; MBBI != MBBE; ++MBBI)
if (MBBI->modifiesRegister(SrcReg, TRI) ||
(SrcReg2 && MBBI->modifiesRegister(SrcReg2, TRI)))
return false;
// Read the branch mask and target.
MachineOperand CCMask(MBBI->getOperand(1));
MachineOperand Target(MBBI->getOperand(2));
assert((CCMask.getImm() & ~SystemZ::CCMASK_ICMP) == 0 &&
"Invalid condition-code mask for integer comparison");
// Clear out all current operands.
int CCUse = MBBI->findRegisterUseOperandIdx(SystemZ::CC, false, TRI);
assert(CCUse >= 0 && "BRC must use CC");
Branch->RemoveOperand(CCUse);
Branch->RemoveOperand(2);
Branch->RemoveOperand(1);
Branch->RemoveOperand(0);
// Rebuild Branch as a fused compare and branch.
Branch->setDesc(TII->get(FusedOpcode));
MachineInstrBuilder(*Branch->getParent()->getParent(), Branch)
.addOperand(Compare->getOperand(0))
.addOperand(Compare->getOperand(1))
.addOperand(CCMask)
.addOperand(Target)
.addReg(SystemZ::CC, RegState::ImplicitDefine);
// Clear any intervening kills of SrcReg and SrcReg2.
MBBI = Compare;
for (++MBBI; MBBI != MBBE; ++MBBI) {
MBBI->clearRegisterKills(SrcReg, TRI);
if (SrcReg2)
MBBI->clearRegisterKills(SrcReg2, TRI);
}
FusedComparisons += 1;
return true;
}
bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
SmallSetVector<MachineInstr*, 8> MaybeDeadCopies; // Candidates for deletion
DenseMap<unsigned, MachineInstr*> AvailCopyMap; // Def -> available copies map
DenseMap<unsigned, MachineInstr*> CopyMap; // Def -> copies map
SourceMap SrcMap; // Src -> Def map
DEBUG(dbgs() << "MCP: CopyPropagateBlock " << MBB.getName() << "\n");
bool Changed = false;
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) {
MachineInstr *MI = &*I;
++I;
if (MI->isCopy()) {
unsigned Def = MI->getOperand(0).getReg();
unsigned Src = MI->getOperand(1).getReg();
if (TargetRegisterInfo::isVirtualRegister(Def) ||
TargetRegisterInfo::isVirtualRegister(Src))
report_fatal_error("MachineCopyPropagation should be run after"
" register allocation!");
DenseMap<unsigned, MachineInstr*>::iterator CI = AvailCopyMap.find(Src);
if (CI != AvailCopyMap.end()) {
MachineInstr *CopyMI = CI->second;
if (!MRI->isReserved(Def) &&
(!MRI->isReserved(Src) || NoInterveningSideEffect(CopyMI, MI)) &&
isNopCopy(CopyMI, Def, Src, TRI)) {
// The two copies cancel out and the source of the first copy
// hasn't been overridden, eliminate the second one. e.g.
// %ECX<def> = COPY %EAX<kill>
// ... nothing clobbered EAX.
// %EAX<def> = COPY %ECX
// =>
// %ECX<def> = COPY %EAX
//
// Also avoid eliminating a copy from reserved registers unless the
// definition is proven not clobbered. e.g.
// %RSP<def> = COPY %RAX
// CALL
// %RAX<def> = COPY %RSP
DEBUG(dbgs() << "MCP: copy is a NOP, removing: "; MI->dump());
// Clear any kills of Def between CopyMI and MI. This extends the
// live range.
for (MachineBasicBlock::iterator I = CopyMI, E = MI; I != E; ++I)
I->clearRegisterKills(Def, TRI);
removeCopy(MI);
Changed = true;
++NumDeletes;
continue;
}
}
// The CC users in CCUsers are testing the result of a comparison of some
// value X against zero and we know that any CC value produced by MI
// would also reflect the value of X. Try to adjust CCUsers so that
// they test the result of MI directly, returning true on success.
// Leave everything unchanged on failure.
bool SystemZElimCompare::
adjustCCMasksForInstr(MachineInstr *MI, MachineInstr *Compare,
SmallVectorImpl<MachineInstr *> &CCUsers) {
int Opcode = MI->getOpcode();
const MCInstrDesc &Desc = TII->get(Opcode);
unsigned MIFlags = Desc.TSFlags;
// See which compare-style condition codes are available.
unsigned ReusableCCMask = SystemZII::getCompareZeroCCMask(MIFlags);
// For unsigned comparisons with zero, only equality makes sense.
unsigned CompareFlags = Compare->getDesc().TSFlags;
if (CompareFlags & SystemZII::IsLogical)
ReusableCCMask &= SystemZ::CCMASK_CMP_EQ;
if (ReusableCCMask == 0)
return false;
unsigned CCValues = SystemZII::getCCValues(MIFlags);
assert((ReusableCCMask & ~CCValues) == 0 && "Invalid CCValues");
// Now check whether these flags are enough for all users.
SmallVector<MachineOperand *, 4> AlterMasks;
for (unsigned int I = 0, E = CCUsers.size(); I != E; ++I) {
MachineInstr *MI = CCUsers[I];
// Fail if this isn't a use of CC that we understand.
unsigned Flags = MI->getDesc().TSFlags;
unsigned FirstOpNum;
if (Flags & SystemZII::CCMaskFirst)
FirstOpNum = 0;
else if (Flags & SystemZII::CCMaskLast)
FirstOpNum = MI->getNumExplicitOperands() - 2;
else
return false;
// Check whether the instruction predicate treats all CC values
// outside of ReusableCCMask in the same way. In that case it
// doesn't matter what those CC values mean.
unsigned CCValid = MI->getOperand(FirstOpNum).getImm();
unsigned CCMask = MI->getOperand(FirstOpNum + 1).getImm();
unsigned OutValid = ~ReusableCCMask & CCValid;
unsigned OutMask = ~ReusableCCMask & CCMask;
if (OutMask != 0 && OutMask != OutValid)
return false;
AlterMasks.push_back(&MI->getOperand(FirstOpNum));
AlterMasks.push_back(&MI->getOperand(FirstOpNum + 1));
}
// All users are OK. Adjust the masks for MI.
for (unsigned I = 0, E = AlterMasks.size(); I != E; I += 2) {
AlterMasks[I]->setImm(CCValues);
unsigned CCMask = AlterMasks[I + 1]->getImm();
if (CCMask & ~ReusableCCMask)
AlterMasks[I + 1]->setImm((CCMask & ReusableCCMask) |
(CCValues & ~ReusableCCMask));
}
// CC is now live after MI.
int CCDef = MI->findRegisterDefOperandIdx(SystemZ::CC, false, true, TRI);
assert(CCDef >= 0 && "Couldn't find CC set");
MI->getOperand(CCDef).setIsDead(false);
// Clear any intervening kills of CC.
MachineBasicBlock::iterator MBBI = MI, MBBE = Compare;
for (++MBBI; MBBI != MBBE; ++MBBI)
MBBI->clearRegisterKills(SystemZ::CC, TRI);
return true;
}
// Try to fuse comparison instruction Compare into a later branch.
// Return true on success and if Compare is therefore redundant.
bool SystemZElimCompare::fuseCompareOperations(
MachineInstr &Compare, SmallVectorImpl<MachineInstr *> &CCUsers) {
// See whether we have a single branch with which to fuse.
if (CCUsers.size() != 1)
return false;
MachineInstr *Branch = CCUsers[0];
SystemZII::FusedCompareType Type;
switch (Branch->getOpcode()) {
case SystemZ::BRC:
Type = SystemZII::CompareAndBranch;
break;
case SystemZ::CondReturn:
Type = SystemZII::CompareAndReturn;
break;
case SystemZ::CallBCR:
Type = SystemZII::CompareAndSibcall;
break;
case SystemZ::CondTrap:
Type = SystemZII::CompareAndTrap;
break;
default:
return false;
}
// See whether we have a comparison that can be fused.
unsigned FusedOpcode =
TII->getFusedCompare(Compare.getOpcode(), Type, &Compare);
if (!FusedOpcode)
return false;
// Make sure that the operands are available at the branch.
// SrcReg2 is the register if the source operand is a register,
// 0 if the source operand is immediate, and the base register
// if the source operand is memory (index is not supported).
unsigned SrcReg = Compare.getOperand(0).getReg();
unsigned SrcReg2 =
Compare.getOperand(1).isReg() ? Compare.getOperand(1).getReg() : 0;
MachineBasicBlock::iterator MBBI = Compare, MBBE = Branch;
for (++MBBI; MBBI != MBBE; ++MBBI)
if (MBBI->modifiesRegister(SrcReg, TRI) ||
(SrcReg2 && MBBI->modifiesRegister(SrcReg2, TRI)))
return false;
// Read the branch mask, target (if applicable), regmask (if applicable).
MachineOperand CCMask(MBBI->getOperand(1));
assert((CCMask.getImm() & ~SystemZ::CCMASK_ICMP) == 0 &&
"Invalid condition-code mask for integer comparison");
// This is only valid for CompareAndBranch.
MachineOperand Target(MBBI->getOperand(
Type == SystemZII::CompareAndBranch ? 2 : 0));
const uint32_t *RegMask;
if (Type == SystemZII::CompareAndSibcall)
RegMask = MBBI->getOperand(2).getRegMask();
// Clear out all current operands.
int CCUse = MBBI->findRegisterUseOperandIdx(SystemZ::CC, false, TRI);
assert(CCUse >= 0 && "BRC/BCR must use CC");
Branch->RemoveOperand(CCUse);
// Remove target (branch) or regmask (sibcall).
if (Type == SystemZII::CompareAndBranch ||
Type == SystemZII::CompareAndSibcall)
Branch->RemoveOperand(2);
Branch->RemoveOperand(1);
Branch->RemoveOperand(0);
// Rebuild Branch as a fused compare and branch.
// SrcNOps is the number of MI operands of the compare instruction
// that we need to copy over.
unsigned SrcNOps = 2;
if (FusedOpcode == SystemZ::CLT || FusedOpcode == SystemZ::CLGT)
SrcNOps = 3;
Branch->setDesc(TII->get(FusedOpcode));
MachineInstrBuilder MIB(*Branch->getParent()->getParent(), Branch);
for (unsigned I = 0; I < SrcNOps; I++)
MIB.addOperand(Compare.getOperand(I));
MIB.addOperand(CCMask);
if (Type == SystemZII::CompareAndBranch) {
// Only conditional branches define CC, as they may be converted back
// to a non-fused branch because of a long displacement. Conditional
// returns don't have that problem.
MIB.addOperand(Target)
.addReg(SystemZ::CC, RegState::ImplicitDefine | RegState::Dead);
}
if (Type == SystemZII::CompareAndSibcall)
MIB.addRegMask(RegMask);
// Clear any intervening kills of SrcReg and SrcReg2.
MBBI = Compare;
for (++MBBI; MBBI != MBBE; ++MBBI) {
MBBI->clearRegisterKills(SrcReg, TRI);
if (SrcReg2)
MBBI->clearRegisterKills(SrcReg2, TRI);
}
FusedComparisons += 1;
return true;
}
bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) {
SmallSetVector<MachineInstr*, 8> MaybeDeadCopies; // Candidates for deletion
DenseMap<unsigned, MachineInstr*> AvailCopyMap; // Def -> available copies map
DenseMap<unsigned, MachineInstr*> CopyMap; // Def -> copies map
SourceMap SrcMap; // Src -> Def map
bool Changed = false;
for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) {
MachineInstr *MI = &*I;
++I;
if (MI->isCopy()) {
unsigned Def = MI->getOperand(0).getReg();
unsigned Src = MI->getOperand(1).getReg();
if (TargetRegisterInfo::isVirtualRegister(Def) ||
TargetRegisterInfo::isVirtualRegister(Src))
report_fatal_error("MachineCopyPropagation should be run after"
" register allocation!");
DenseMap<unsigned, MachineInstr*>::iterator CI = AvailCopyMap.find(Src);
if (CI != AvailCopyMap.end()) {
MachineInstr *CopyMI = CI->second;
if (!MRI->isReserved(Def) &&
(!MRI->isReserved(Src) || NoInterveningSideEffect(CopyMI, MI)) &&
isNopCopy(CopyMI, Def, Src, TRI)) {
// The two copies cancel out and the source of the first copy
// hasn't been overridden, eliminate the second one. e.g.
// %ECX<def> = COPY %EAX<kill>
// ... nothing clobbered EAX.
// %EAX<def> = COPY %ECX
// =>
// %ECX<def> = COPY %EAX
//
// Also avoid eliminating a copy from reserved registers unless the
// definition is proven not clobbered. e.g.
// %RSP<def> = COPY %RAX
// CALL
// %RAX<def> = COPY %RSP
// Clear any kills of Def between CopyMI and MI. This extends the
// live range.
for (MachineBasicBlock::iterator I = CopyMI, E = MI; I != E; ++I)
I->clearRegisterKills(Def, TRI);
removeCopy(MI);
Changed = true;
++NumDeletes;
continue;
}
}
// If Src is defined by a previous copy, it cannot be eliminated.
for (MCRegAliasIterator AI(Src, TRI, true); AI.isValid(); ++AI) {
CI = CopyMap.find(*AI);
if (CI != CopyMap.end())
MaybeDeadCopies.remove(CI->second);
}
// Copy is now a candidate for deletion.
MaybeDeadCopies.insert(MI);
// If 'Src' is previously source of another copy, then this earlier copy's
// source is no longer available. e.g.
// %xmm9<def> = copy %xmm2
// ...
// %xmm2<def> = copy %xmm0
// ...
// %xmm2<def> = copy %xmm9
SourceNoLongerAvailable(Def, SrcMap, AvailCopyMap);
// Remember Def is defined by the copy.
// ... Make sure to clear the def maps of aliases first.
for (MCRegAliasIterator AI(Def, TRI, false); AI.isValid(); ++AI) {
CopyMap.erase(*AI);
AvailCopyMap.erase(*AI);
}
CopyMap[Def] = MI;
AvailCopyMap[Def] = MI;
for (MCSubRegIterator SR(Def, TRI); SR.isValid(); ++SR) {
CopyMap[*SR] = MI;
AvailCopyMap[*SR] = MI;
}
// Remember source that's copied to Def. Once it's clobbered, then
// it's no longer available for copy propagation.
if (std::find(SrcMap[Src].begin(), SrcMap[Src].end(), Def) ==
SrcMap[Src].end()) {
SrcMap[Src].push_back(Def);
}
continue;
}
// Not a copy.
SmallVector<unsigned, 2> Defs;
int RegMaskOpNum = -1;
for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) {
MachineOperand &MO = MI->getOperand(i);
if (MO.isRegMask())
RegMaskOpNum = i;
if (!MO.isReg())
continue;
unsigned Reg = MO.getReg();
if (!Reg)
continue;
if (TargetRegisterInfo::isVirtualRegister(Reg))
report_fatal_error("MachineCopyPropagation should be run after"
" register allocation!");
if (MO.isDef()) {
Defs.push_back(Reg);
continue;
}
// If 'Reg' is defined by a copy, the copy is no longer a candidate
// for elimination.
for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
DenseMap<unsigned, MachineInstr*>::iterator CI = CopyMap.find(*AI);
if (CI != CopyMap.end())
MaybeDeadCopies.remove(CI->second);
}
}
// The instruction has a register mask operand which means that it clobbers
// a large set of registers. It is possible to use the register mask to
// prune the available copies, but treat it like a basic block boundary for
// now.
if (RegMaskOpNum >= 0) {
// Erase any MaybeDeadCopies whose destination register is clobbered.
const MachineOperand &MaskMO = MI->getOperand(RegMaskOpNum);
for (SmallSetVector<MachineInstr*, 8>::iterator
DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end();
DI != DE; ++DI) {
unsigned Reg = (*DI)->getOperand(0).getReg();
if (MRI->isReserved(Reg) || !MaskMO.clobbersPhysReg(Reg))
continue;
removeCopy(*DI);
Changed = true;
++NumDeletes;
}
// Clear all data structures as if we were beginning a new basic block.
MaybeDeadCopies.clear();
AvailCopyMap.clear();
CopyMap.clear();
SrcMap.clear();
continue;
}
for (unsigned i = 0, e = Defs.size(); i != e; ++i) {
unsigned Reg = Defs[i];
// No longer defined by a copy.
for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) {
CopyMap.erase(*AI);
AvailCopyMap.erase(*AI);
}
// If 'Reg' is previously source of a copy, it is no longer available for
// copy propagation.
SourceNoLongerAvailable(Reg, SrcMap, AvailCopyMap);
}
}
// If MBB doesn't have successors, delete the copies whose defs are not used.
// If MBB does have successors, then conservative assume the defs are live-out
// since we don't want to trust live-in lists.
if (MBB.succ_empty()) {
for (SmallSetVector<MachineInstr*, 8>::iterator
DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end();
DI != DE; ++DI) {
if (!MRI->isReserved((*DI)->getOperand(0).getReg())) {
removeCopy(*DI);
Changed = true;
++NumDeletes;
}
}
}
return Changed;
}
bool AArch64RedundantCopyElimination::optimizeCopy(MachineBasicBlock *MBB) {
// Check if the current basic block has a single predecessor.
if (MBB->pred_size() != 1)
return false;
MachineBasicBlock *PredMBB = *MBB->pred_begin();
MachineBasicBlock::iterator CompBr = PredMBB->getLastNonDebugInstr();
if (CompBr == PredMBB->end() || PredMBB->succ_size() != 2)
return false;
++CompBr;
do {
--CompBr;
if (guaranteesZeroRegInBlock(*CompBr, MBB))
break;
} while (CompBr != PredMBB->begin() && CompBr->isTerminator());
// We've not found a CBZ/CBNZ, time to bail out.
if (!guaranteesZeroRegInBlock(*CompBr, MBB))
return false;
unsigned TargetReg = CompBr->getOperand(0).getReg();
if (!TargetReg)
return false;
assert(TargetRegisterInfo::isPhysicalRegister(TargetReg) &&
"Expect physical register");
// Remember all registers aliasing with TargetReg.
SmallSetVector<unsigned, 8> TargetRegs;
for (MCRegAliasIterator AI(TargetReg, TRI, true); AI.isValid(); ++AI)
TargetRegs.insert(*AI);
bool Changed = false;
MachineBasicBlock::iterator LastChange = MBB->begin();
unsigned SmallestDef = TargetReg;
// Remove redundant Copy instructions unless TargetReg is modified.
for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;) {
MachineInstr *MI = &*I;
++I;
if (MI->isCopy() && MI->getOperand(0).isReg() &&
MI->getOperand(1).isReg()) {
unsigned DefReg = MI->getOperand(0).getReg();
unsigned SrcReg = MI->getOperand(1).getReg();
if ((SrcReg == AArch64::XZR || SrcReg == AArch64::WZR) &&
!MRI->isReserved(DefReg) &&
(TargetReg == DefReg || TRI->isSuperRegister(DefReg, TargetReg))) {
DEBUG(dbgs() << "Remove redundant Copy : ");
DEBUG((MI)->print(dbgs()));
MI->eraseFromParent();
Changed = true;
LastChange = I;
NumCopiesRemoved++;
SmallestDef =
TRI->isSubRegister(SmallestDef, DefReg) ? DefReg : SmallestDef;
continue;
}
}
if (MI->modifiesRegister(TargetReg, TRI))
break;
}
if (!Changed)
return false;
// Otherwise, we have to fixup the use-def chain, starting with the
// CBZ/CBNZ. Conservatively mark as much as we can live.
CompBr->clearRegisterKills(SmallestDef, TRI);
if (std::none_of(TargetRegs.begin(), TargetRegs.end(),
[&](unsigned Reg) { return MBB->isLiveIn(Reg); }))
MBB->addLiveIn(TargetReg);
// Clear any kills of TargetReg between CompBr and the last removed COPY.
for (MachineInstr &MMI :
make_range(MBB->begin()->getIterator(), LastChange->getIterator()))
MMI.clearRegisterKills(SmallestDef, TRI);
return true;
}
