// Try to fuse comparison instruction Compare into a later branch. // Return true on success and if Compare is therefore redundant. bool SystemZElimCompare:: fuseCompareAndBranch(MachineInstr *Compare, SmallVectorImpl<MachineInstr *> &CCUsers) { // See whether we have a comparison that can be fused. unsigned FusedOpcode = TII->getCompareAndBranch(Compare->getOpcode(), Compare); if (!FusedOpcode) return false; // See whether we have a single branch with which to fuse. if (CCUsers.size() != 1) return false; MachineInstr *Branch = CCUsers[0]; if (Branch->getOpcode() != SystemZ::BRC) return false; // Make sure that the operands are available at the branch. unsigned SrcReg = Compare->getOperand(0).getReg(); unsigned SrcReg2 = (Compare->getOperand(1).isReg() ? Compare->getOperand(1).getReg() : 0); MachineBasicBlock::iterator MBBI = Compare, MBBE = Branch; for (++MBBI; MBBI != MBBE; ++MBBI) if (MBBI->modifiesRegister(SrcReg, TRI) || (SrcReg2 && MBBI->modifiesRegister(SrcReg2, TRI))) return false; // Read the branch mask and target. MachineOperand CCMask(MBBI->getOperand(1)); MachineOperand Target(MBBI->getOperand(2)); assert((CCMask.getImm() & ~SystemZ::CCMASK_ICMP) == 0 && "Invalid condition-code mask for integer comparison"); // Clear out all current operands. int CCUse = MBBI->findRegisterUseOperandIdx(SystemZ::CC, false, TRI); assert(CCUse >= 0 && "BRC must use CC"); Branch->RemoveOperand(CCUse); Branch->RemoveOperand(2); Branch->RemoveOperand(1); Branch->RemoveOperand(0); // Rebuild Branch as a fused compare and branch. Branch->setDesc(TII->get(FusedOpcode)); MachineInstrBuilder(*Branch->getParent()->getParent(), Branch) .addOperand(Compare->getOperand(0)) .addOperand(Compare->getOperand(1)) .addOperand(CCMask) .addOperand(Target) .addReg(SystemZ::CC, RegState::ImplicitDefine); // Clear any intervening kills of SrcReg and SrcReg2. MBBI = Compare; for (++MBBI; MBBI != MBBE; ++MBBI) { MBBI->clearRegisterKills(SrcReg, TRI); if (SrcReg2) MBBI->clearRegisterKills(SrcReg2, TRI); } FusedComparisons += 1; return true; }
bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) { SmallSetVector<MachineInstr*, 8> MaybeDeadCopies; // Candidates for deletion DenseMap<unsigned, MachineInstr*> AvailCopyMap; // Def -> available copies map DenseMap<unsigned, MachineInstr*> CopyMap; // Def -> copies map SourceMap SrcMap; // Src -> Def map DEBUG(dbgs() << "MCP: CopyPropagateBlock " << MBB.getName() << "\n"); bool Changed = false; for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) { MachineInstr *MI = &*I; ++I; if (MI->isCopy()) { unsigned Def = MI->getOperand(0).getReg(); unsigned Src = MI->getOperand(1).getReg(); if (TargetRegisterInfo::isVirtualRegister(Def) || TargetRegisterInfo::isVirtualRegister(Src)) report_fatal_error("MachineCopyPropagation should be run after" " register allocation!"); DenseMap<unsigned, MachineInstr*>::iterator CI = AvailCopyMap.find(Src); if (CI != AvailCopyMap.end()) { MachineInstr *CopyMI = CI->second; if (!MRI->isReserved(Def) && (!MRI->isReserved(Src) || NoInterveningSideEffect(CopyMI, MI)) && isNopCopy(CopyMI, Def, Src, TRI)) { // The two copies cancel out and the source of the first copy // hasn't been overridden, eliminate the second one. e.g. // %ECX<def> = COPY %EAX<kill> // ... nothing clobbered EAX. // %EAX<def> = COPY %ECX // => // %ECX<def> = COPY %EAX // // Also avoid eliminating a copy from reserved registers unless the // definition is proven not clobbered. e.g. // %RSP<def> = COPY %RAX // CALL // %RAX<def> = COPY %RSP DEBUG(dbgs() << "MCP: copy is a NOP, removing: "; MI->dump()); // Clear any kills of Def between CopyMI and MI. This extends the // live range. for (MachineBasicBlock::iterator I = CopyMI, E = MI; I != E; ++I) I->clearRegisterKills(Def, TRI); removeCopy(MI); Changed = true; ++NumDeletes; continue; } }
// The CC users in CCUsers are testing the result of a comparison of some // value X against zero and we know that any CC value produced by MI // would also reflect the value of X. Try to adjust CCUsers so that // they test the result of MI directly, returning true on success. // Leave everything unchanged on failure. bool SystemZElimCompare:: adjustCCMasksForInstr(MachineInstr *MI, MachineInstr *Compare, SmallVectorImpl<MachineInstr *> &CCUsers) { int Opcode = MI->getOpcode(); const MCInstrDesc &Desc = TII->get(Opcode); unsigned MIFlags = Desc.TSFlags; // See which compare-style condition codes are available. unsigned ReusableCCMask = SystemZII::getCompareZeroCCMask(MIFlags); // For unsigned comparisons with zero, only equality makes sense. unsigned CompareFlags = Compare->getDesc().TSFlags; if (CompareFlags & SystemZII::IsLogical) ReusableCCMask &= SystemZ::CCMASK_CMP_EQ; if (ReusableCCMask == 0) return false; unsigned CCValues = SystemZII::getCCValues(MIFlags); assert((ReusableCCMask & ~CCValues) == 0 && "Invalid CCValues"); // Now check whether these flags are enough for all users. SmallVector<MachineOperand *, 4> AlterMasks; for (unsigned int I = 0, E = CCUsers.size(); I != E; ++I) { MachineInstr *MI = CCUsers[I]; // Fail if this isn't a use of CC that we understand. unsigned Flags = MI->getDesc().TSFlags; unsigned FirstOpNum; if (Flags & SystemZII::CCMaskFirst) FirstOpNum = 0; else if (Flags & SystemZII::CCMaskLast) FirstOpNum = MI->getNumExplicitOperands() - 2; else return false; // Check whether the instruction predicate treats all CC values // outside of ReusableCCMask in the same way. In that case it // doesn't matter what those CC values mean. unsigned CCValid = MI->getOperand(FirstOpNum).getImm(); unsigned CCMask = MI->getOperand(FirstOpNum + 1).getImm(); unsigned OutValid = ~ReusableCCMask & CCValid; unsigned OutMask = ~ReusableCCMask & CCMask; if (OutMask != 0 && OutMask != OutValid) return false; AlterMasks.push_back(&MI->getOperand(FirstOpNum)); AlterMasks.push_back(&MI->getOperand(FirstOpNum + 1)); } // All users are OK. Adjust the masks for MI. for (unsigned I = 0, E = AlterMasks.size(); I != E; I += 2) { AlterMasks[I]->setImm(CCValues); unsigned CCMask = AlterMasks[I + 1]->getImm(); if (CCMask & ~ReusableCCMask) AlterMasks[I + 1]->setImm((CCMask & ReusableCCMask) | (CCValues & ~ReusableCCMask)); } // CC is now live after MI. int CCDef = MI->findRegisterDefOperandIdx(SystemZ::CC, false, true, TRI); assert(CCDef >= 0 && "Couldn't find CC set"); MI->getOperand(CCDef).setIsDead(false); // Clear any intervening kills of CC. MachineBasicBlock::iterator MBBI = MI, MBBE = Compare; for (++MBBI; MBBI != MBBE; ++MBBI) MBBI->clearRegisterKills(SystemZ::CC, TRI); return true; }
// Try to fuse comparison instruction Compare into a later branch. // Return true on success and if Compare is therefore redundant. bool SystemZElimCompare::fuseCompareOperations( MachineInstr &Compare, SmallVectorImpl<MachineInstr *> &CCUsers) { // See whether we have a single branch with which to fuse. if (CCUsers.size() != 1) return false; MachineInstr *Branch = CCUsers[0]; SystemZII::FusedCompareType Type; switch (Branch->getOpcode()) { case SystemZ::BRC: Type = SystemZII::CompareAndBranch; break; case SystemZ::CondReturn: Type = SystemZII::CompareAndReturn; break; case SystemZ::CallBCR: Type = SystemZII::CompareAndSibcall; break; case SystemZ::CondTrap: Type = SystemZII::CompareAndTrap; break; default: return false; } // See whether we have a comparison that can be fused. unsigned FusedOpcode = TII->getFusedCompare(Compare.getOpcode(), Type, &Compare); if (!FusedOpcode) return false; // Make sure that the operands are available at the branch. // SrcReg2 is the register if the source operand is a register, // 0 if the source operand is immediate, and the base register // if the source operand is memory (index is not supported). unsigned SrcReg = Compare.getOperand(0).getReg(); unsigned SrcReg2 = Compare.getOperand(1).isReg() ? Compare.getOperand(1).getReg() : 0; MachineBasicBlock::iterator MBBI = Compare, MBBE = Branch; for (++MBBI; MBBI != MBBE; ++MBBI) if (MBBI->modifiesRegister(SrcReg, TRI) || (SrcReg2 && MBBI->modifiesRegister(SrcReg2, TRI))) return false; // Read the branch mask, target (if applicable), regmask (if applicable). MachineOperand CCMask(MBBI->getOperand(1)); assert((CCMask.getImm() & ~SystemZ::CCMASK_ICMP) == 0 && "Invalid condition-code mask for integer comparison"); // This is only valid for CompareAndBranch. MachineOperand Target(MBBI->getOperand( Type == SystemZII::CompareAndBranch ? 2 : 0)); const uint32_t *RegMask; if (Type == SystemZII::CompareAndSibcall) RegMask = MBBI->getOperand(2).getRegMask(); // Clear out all current operands. int CCUse = MBBI->findRegisterUseOperandIdx(SystemZ::CC, false, TRI); assert(CCUse >= 0 && "BRC/BCR must use CC"); Branch->RemoveOperand(CCUse); // Remove target (branch) or regmask (sibcall). if (Type == SystemZII::CompareAndBranch || Type == SystemZII::CompareAndSibcall) Branch->RemoveOperand(2); Branch->RemoveOperand(1); Branch->RemoveOperand(0); // Rebuild Branch as a fused compare and branch. // SrcNOps is the number of MI operands of the compare instruction // that we need to copy over. unsigned SrcNOps = 2; if (FusedOpcode == SystemZ::CLT || FusedOpcode == SystemZ::CLGT) SrcNOps = 3; Branch->setDesc(TII->get(FusedOpcode)); MachineInstrBuilder MIB(*Branch->getParent()->getParent(), Branch); for (unsigned I = 0; I < SrcNOps; I++) MIB.addOperand(Compare.getOperand(I)); MIB.addOperand(CCMask); if (Type == SystemZII::CompareAndBranch) { // Only conditional branches define CC, as they may be converted back // to a non-fused branch because of a long displacement. Conditional // returns don't have that problem. MIB.addOperand(Target) .addReg(SystemZ::CC, RegState::ImplicitDefine | RegState::Dead); } if (Type == SystemZII::CompareAndSibcall) MIB.addRegMask(RegMask); // Clear any intervening kills of SrcReg and SrcReg2. MBBI = Compare; for (++MBBI; MBBI != MBBE; ++MBBI) { MBBI->clearRegisterKills(SrcReg, TRI); if (SrcReg2) MBBI->clearRegisterKills(SrcReg2, TRI); } FusedComparisons += 1; return true; }
bool MachineCopyPropagation::CopyPropagateBlock(MachineBasicBlock &MBB) { SmallSetVector<MachineInstr*, 8> MaybeDeadCopies; // Candidates for deletion DenseMap<unsigned, MachineInstr*> AvailCopyMap; // Def -> available copies map DenseMap<unsigned, MachineInstr*> CopyMap; // Def -> copies map SourceMap SrcMap; // Src -> Def map bool Changed = false; for (MachineBasicBlock::iterator I = MBB.begin(), E = MBB.end(); I != E; ) { MachineInstr *MI = &*I; ++I; if (MI->isCopy()) { unsigned Def = MI->getOperand(0).getReg(); unsigned Src = MI->getOperand(1).getReg(); if (TargetRegisterInfo::isVirtualRegister(Def) || TargetRegisterInfo::isVirtualRegister(Src)) report_fatal_error("MachineCopyPropagation should be run after" " register allocation!"); DenseMap<unsigned, MachineInstr*>::iterator CI = AvailCopyMap.find(Src); if (CI != AvailCopyMap.end()) { MachineInstr *CopyMI = CI->second; if (!MRI->isReserved(Def) && (!MRI->isReserved(Src) || NoInterveningSideEffect(CopyMI, MI)) && isNopCopy(CopyMI, Def, Src, TRI)) { // The two copies cancel out and the source of the first copy // hasn't been overridden, eliminate the second one. e.g. // %ECX<def> = COPY %EAX<kill> // ... nothing clobbered EAX. // %EAX<def> = COPY %ECX // => // %ECX<def> = COPY %EAX // // Also avoid eliminating a copy from reserved registers unless the // definition is proven not clobbered. e.g. // %RSP<def> = COPY %RAX // CALL // %RAX<def> = COPY %RSP // Clear any kills of Def between CopyMI and MI. This extends the // live range. for (MachineBasicBlock::iterator I = CopyMI, E = MI; I != E; ++I) I->clearRegisterKills(Def, TRI); removeCopy(MI); Changed = true; ++NumDeletes; continue; } } // If Src is defined by a previous copy, it cannot be eliminated. for (MCRegAliasIterator AI(Src, TRI, true); AI.isValid(); ++AI) { CI = CopyMap.find(*AI); if (CI != CopyMap.end()) MaybeDeadCopies.remove(CI->second); } // Copy is now a candidate for deletion. MaybeDeadCopies.insert(MI); // If 'Src' is previously source of another copy, then this earlier copy's // source is no longer available. e.g. // %xmm9<def> = copy %xmm2 // ... // %xmm2<def> = copy %xmm0 // ... // %xmm2<def> = copy %xmm9 SourceNoLongerAvailable(Def, SrcMap, AvailCopyMap); // Remember Def is defined by the copy. // ... Make sure to clear the def maps of aliases first. for (MCRegAliasIterator AI(Def, TRI, false); AI.isValid(); ++AI) { CopyMap.erase(*AI); AvailCopyMap.erase(*AI); } CopyMap[Def] = MI; AvailCopyMap[Def] = MI; for (MCSubRegIterator SR(Def, TRI); SR.isValid(); ++SR) { CopyMap[*SR] = MI; AvailCopyMap[*SR] = MI; } // Remember source that's copied to Def. Once it's clobbered, then // it's no longer available for copy propagation. if (std::find(SrcMap[Src].begin(), SrcMap[Src].end(), Def) == SrcMap[Src].end()) { SrcMap[Src].push_back(Def); } continue; } // Not a copy. SmallVector<unsigned, 2> Defs; int RegMaskOpNum = -1; for (unsigned i = 0, e = MI->getNumOperands(); i != e; ++i) { MachineOperand &MO = MI->getOperand(i); if (MO.isRegMask()) RegMaskOpNum = i; if (!MO.isReg()) continue; unsigned Reg = MO.getReg(); if (!Reg) continue; if (TargetRegisterInfo::isVirtualRegister(Reg)) report_fatal_error("MachineCopyPropagation should be run after" " register allocation!"); if (MO.isDef()) { Defs.push_back(Reg); continue; } // If 'Reg' is defined by a copy, the copy is no longer a candidate // for elimination. for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) { DenseMap<unsigned, MachineInstr*>::iterator CI = CopyMap.find(*AI); if (CI != CopyMap.end()) MaybeDeadCopies.remove(CI->second); } } // The instruction has a register mask operand which means that it clobbers // a large set of registers. It is possible to use the register mask to // prune the available copies, but treat it like a basic block boundary for // now. if (RegMaskOpNum >= 0) { // Erase any MaybeDeadCopies whose destination register is clobbered. const MachineOperand &MaskMO = MI->getOperand(RegMaskOpNum); for (SmallSetVector<MachineInstr*, 8>::iterator DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end(); DI != DE; ++DI) { unsigned Reg = (*DI)->getOperand(0).getReg(); if (MRI->isReserved(Reg) || !MaskMO.clobbersPhysReg(Reg)) continue; removeCopy(*DI); Changed = true; ++NumDeletes; } // Clear all data structures as if we were beginning a new basic block. MaybeDeadCopies.clear(); AvailCopyMap.clear(); CopyMap.clear(); SrcMap.clear(); continue; } for (unsigned i = 0, e = Defs.size(); i != e; ++i) { unsigned Reg = Defs[i]; // No longer defined by a copy. for (MCRegAliasIterator AI(Reg, TRI, true); AI.isValid(); ++AI) { CopyMap.erase(*AI); AvailCopyMap.erase(*AI); } // If 'Reg' is previously source of a copy, it is no longer available for // copy propagation. SourceNoLongerAvailable(Reg, SrcMap, AvailCopyMap); } } // If MBB doesn't have successors, delete the copies whose defs are not used. // If MBB does have successors, then conservative assume the defs are live-out // since we don't want to trust live-in lists. if (MBB.succ_empty()) { for (SmallSetVector<MachineInstr*, 8>::iterator DI = MaybeDeadCopies.begin(), DE = MaybeDeadCopies.end(); DI != DE; ++DI) { if (!MRI->isReserved((*DI)->getOperand(0).getReg())) { removeCopy(*DI); Changed = true; ++NumDeletes; } } } return Changed; }
bool AArch64RedundantCopyElimination::optimizeCopy(MachineBasicBlock *MBB) { // Check if the current basic block has a single predecessor. if (MBB->pred_size() != 1) return false; MachineBasicBlock *PredMBB = *MBB->pred_begin(); MachineBasicBlock::iterator CompBr = PredMBB->getLastNonDebugInstr(); if (CompBr == PredMBB->end() || PredMBB->succ_size() != 2) return false; ++CompBr; do { --CompBr; if (guaranteesZeroRegInBlock(*CompBr, MBB)) break; } while (CompBr != PredMBB->begin() && CompBr->isTerminator()); // We've not found a CBZ/CBNZ, time to bail out. if (!guaranteesZeroRegInBlock(*CompBr, MBB)) return false; unsigned TargetReg = CompBr->getOperand(0).getReg(); if (!TargetReg) return false; assert(TargetRegisterInfo::isPhysicalRegister(TargetReg) && "Expect physical register"); // Remember all registers aliasing with TargetReg. SmallSetVector<unsigned, 8> TargetRegs; for (MCRegAliasIterator AI(TargetReg, TRI, true); AI.isValid(); ++AI) TargetRegs.insert(*AI); bool Changed = false; MachineBasicBlock::iterator LastChange = MBB->begin(); unsigned SmallestDef = TargetReg; // Remove redundant Copy instructions unless TargetReg is modified. for (MachineBasicBlock::iterator I = MBB->begin(), E = MBB->end(); I != E;) { MachineInstr *MI = &*I; ++I; if (MI->isCopy() && MI->getOperand(0).isReg() && MI->getOperand(1).isReg()) { unsigned DefReg = MI->getOperand(0).getReg(); unsigned SrcReg = MI->getOperand(1).getReg(); if ((SrcReg == AArch64::XZR || SrcReg == AArch64::WZR) && !MRI->isReserved(DefReg) && (TargetReg == DefReg || TRI->isSuperRegister(DefReg, TargetReg))) { DEBUG(dbgs() << "Remove redundant Copy : "); DEBUG((MI)->print(dbgs())); MI->eraseFromParent(); Changed = true; LastChange = I; NumCopiesRemoved++; SmallestDef = TRI->isSubRegister(SmallestDef, DefReg) ? DefReg : SmallestDef; continue; } } if (MI->modifiesRegister(TargetReg, TRI)) break; } if (!Changed) return false; // Otherwise, we have to fixup the use-def chain, starting with the // CBZ/CBNZ. Conservatively mark as much as we can live. CompBr->clearRegisterKills(SmallestDef, TRI); if (std::none_of(TargetRegs.begin(), TargetRegs.end(), [&](unsigned Reg) { return MBB->isLiveIn(Reg); })) MBB->addLiveIn(TargetReg); // Clear any kills of TargetReg between CompBr and the last removed COPY. for (MachineInstr &MMI : make_range(MBB->begin()->getIterator(), LastChange->getIterator())) MMI.clearRegisterKills(SmallestDef, TRI); return true; }