bool RegisterBankInfo::InstructionMapping::verify(
const MachineInstr &MI) const {
// Check that all the register operands are properly mapped.
// Check the constructor invariant.
// For PHI, we only care about mapping the definition.
assert(NumOperands == (isCopyLike(MI) ? 1 : MI.getNumOperands()) &&
"NumOperands must match, see constructor");
assert(MI.getParent() && MI.getMF() &&
"MI must be connected to a MachineFunction");
const MachineFunction &MF = *MI.getMF();
const RegisterBankInfo *RBI = MF.getSubtarget().getRegBankInfo();
(void)RBI;
for (unsigned Idx = 0; Idx < NumOperands; ++Idx) {
const MachineOperand &MO = MI.getOperand(Idx);
if (!MO.isReg()) {
assert(!getOperandMapping(Idx).isValid() &&
"We should not care about non-reg mapping");
continue;
}
unsigned Reg = MO.getReg();
if (!Reg)
continue;
assert(getOperandMapping(Idx).isValid() &&
"We must have a mapping for reg operands");
const RegisterBankInfo::ValueMapping &MOMapping = getOperandMapping(Idx);
(void)MOMapping;
// Register size in bits.
// This size must match what the mapping expects.
assert(MOMapping.verify(RBI->getSizeInBits(
Reg, MF.getRegInfo(), *MF.getSubtarget().getRegisterInfo())) &&
"Value mapping is invalid");
}
return true;
}
void MIPrinter::print(const MachineInstr &MI) {
const auto *MF = MI.getMF();
const auto &MRI = MF->getRegInfo();
const auto &SubTarget = MF->getSubtarget();
const auto *TRI = SubTarget.getRegisterInfo();
assert(TRI && "Expected target register info");
const auto *TII = SubTarget.getInstrInfo();
assert(TII && "Expected target instruction info");
if (MI.isCFIInstruction())
assert(MI.getNumOperands() == 1 && "Expected 1 operand in CFI instruction");
SmallBitVector PrintedTypes(8);
bool ShouldPrintRegisterTies = MI.hasComplexRegisterTies();
unsigned I = 0, E = MI.getNumOperands();
for (; I < E && MI.getOperand(I).isReg() && MI.getOperand(I).isDef() &&
!MI.getOperand(I).isImplicit();
++I) {
if (I)
OS << ", ";
print(MI, I, TRI, ShouldPrintRegisterTies,
MI.getTypeToPrint(I, PrintedTypes, MRI),
/*PrintDef=*/false);
}
if (I)
OS << " = ";
if (MI.getFlag(MachineInstr::FrameSetup))
OS << "frame-setup ";
OS << TII->getName(MI.getOpcode());
if (I < E)
OS << ' ';
bool NeedComma = false;
for (; I < E; ++I) {
if (NeedComma)
OS << ", ";
print(MI, I, TRI, ShouldPrintRegisterTies,
MI.getTypeToPrint(I, PrintedTypes, MRI));
NeedComma = true;
}
if (MI.getDebugLoc()) {
if (NeedComma)
OS << ',';
OS << " debug-location ";
MI.getDebugLoc()->printAsOperand(OS, MST);
}
if (!MI.memoperands_empty()) {
OS << " :: ";
const LLVMContext &Context = MF->getFunction().getContext();
bool NeedComma = false;
for (const auto *Op : MI.memoperands()) {
if (NeedComma)
OS << ", ";
print(Context, *TII, *Op);
NeedComma = true;
}
}
}
void TargetInstrInfo::genAlternativeCodeSequence(
MachineInstr &Root, MachineCombinerPattern Pattern,
SmallVectorImpl<MachineInstr *> &InsInstrs,
SmallVectorImpl<MachineInstr *> &DelInstrs,
DenseMap<unsigned, unsigned> &InstIdxForVirtReg) const {
MachineRegisterInfo &MRI = Root.getMF()->getRegInfo();
// Select the previous instruction in the sequence based on the input pattern.
MachineInstr *Prev = nullptr;
switch (Pattern) {
case MachineCombinerPattern::REASSOC_AX_BY:
case MachineCombinerPattern::REASSOC_XA_BY:
Prev = MRI.getUniqueVRegDef(Root.getOperand(1).getReg());
break;
case MachineCombinerPattern::REASSOC_AX_YB:
case MachineCombinerPattern::REASSOC_XA_YB:
Prev = MRI.getUniqueVRegDef(Root.getOperand(2).getReg());
break;
default:
break;
}
assert(Prev && "Unknown pattern for machine combiner");
reassociateOps(Root, *Prev, Pattern, InsInstrs, DelInstrs, InstIdxForVirtReg);
}
void MIPrinter::print(const MachineInstr &MI, unsigned OpIdx,
const TargetRegisterInfo *TRI,
bool ShouldPrintRegisterTies, LLT TypeToPrint,
bool PrintDef) {
const MachineOperand &Op = MI.getOperand(OpIdx);
switch (Op.getType()) {
case MachineOperand::MO_Immediate:
if (MI.isOperandSubregIdx(OpIdx)) {
MachineOperand::printTargetFlags(OS, Op);
MachineOperand::printSubRegIdx(OS, Op.getImm(), TRI);
break;
}
LLVM_FALLTHROUGH;
case MachineOperand::MO_Register:
case MachineOperand::MO_CImmediate:
case MachineOperand::MO_FPImmediate:
case MachineOperand::MO_MachineBasicBlock:
case MachineOperand::MO_ConstantPoolIndex:
case MachineOperand::MO_TargetIndex:
case MachineOperand::MO_JumpTableIndex:
case MachineOperand::MO_ExternalSymbol:
case MachineOperand::MO_GlobalAddress:
case MachineOperand::MO_RegisterLiveOut:
case MachineOperand::MO_Metadata:
case MachineOperand::MO_MCSymbol:
case MachineOperand::MO_CFIIndex:
case MachineOperand::MO_IntrinsicID:
case MachineOperand::MO_Predicate:
case MachineOperand::MO_BlockAddress: {
unsigned TiedOperandIdx = 0;
if (ShouldPrintRegisterTies && Op.isReg() && Op.isTied() && !Op.isDef())
TiedOperandIdx = Op.getParent()->findTiedOperandIdx(OpIdx);
const TargetIntrinsicInfo *TII = MI.getMF()->getTarget().getIntrinsicInfo();
Op.print(OS, MST, TypeToPrint, PrintDef, /*IsStandalone=*/false,
ShouldPrintRegisterTies, TiedOperandIdx, TRI, TII);
break;
}
case MachineOperand::MO_FrameIndex:
printStackObjectReference(Op.getIndex());
break;
case MachineOperand::MO_RegisterMask: {
auto RegMaskInfo = RegisterMaskIds.find(Op.getRegMask());
if (RegMaskInfo != RegisterMaskIds.end())
OS << StringRef(TRI->getRegMaskNames()[RegMaskInfo->second]).lower();
else
printCustomRegMask(Op.getRegMask(), OS, TRI);
break;
}
}
}
void GCNRPTracker::reset(const MachineInstr &MI,
const LiveRegSet *LiveRegsCopy,
bool After) {
const MachineFunction &MF = *MI.getMF();
MRI = &MF.getRegInfo();
if (LiveRegsCopy) {
if (&LiveRegs != LiveRegsCopy)
LiveRegs = *LiveRegsCopy;
} else {
LiveRegs = After ? getLiveRegsAfter(MI, LIS)
: getLiveRegsBefore(MI, LIS);
}
MaxPressure = CurPressure = getRegPressure(*MRI, LiveRegs);
}
int TargetInstrInfo::getSPAdjust(const MachineInstr &MI) const {
const MachineFunction *MF = MI.getMF();
const TargetFrameLowering *TFI = MF->getSubtarget().getFrameLowering();
bool StackGrowsDown =
TFI->getStackGrowthDirection() == TargetFrameLowering::StackGrowsDown;
unsigned FrameSetupOpcode = getCallFrameSetupOpcode();
unsigned FrameDestroyOpcode = getCallFrameDestroyOpcode();
if (!isFrameInstr(MI))
return 0;
int SPAdj = TFI->alignSPAdjust(getFrameSize(MI));
if ((!StackGrowsDown && MI.getOpcode() == FrameSetupOpcode) ||
(StackGrowsDown && MI.getOpcode() == FrameDestroyOpcode))
SPAdj = -SPAdj;
return SPAdj;
}
void MIPrinter::print(const MachineInstr &MI) {
const auto *MF = MI.getMF();
const auto &MRI = MF->getRegInfo();
const auto &SubTarget = MF->getSubtarget();
const auto *TRI = SubTarget.getRegisterInfo();
assert(TRI && "Expected target register info");
const auto *TII = SubTarget.getInstrInfo();
assert(TII && "Expected target instruction info");
if (MI.isCFIInstruction())
assert(MI.getNumOperands() == 1 && "Expected 1 operand in CFI instruction");
SmallBitVector PrintedTypes(8);
bool ShouldPrintRegisterTies = MI.hasComplexRegisterTies();
unsigned I = 0, E = MI.getNumOperands();
for (; I < E && MI.getOperand(I).isReg() && MI.getOperand(I).isDef() &&
!MI.getOperand(I).isImplicit();
++I) {
if (I)
OS << ", ";
print(MI, I, TRI, ShouldPrintRegisterTies,
MI.getTypeToPrint(I, PrintedTypes, MRI),
/*PrintDef=*/false);
}
if (I)
OS << " = ";
if (MI.getFlag(MachineInstr::FrameSetup))
OS << "frame-setup ";
if (MI.getFlag(MachineInstr::FrameDestroy))
OS << "frame-destroy ";
if (MI.getFlag(MachineInstr::FmNoNans))
OS << "nnan ";
if (MI.getFlag(MachineInstr::FmNoInfs))
OS << "ninf ";
if (MI.getFlag(MachineInstr::FmNsz))
OS << "nsz ";
if (MI.getFlag(MachineInstr::FmArcp))
OS << "arcp ";
if (MI.getFlag(MachineInstr::FmContract))
OS << "contract ";
if (MI.getFlag(MachineInstr::FmAfn))
OS << "afn ";
if (MI.getFlag(MachineInstr::FmReassoc))
OS << "reassoc ";
if (MI.getFlag(MachineInstr::NoUWrap))
OS << "nuw ";
if (MI.getFlag(MachineInstr::NoSWrap))
OS << "nsw ";
if (MI.getFlag(MachineInstr::IsExact))
OS << "exact ";
OS << TII->getName(MI.getOpcode());
if (I < E)
OS << ' ';
bool NeedComma = false;
for (; I < E; ++I) {
if (NeedComma)
OS << ", ";
print(MI, I, TRI, ShouldPrintRegisterTies,
MI.getTypeToPrint(I, PrintedTypes, MRI));
NeedComma = true;
}
// Print any optional symbols attached to this instruction as-if they were
// operands.
if (MCSymbol *PreInstrSymbol = MI.getPreInstrSymbol()) {
if (NeedComma)
OS << ',';
OS << " pre-instr-symbol ";
MachineOperand::printSymbol(OS, *PreInstrSymbol);
NeedComma = true;
}
if (MCSymbol *PostInstrSymbol = MI.getPostInstrSymbol()) {
if (NeedComma)
OS << ',';
OS << " post-instr-symbol ";
MachineOperand::printSymbol(OS, *PostInstrSymbol);
NeedComma = true;
}
if (const DebugLoc &DL = MI.getDebugLoc()) {
if (NeedComma)
OS << ',';
OS << " debug-location ";
DL->printAsOperand(OS, MST);
}
if (!MI.memoperands_empty()) {
OS << " :: ";
const LLVMContext &Context = MF->getFunction().getContext();
const MachineFrameInfo &MFI = MF->getFrameInfo();
bool NeedComma = false;
for (const auto *Op : MI.memoperands()) {
if (NeedComma)
OS << ", ";
Op->print(OS, MST, SSNs, Context, &MFI, TII);
NeedComma = true;
}
}
}
bool TargetInstrInfo::isReallyTriviallyReMaterializableGeneric(
const MachineInstr &MI, AliasAnalysis *AA) const {
const MachineFunction &MF = *MI.getMF();
const MachineRegisterInfo &MRI = MF.getRegInfo();
// Remat clients assume operand 0 is the defined register.
if (!MI.getNumOperands() || !MI.getOperand(0).isReg())
return false;
unsigned DefReg = MI.getOperand(0).getReg();
// A sub-register definition can only be rematerialized if the instruction
// doesn't read the other parts of the register. Otherwise it is really a
// read-modify-write operation on the full virtual register which cannot be
// moved safely.
if (TargetRegisterInfo::isVirtualRegister(DefReg) &&
MI.getOperand(0).getSubReg() && MI.readsVirtualRegister(DefReg))
return false;
// A load from a fixed stack slot can be rematerialized. This may be
// redundant with subsequent checks, but it's target-independent,
// simple, and a common case.
int FrameIdx = 0;
if (isLoadFromStackSlot(MI, FrameIdx) &&
MF.getFrameInfo().isImmutableObjectIndex(FrameIdx))
return true;
// Avoid instructions obviously unsafe for remat.
if (MI.isNotDuplicable() || MI.mayStore() || MI.hasUnmodeledSideEffects())
return false;
// Don't remat inline asm. We have no idea how expensive it is
// even if it's side effect free.
if (MI.isInlineAsm())
return false;
// Avoid instructions which load from potentially varying memory.
if (MI.mayLoad() && !MI.isDereferenceableInvariantLoad(AA))
return false;
// If any of the registers accessed are non-constant, conservatively assume
// the instruction is not rematerializable.
for (unsigned i = 0, e = MI.getNumOperands(); i != e; ++i) {
const MachineOperand &MO = MI.getOperand(i);
if (!MO.isReg()) continue;
unsigned Reg = MO.getReg();
if (Reg == 0)
continue;
// Check for a well-behaved physical register.
if (TargetRegisterInfo::isPhysicalRegister(Reg)) {
if (MO.isUse()) {
// If the physreg has no defs anywhere, it's just an ambient register
// and we can freely move its uses. Alternatively, if it's allocatable,
// it could get allocated to something with a def during allocation.
if (!MRI.isConstantPhysReg(Reg))
return false;
} else {
// A physreg def. We can't remat it.
return false;
}
continue;
}
// Only allow one virtual-register def. There may be multiple defs of the
// same virtual register, though.
if (MO.isDef() && Reg != DefReg)
return false;
// Don't allow any virtual-register uses. Rematting an instruction with
// virtual register uses would length the live ranges of the uses, which
// is not necessarily a good idea, certainly not "trivial".
if (MO.isUse())
return false;
}
// Everything checked out.
return true;
}
/// Attempt the reassociation transformation to reduce critical path length.
/// See the above comments before getMachineCombinerPatterns().
void TargetInstrInfo::reassociateOps(
MachineInstr &Root, MachineInstr &Prev,
MachineCombinerPattern Pattern,
SmallVectorImpl<MachineInstr *> &InsInstrs,
SmallVectorImpl<MachineInstr *> &DelInstrs,
DenseMap<unsigned, unsigned> &InstrIdxForVirtReg) const {
MachineFunction *MF = Root.getMF();
MachineRegisterInfo &MRI = MF->getRegInfo();
const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
const TargetRegisterClass *RC = Root.getRegClassConstraint(0, TII, TRI);
// This array encodes the operand index for each parameter because the
// operands may be commuted. Each row corresponds to a pattern value,
// and each column specifies the index of A, B, X, Y.
unsigned OpIdx[4][4] = {
{ 1, 1, 2, 2 },
{ 1, 2, 2, 1 },
{ 2, 1, 1, 2 },
{ 2, 2, 1, 1 }
};
int Row;
switch (Pattern) {
case MachineCombinerPattern::REASSOC_AX_BY: Row = 0; break;
case MachineCombinerPattern::REASSOC_AX_YB: Row = 1; break;
case MachineCombinerPattern::REASSOC_XA_BY: Row = 2; break;
case MachineCombinerPattern::REASSOC_XA_YB: Row = 3; break;
default: llvm_unreachable("unexpected MachineCombinerPattern");
}
MachineOperand &OpA = Prev.getOperand(OpIdx[Row][0]);
MachineOperand &OpB = Root.getOperand(OpIdx[Row][1]);
MachineOperand &OpX = Prev.getOperand(OpIdx[Row][2]);
MachineOperand &OpY = Root.getOperand(OpIdx[Row][3]);
MachineOperand &OpC = Root.getOperand(0);
unsigned RegA = OpA.getReg();
unsigned RegB = OpB.getReg();
unsigned RegX = OpX.getReg();
unsigned RegY = OpY.getReg();
unsigned RegC = OpC.getReg();
if (TargetRegisterInfo::isVirtualRegister(RegA))
MRI.constrainRegClass(RegA, RC);
if (TargetRegisterInfo::isVirtualRegister(RegB))
MRI.constrainRegClass(RegB, RC);
if (TargetRegisterInfo::isVirtualRegister(RegX))
MRI.constrainRegClass(RegX, RC);
if (TargetRegisterInfo::isVirtualRegister(RegY))
MRI.constrainRegClass(RegY, RC);
if (TargetRegisterInfo::isVirtualRegister(RegC))
MRI.constrainRegClass(RegC, RC);
// Create a new virtual register for the result of (X op Y) instead of
// recycling RegB because the MachineCombiner's computation of the critical
// path requires a new register definition rather than an existing one.
unsigned NewVR = MRI.createVirtualRegister(RC);
InstrIdxForVirtReg.insert(std::make_pair(NewVR, 0));
unsigned Opcode = Root.getOpcode();
bool KillA = OpA.isKill();
bool KillX = OpX.isKill();
bool KillY = OpY.isKill();
// Create new instructions for insertion.
MachineInstrBuilder MIB1 =
BuildMI(*MF, Prev.getDebugLoc(), TII->get(Opcode), NewVR)
.addReg(RegX, getKillRegState(KillX))
.addReg(RegY, getKillRegState(KillY));
MachineInstrBuilder MIB2 =
BuildMI(*MF, Root.getDebugLoc(), TII->get(Opcode), RegC)
.addReg(RegA, getKillRegState(KillA))
.addReg(NewVR, getKillRegState(true));
setSpecialOperandAttr(Root, Prev, *MIB1, *MIB2);
// Record new instructions for insertion and old instructions for deletion.
InsInstrs.push_back(MIB1);
InsInstrs.push_back(MIB2);
DelInstrs.push_back(&Prev);
DelInstrs.push_back(&Root);
}