/// Based on the use to defs information (in ADRPMode), compute the
/// opportunities of LOH ADRP-related.
static void computeADRP(const InstrToInstrs &UseToDefs,
AArch64FunctionInfo &AArch64FI,
const MachineDominatorTree *MDT) {
DEBUG(dbgs() << "*** Compute LOH for ADRP\n");
for (const auto &Entry : UseToDefs) {
unsigned Size = Entry.second.size();
if (Size == 0)
continue;
if (Size == 1) {
const MachineInstr *L2 = *Entry.second.begin();
const MachineInstr *L1 = Entry.first;
if (!MDT->dominates(L2, L1)) {
DEBUG(dbgs() << "Dominance check failed:\n" << *L2 << '\n' << *L1
<< '\n');
continue;
}
DEBUG(dbgs() << "Record AdrpAdrp:\n" << *L2 << '\n' << *L1 << '\n');
SmallVector<const MachineInstr *, 2> Args;
Args.push_back(L2);
Args.push_back(L1);
AArch64FI.addLOHDirective(MCLOH_AdrpAdrp, Args);
++NumADRPSimpleCandidate;
}
#ifdef DEBUG
else if (Size == 2)
++NumADRPComplexCandidate2;
else if (Size == 3)
++NumADRPComplexCandidate3;
else
++NumADRPComplexCandidateOther;
#endif
// if Size < 1, the use should have been removed from the candidates
assert(Size >= 1 && "No reaching defs for that use!");
}
}
static bool registerADRCandidate(const MachineInstr &Use,
const InstrToInstrs &UseToDefs,
const InstrToInstrs *DefsPerColorToUses,
AArch64FunctionInfo &AArch64FI,
SetOfMachineInstr *InvolvedInLOHs,
const MapRegToId &RegToId) {
// Look for opportunities to turn ADRP -> ADD or
// ADRP -> LDR GOTPAGEOFF into ADR.
// If ADRP has more than one use. Give up.
if (Use.getOpcode() != AArch64::ADDXri &&
(Use.getOpcode() != AArch64::LDRXui ||
!(Use.getOperand(2).getTargetFlags() & AArch64II::MO_GOT)))
return false;
InstrToInstrs::const_iterator It = UseToDefs.find(&Use);
// The map may contain garbage that we need to ignore.
if (It == UseToDefs.end() || It->second.empty())
return false;
const MachineInstr &Def = **It->second.begin();
if (Def.getOpcode() != AArch64::ADRP)
return false;
// Check the number of users of ADRP.
const SetOfMachineInstr *Users =
getUses(DefsPerColorToUses,
RegToId.find(Def.getOperand(0).getReg())->second, Def);
if (Users->size() > 1) {
++NumADRComplexCandidate;
return false;
}
++NumADRSimpleCandidate;
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(&Def)) &&
"ADRP already involved in LOH.");
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(&Use)) &&
"ADD already involved in LOH.");
DEBUG(dbgs() << "Record AdrpAdd\n" << Def << '\n' << Use << '\n');
SmallVector<const MachineInstr *, 2> Args;
Args.push_back(&Def);
Args.push_back(&Use);
AArch64FI.addLOHDirective(Use.getOpcode() == AArch64::ADDXri ? MCLOH_AdrpAdd
: MCLOH_AdrpLdrGot,
Args);
return true;
}
/// Update state when seeing and ADRP instruction.
static void handleADRP(const MachineInstr &MI, AArch64FunctionInfo &AFI,
LOHInfo &Info) {
if (Info.LastADRP != nullptr) {
DEBUG(dbgs() << "Adding MCLOH_AdrpAdrp:\n"
<< '\t' << MI << '\n'
<< '\t' << *Info.LastADRP << '\n');
AFI.addLOHDirective(MCLOH_AdrpAdrp, {&MI, Info.LastADRP});
++NumADRPSimpleCandidate;
}
// Produce LOH directive if possible.
if (Info.IsCandidate) {
switch (Info.Type) {
case MCLOH_AdrpAdd:
DEBUG(dbgs() << "Adding MCLOH_AdrpAdd:\n"
<< '\t' << MI << '\n'
<< '\t' << *Info.MI0 << '\n');
AFI.addLOHDirective(MCLOH_AdrpAdd, {&MI, Info.MI0});
++NumADRSimpleCandidate;
break;
case MCLOH_AdrpLdr:
if (supportLoadFromLiteral(*Info.MI0)) {
DEBUG(dbgs() << "Adding MCLOH_AdrpLdr:\n"
<< '\t' << MI << '\n'
<< '\t' << *Info.MI0 << '\n');
AFI.addLOHDirective(MCLOH_AdrpLdr, {&MI, Info.MI0});
++NumADRPToLDR;
}
break;
case MCLOH_AdrpAddLdr:
DEBUG(dbgs() << "Adding MCLOH_AdrpAddLdr:\n"
<< '\t' << MI << '\n'
<< '\t' << *Info.MI1 << '\n'
<< '\t' << *Info.MI0 << '\n');
AFI.addLOHDirective(MCLOH_AdrpAddLdr, {&MI, Info.MI1, Info.MI0});
++NumADDToLDR;
break;
case MCLOH_AdrpAddStr:
if (Info.MI1 != nullptr) {
DEBUG(dbgs() << "Adding MCLOH_AdrpAddStr:\n"
<< '\t' << MI << '\n'
<< '\t' << *Info.MI1 << '\n'
<< '\t' << *Info.MI0 << '\n');
AFI.addLOHDirective(MCLOH_AdrpAddStr, {&MI, Info.MI1, Info.MI0});
++NumADDToSTR;
}
break;
case MCLOH_AdrpLdrGotLdr:
DEBUG(dbgs() << "Adding MCLOH_AdrpLdrGotLdr:\n"
<< '\t' << MI << '\n'
<< '\t' << *Info.MI1 << '\n'
<< '\t' << *Info.MI0 << '\n');
AFI.addLOHDirective(MCLOH_AdrpLdrGotLdr, {&MI, Info.MI1, Info.MI0});
++NumLDRToLDR;
break;
case MCLOH_AdrpLdrGotStr:
DEBUG(dbgs() << "Adding MCLOH_AdrpLdrGotStr:\n"
<< '\t' << MI << '\n'
<< '\t' << *Info.MI1 << '\n'
<< '\t' << *Info.MI0 << '\n');
AFI.addLOHDirective(MCLOH_AdrpLdrGotStr, {&MI, Info.MI1, Info.MI0});
++NumLDRToSTR;
break;
case MCLOH_AdrpLdrGot:
DEBUG(dbgs() << "Adding MCLOH_AdrpLdrGot:\n"
<< '\t' << MI << '\n'
<< '\t' << *Info.MI0 << '\n');
AFI.addLOHDirective(MCLOH_AdrpLdrGot, {&MI, Info.MI0});
break;
case MCLOH_AdrpAdrp:
llvm_unreachable("MCLOH_AdrpAdrp not used in state machine");
}
}
handleClobber(Info);
Info.LastADRP = &MI;
}
/// Based on the use to defs information (in non-ADRPMode), compute the
/// opportunities of LOH non-ADRP-related
static void computeOthers(const InstrToInstrs &UseToDefs,
const InstrToInstrs *DefsPerColorToUses,
AArch64FunctionInfo &AArch64FI, const MapRegToId &RegToId,
const MachineDominatorTree *MDT) {
SetOfMachineInstr *InvolvedInLOHs = nullptr;
#ifdef DEBUG
SetOfMachineInstr InvolvedInLOHsStorage;
InvolvedInLOHs = &InvolvedInLOHsStorage;
#endif // DEBUG
DEBUG(dbgs() << "*** Compute LOH for Others\n");
// ADRP -> ADD/LDR -> LDR/STR pattern.
// Fall back to ADRP -> ADD pattern if we fail to catch the bigger pattern.
// FIXME: When the statistics are not important,
// This initial filtering loop can be merged into the next loop.
// Currently, we didn't do it to have the same code for both DEBUG and
// NDEBUG builds. Indeed, the iterator of the second loop would need
// to be changed.
SetOfMachineInstr PotentialCandidates;
SetOfMachineInstr PotentialADROpportunities;
for (auto &Use : UseToDefs) {
// If no definition is available, this is a non candidate.
if (Use.second.empty())
continue;
// Keep only instructions that are load or store and at the end of
// a ADRP -> ADD/LDR/Nothing chain.
// We already filtered out the no-chain cases.
if (!isCandidate(Use.first, UseToDefs, MDT)) {
PotentialADROpportunities.insert(Use.first);
continue;
}
PotentialCandidates.insert(Use.first);
}
// Make the following distinctions for statistics as the linker does
// know how to decode instructions:
// - ADD/LDR/Nothing make there different patterns.
// - LDR/STR make two different patterns.
// Hence, 6 - 1 base patterns.
// (because ADRP-> Nothing -> STR is not simplifiable)
// The linker is only able to have a simple semantic, i.e., if pattern A
// do B.
// However, we want to see the opportunity we may miss if we were able to
// catch more complex cases.
// PotentialCandidates are result of a chain ADRP -> ADD/LDR ->
// A potential candidate becomes a candidate, if its current immediate
// operand is zero and all nodes of the chain have respectively only one user
#ifdef DEBUG
SetOfMachineInstr DefsOfPotentialCandidates;
#endif
for (const MachineInstr *Candidate : PotentialCandidates) {
// Get the definition of the candidate i.e., ADD or LDR.
const MachineInstr *Def = *UseToDefs.find(Candidate)->second.begin();
// Record the elements of the chain.
const MachineInstr *L1 = Def;
const MachineInstr *L2 = nullptr;
unsigned ImmediateDefOpc = Def->getOpcode();
if (Def->getOpcode() != AArch64::ADRP) {
// Check the number of users of this node.
const SetOfMachineInstr *Users =
getUses(DefsPerColorToUses,
RegToId.find(Def->getOperand(0).getReg())->second, *Def);
if (Users->size() > 1) {
#ifdef DEBUG
// if all the uses of this def are in potential candidate, this is
// a complex candidate of level 2.
bool IsLevel2 = true;
for (const MachineInstr *MI : *Users) {
if (!PotentialCandidates.count(MI)) {
++NumTooCplxLvl2;
IsLevel2 = false;
break;
}
}
if (IsLevel2)
++NumCplxLvl2;
#endif // DEBUG
PotentialADROpportunities.insert(Def);
continue;
}
L2 = Def;
Def = *UseToDefs.find(Def)->second.begin();
L1 = Def;
} // else the element in the middle of the chain is nothing, thus
// Def already contains the first element of the chain.
// Check the number of users of the first node in the chain, i.e., ADRP
const SetOfMachineInstr *Users =
getUses(DefsPerColorToUses,
RegToId.find(Def->getOperand(0).getReg())->second, *Def);
if (Users->size() > 1) {
#ifdef DEBUG
// if all the uses of this def are in the defs of the potential candidate,
// this is a complex candidate of level 1
if (DefsOfPotentialCandidates.empty()) {
// lazy init
DefsOfPotentialCandidates = PotentialCandidates;
for (const MachineInstr *Candidate : PotentialCandidates) {
if (!UseToDefs.find(Candidate)->second.empty())
DefsOfPotentialCandidates.insert(
*UseToDefs.find(Candidate)->second.begin());
}
}
bool Found = false;
for (auto &Use : *Users) {
if (!DefsOfPotentialCandidates.count(Use)) {
++NumTooCplxLvl1;
Found = true;
break;
}
}
if (!Found)
++NumCplxLvl1;
#endif // DEBUG
continue;
}
bool IsL2Add = (ImmediateDefOpc == AArch64::ADDXri);
// If the chain is three instructions long and ldr is the second element,
// then this ldr must load form GOT, otherwise this is not a correct chain.
if (L2 && !IsL2Add &&
!(L2->getOperand(2).getTargetFlags() & AArch64II::MO_GOT))
continue;
SmallVector<const MachineInstr *, 3> Args;
MCLOHType Kind;
if (isCandidateLoad(Candidate)) {
if (!L2) {
// At this point, the candidate LOH indicates that the ldr instruction
// may use a direct access to the symbol. There is not such encoding
// for loads of byte and half.
if (!supportLoadFromLiteral(Candidate))
continue;
DEBUG(dbgs() << "Record AdrpLdr:\n" << *L1 << '\n' << *Candidate
<< '\n');
Kind = MCLOH_AdrpLdr;
Args.push_back(L1);
Args.push_back(Candidate);
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(L1)) &&
"L1 already involved in LOH.");
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(Candidate)) &&
"Candidate already involved in LOH.");
++NumADRPToLDR;
} else {
DEBUG(dbgs() << "Record Adrp" << (IsL2Add ? "Add" : "LdrGot")
<< "Ldr:\n" << *L1 << '\n' << *L2 << '\n' << *Candidate
<< '\n');
Kind = IsL2Add ? MCLOH_AdrpAddLdr : MCLOH_AdrpLdrGotLdr;
Args.push_back(L1);
Args.push_back(L2);
Args.push_back(Candidate);
PotentialADROpportunities.remove(L2);
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(L1)) &&
"L1 already involved in LOH.");
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(L2)) &&
"L2 already involved in LOH.");
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(Candidate)) &&
"Candidate already involved in LOH.");
#ifdef DEBUG
// get the immediate of the load
if (Candidate->getOperand(2).getImm() == 0)
if (ImmediateDefOpc == AArch64::ADDXri)
++NumADDToLDR;
else
++NumLDRToLDR;
else if (ImmediateDefOpc == AArch64::ADDXri)
++NumADDToLDRWithImm;
else
++NumLDRToLDRWithImm;
#endif // DEBUG
}
} else {
if (ImmediateDefOpc == AArch64::ADRP)
continue;
else {
DEBUG(dbgs() << "Record Adrp" << (IsL2Add ? "Add" : "LdrGot")
<< "Str:\n" << *L1 << '\n' << *L2 << '\n' << *Candidate
<< '\n');
Kind = IsL2Add ? MCLOH_AdrpAddStr : MCLOH_AdrpLdrGotStr;
Args.push_back(L1);
Args.push_back(L2);
Args.push_back(Candidate);
PotentialADROpportunities.remove(L2);
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(L1)) &&
"L1 already involved in LOH.");
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(L2)) &&
"L2 already involved in LOH.");
assert((!InvolvedInLOHs || InvolvedInLOHs->insert(Candidate)) &&
"Candidate already involved in LOH.");
#ifdef DEBUG
// get the immediate of the store
if (Candidate->getOperand(2).getImm() == 0)
if (ImmediateDefOpc == AArch64::ADDXri)
++NumADDToSTR;
else
++NumLDRToSTR;
else if (ImmediateDefOpc == AArch64::ADDXri)
++NumADDToSTRWithImm;
else
++NumLDRToSTRWithImm;
#endif // DEBUG
}
}
AArch64FI.addLOHDirective(Kind, Args);
}
// Now, we grabbed all the big patterns, check ADR opportunities.
for (const MachineInstr *Candidate : PotentialADROpportunities)
registerADRCandidate(*Candidate, UseToDefs, DefsPerColorToUses, AArch64FI,
InvolvedInLOHs, RegToId);
}