/// Split the basic block containing MI into two blocks, which are joined by
/// an unconditional branch. Update data structures and renumber blocks to
/// account for this change and returns the newly created block.
MachineBasicBlock *BranchRelaxation::splitBlockBeforeInstr(MachineInstr &MI,
MachineBasicBlock *DestBB) {
MachineBasicBlock *OrigBB = MI.getParent();
// Create a new MBB for the code after the OrigBB.
MachineBasicBlock *NewBB =
MF->CreateMachineBasicBlock(OrigBB->getBasicBlock());
MF->insert(++OrigBB->getIterator(), NewBB);
// Splice the instructions starting with MI over to NewBB.
NewBB->splice(NewBB->end(), OrigBB, MI.getIterator(), OrigBB->end());
// Add an unconditional branch from OrigBB to NewBB.
// Note the new unconditional branch is not being recorded.
// There doesn't seem to be meaningful DebugInfo available; this doesn't
// correspond to anything in the source.
TII->insertUnconditionalBranch(*OrigBB, NewBB, DebugLoc());
// Insert an entry into BlockInfo to align it properly with the block numbers.
BlockInfo.insert(BlockInfo.begin() + NewBB->getNumber(), BasicBlockInfo());
NewBB->transferSuccessors(OrigBB);
OrigBB->addSuccessor(NewBB);
OrigBB->addSuccessor(DestBB);
// Cleanup potential unconditional branch to successor block.
// Note that updateTerminator may change the size of the blocks.
NewBB->updateTerminator();
OrigBB->updateTerminator();
// Figure out how large the OrigBB is. As the first half of the original
// block, it cannot contain a tablejump. The size includes
// the new jump we added. (It should be possible to do this without
// recounting everything, but it's very confusing, and this is rarely
// executed.)
BlockInfo[OrigBB->getNumber()].Size = computeBlockSize(*OrigBB);
// Figure out how large the NewMBB is. As the second half of the original
// block, it may contain a tablejump.
BlockInfo[NewBB->getNumber()].Size = computeBlockSize(*NewBB);
// All BBOffsets following these blocks must be modified.
adjustBlockOffsets(*OrigBB);
// Need to fix live-in lists if we track liveness.
if (TRI->trackLivenessAfterRegAlloc(*MF))
computeLiveIns(LiveRegs, *TRI, *NewBB);
++NumSplit;
return NewBB;
}
void X86RetpolineThunks::populateThunk(MachineFunction &MF,
unsigned Reg) {
// Set MF properties. We never use vregs...
MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs);
// Grab the entry MBB and erase any other blocks. O0 codegen appears to
// generate two bbs for the entry block.
MachineBasicBlock *Entry = &MF.front();
Entry->clear();
while (MF.size() > 1)
MF.erase(std::next(MF.begin()));
MachineBasicBlock *CaptureSpec = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
MachineBasicBlock *CallTarget = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
MCSymbol *TargetSym = MF.getContext().createTempSymbol();
MF.push_back(CaptureSpec);
MF.push_back(CallTarget);
const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
const unsigned RetOpc = Is64Bit ? X86::RETQ : X86::RETL;
Entry->addLiveIn(Reg);
BuildMI(Entry, DebugLoc(), TII->get(CallOpc)).addSym(TargetSym);
// The MIR verifier thinks that the CALL in the entry block will fall through
// to CaptureSpec, so mark it as the successor. Technically, CaptureTarget is
// the successor, but the MIR verifier doesn't know how to cope with that.
Entry->addSuccessor(CaptureSpec);
// In the capture loop for speculation, we want to stop the processor from
// speculating as fast as possible. On Intel processors, the PAUSE instruction
// will block speculation without consuming any execution resources. On AMD
// processors, the PAUSE instruction is (essentially) a nop, so we also use an
// LFENCE instruction which they have advised will stop speculation as well
// with minimal resource utilization. We still end the capture with a jump to
// form an infinite loop to fully guarantee that no matter what implementation
// of the x86 ISA, speculating this code path never escapes.
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::PAUSE));
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::LFENCE));
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::JMP_1)).addMBB(CaptureSpec);
CaptureSpec->setHasAddressTaken();
CaptureSpec->addSuccessor(CaptureSpec);
CallTarget->addLiveIn(Reg);
CallTarget->setHasAddressTaken();
CallTarget->setAlignment(4);
insertRegReturnAddrClobber(*CallTarget, Reg);
CallTarget->back().setPreInstrSymbol(MF, TargetSym);
BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc));
}
// MI is a load-register-on-condition pseudo instruction that could not be
// handled as a single hardware instruction. Replace it by a branch sequence.
bool SystemZExpandPseudo::expandLOCRMux(MachineBasicBlock &MBB,
MachineBasicBlock::iterator MBBI,
MachineBasicBlock::iterator &NextMBBI) {
MachineFunction &MF = *MBB.getParent();
const BasicBlock *BB = MBB.getBasicBlock();
MachineInstr &MI = *MBBI;
DebugLoc DL = MI.getDebugLoc();
unsigned DestReg = MI.getOperand(0).getReg();
unsigned SrcReg = MI.getOperand(2).getReg();
unsigned CCValid = MI.getOperand(3).getImm();
unsigned CCMask = MI.getOperand(4).getImm();
LivePhysRegs LiveRegs(TII->getRegisterInfo());
LiveRegs.addLiveOuts(MBB);
for (auto I = std::prev(MBB.end()); I != MBBI; --I)
LiveRegs.stepBackward(*I);
// Splice MBB at MI, moving the rest of the block into RestMBB.
MachineBasicBlock *RestMBB = MF.CreateMachineBasicBlock(BB);
MF.insert(std::next(MachineFunction::iterator(MBB)), RestMBB);
RestMBB->splice(RestMBB->begin(), &MBB, MI, MBB.end());
RestMBB->transferSuccessors(&MBB);
for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I)
RestMBB->addLiveIn(*I);
// Create a new block MoveMBB to hold the move instruction.
MachineBasicBlock *MoveMBB = MF.CreateMachineBasicBlock(BB);
MF.insert(std::next(MachineFunction::iterator(MBB)), MoveMBB);
MoveMBB->addLiveIn(SrcReg);
for (auto I = LiveRegs.begin(); I != LiveRegs.end(); ++I)
MoveMBB->addLiveIn(*I);
// At the end of MBB, create a conditional branch to RestMBB if the
// condition is false, otherwise fall through to MoveMBB.
BuildMI(&MBB, DL, TII->get(SystemZ::BRC))
.addImm(CCValid).addImm(CCMask ^ CCValid).addMBB(RestMBB);
MBB.addSuccessor(RestMBB);
MBB.addSuccessor(MoveMBB);
// In MoveMBB, emit an instruction to move SrcReg into DestReg,
// then fall through to RestMBB.
TII->copyPhysReg(*MoveMBB, MoveMBB->end(), DL, DestReg, SrcReg,
MI.getOperand(2).isKill());
MoveMBB->addSuccessor(RestMBB);
NextMBBI = MBB.end();
MI.eraseFromParent();
return true;
}
bool MIRParserImpl::initializeMachineBasicBlock(
MachineFunction &MF, MachineBasicBlock &MBB,
const yaml::MachineBasicBlock &YamlMBB,
const PerFunctionMIParsingState &PFS) {
MBB.setAlignment(YamlMBB.Alignment);
if (YamlMBB.AddressTaken)
MBB.setHasAddressTaken();
MBB.setIsLandingPad(YamlMBB.IsLandingPad);
SMDiagnostic Error;
// Parse the successors.
for (const auto &MBBSource : YamlMBB.Successors) {
MachineBasicBlock *SuccMBB = nullptr;
if (parseMBBReference(SuccMBB, MBBSource, MF, PFS))
return true;
// TODO: Report an error when adding the same successor more than once.
MBB.addSuccessor(SuccMBB);
}
// Parse the liveins.
for (const auto &LiveInSource : YamlMBB.LiveIns) {
unsigned Reg = 0;
if (parseNamedRegisterReference(Reg, SM, MF, LiveInSource.Value, PFS,
IRSlots, Error))
return error(Error, LiveInSource.SourceRange);
MBB.addLiveIn(Reg);
}
// Parse the instructions.
for (const auto &MISource : YamlMBB.Instructions) {
MachineInstr *MI = nullptr;
if (parseMachineInstr(MI, SM, MF, MISource.Value, PFS, IRSlots, Error))
return error(Error, MISource.SourceRange);
MBB.insert(MBB.end(), MI);
}
return false;
}
static void addEpilogOnlyR10(const TargetInstrInfo *TII, MachineBasicBlock &MBB,
MachineInstr &MI, MachineBasicBlock &TrapBB) {
const DebugLoc &DL = MI.getDebugLoc();
// xor r10, r10
BuildMI(MBB, MI, DL, TII->get(X86::XOR64rr))
.addDef(X86::R10)
.addReg(X86::R10, RegState::Undef)
.addReg(X86::R10, RegState::Undef);
// mov r10, [gs:r10]
addSegmentedMem(BuildMI(MBB, MI, DL, TII->get(X86::MOV64rm)).addDef(X86::R10),
X86::GS, X86::R10);
// mov r10, [gs:r10]
addSegmentedMem(BuildMI(MBB, MI, DL, TII->get(X86::MOV64rm)).addDef(X86::R10),
X86::GS, X86::R10);
// sub QWORD [gs:0], 8
// This instruction should not be moved up to avoid a signal race.
addSegmentedMem(BuildMI(MBB, MI, DL, TII->get(X86::SUB64mi8)), X86::GS, 0)
.addImm(8);
// cmp [rsp], r10
addDirectMem(BuildMI(MBB, MI, DL, TII->get(X86::CMP64mr)), X86::RSP)
.addReg(X86::R10);
// jne trap
BuildMI(MBB, MI, DL, TII->get(X86::JNE_1)).addMBB(&TrapBB);
MBB.addSuccessor(&TrapBB);
}
/// Insert a LOOP marker for a loop starting at MBB (if it's a loop header).
static void PlaceLoopMarker(MachineBasicBlock &MBB, MachineFunction &MF,
SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
const WebAssemblyInstrInfo &TII,
const MachineLoopInfo &MLI) {
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (!Loop || Loop->getHeader() != &MBB)
return;
// The operand of a LOOP is the first block after the loop. If the loop is the
// bottom of the function, insert a dummy block at the end.
MachineBasicBlock *Bottom = LoopBottom(Loop);
auto Iter = next(MachineFunction::iterator(Bottom));
if (Iter == MF.end()) {
MachineBasicBlock *Label = MF.CreateMachineBasicBlock();
// Give it a fake predecessor so that AsmPrinter prints its label.
Label->addSuccessor(Label);
MF.push_back(Label);
Iter = next(MachineFunction::iterator(Bottom));
}
MachineBasicBlock *AfterLoop = &*Iter;
BuildMI(MBB, MBB.begin(), DebugLoc(), TII.get(WebAssembly::LOOP))
.addMBB(AfterLoop);
// Emit a special no-op telling the asm printer that we need a label to close
// the loop scope, even though the destination is only reachable by
// fallthrough.
if (!Bottom->back().isBarrier())
BuildMI(*Bottom, Bottom->end(), DebugLoc(), TII.get(WebAssembly::LOOP_END));
assert((!ScopeTops[AfterLoop->getNumber()] ||
ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) &&
"With RPO we should visit the outer-most loop for a block first.");
if (!ScopeTops[AfterLoop->getNumber()])
ScopeTops[AfterLoop->getNumber()] = &MBB;
}
static void addEpilogLeaf(const TargetInstrInfo *TII, MachineBasicBlock &MBB,
MachineInstr &MI, MachineBasicBlock &TrapBB,
MCPhysReg FreeRegister) {
const DebugLoc &DL = MI.getDebugLoc();
// cmp [rsp], REG
addDirectMem(BuildMI(MBB, MI, DL, TII->get(X86::CMP64mr)), X86::RSP)
.addReg(FreeRegister);
// jne trap
BuildMI(MBB, MI, DL, TII->get(X86::JNE_1)).addMBB(&TrapBB);
MBB.addSuccessor(&TrapBB);
}
void X86RetpolineThunks::populateThunk(MachineFunction &MF,
Optional<unsigned> Reg) {
// Set MF properties. We never use vregs...
MF.getProperties().set(MachineFunctionProperties::Property::NoVRegs);
MachineBasicBlock *Entry = &MF.front();
Entry->clear();
MachineBasicBlock *CaptureSpec = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
MachineBasicBlock *CallTarget = MF.CreateMachineBasicBlock(Entry->getBasicBlock());
MF.push_back(CaptureSpec);
MF.push_back(CallTarget);
const unsigned CallOpc = Is64Bit ? X86::CALL64pcrel32 : X86::CALLpcrel32;
const unsigned RetOpc = Is64Bit ? X86::RETQ : X86::RETL;
BuildMI(Entry, DebugLoc(), TII->get(CallOpc)).addMBB(CallTarget);
Entry->addSuccessor(CallTarget);
Entry->addSuccessor(CaptureSpec);
CallTarget->setHasAddressTaken();
// In the capture loop for speculation, we want to stop the processor from
// speculating as fast as possible. On Intel processors, the PAUSE instruction
// will block speculation without consuming any execution resources. On AMD
// processors, the PAUSE instruction is (essentially) a nop, so we also use an
// LFENCE instruction which they have advised will stop speculation as well
// with minimal resource utilization. We still end the capture with a jump to
// form an infinite loop to fully guarantee that no matter what implementation
// of the x86 ISA, speculating this code path never escapes.
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::PAUSE));
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::LFENCE));
BuildMI(CaptureSpec, DebugLoc(), TII->get(X86::JMP_1)).addMBB(CaptureSpec);
CaptureSpec->setHasAddressTaken();
CaptureSpec->addSuccessor(CaptureSpec);
CallTarget->setAlignment(4);
insertRegReturnAddrClobber(*CallTarget, *Reg);
BuildMI(CallTarget, DebugLoc(), TII->get(RetOpc));
}
MachineBasicBlock *PHIElimination::SplitCriticalEdge(MachineBasicBlock *A,
MachineBasicBlock *B) {
assert(A && B && "Missing MBB end point");
MachineFunction *MF = A->getParent();
// We may need to update A's terminator, but we can't do that if AnalyzeBranch
// fails. If A uses a jump table, we won't touch it.
const TargetInstrInfo *TII = MF->getTarget().getInstrInfo();
MachineBasicBlock *TBB = 0, *FBB = 0;
SmallVector<MachineOperand, 4> Cond;
if (TII->AnalyzeBranch(*A, TBB, FBB, Cond))
return NULL;
++NumSplits;
MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock();
MF->insert(llvm::next(MachineFunction::iterator(A)), NMBB);
DEBUG(dbgs() << "PHIElimination splitting critical edge:"
" BB#" << A->getNumber()
<< " -- BB#" << NMBB->getNumber()
<< " -- BB#" << B->getNumber() << '\n');
A->ReplaceUsesOfBlockWith(B, NMBB);
A->updateTerminator();
// Insert unconditional "jump B" instruction in NMBB if necessary.
NMBB->addSuccessor(B);
if (!NMBB->isLayoutSuccessor(B)) {
Cond.clear();
MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, B, NULL, Cond);
}
// Fix PHI nodes in B so they refer to NMBB instead of A
for (MachineBasicBlock::iterator i = B->begin(), e = B->end();
i != e && i->isPHI(); ++i)
for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2)
if (i->getOperand(ni+1).getMBB() == A)
i->getOperand(ni+1).setMBB(NMBB);
if (LiveVariables *LV=getAnalysisIfAvailable<LiveVariables>())
LV->addNewBlock(NMBB, A, B);
if (MachineDominatorTree *MDT=getAnalysisIfAvailable<MachineDominatorTree>())
MDT->addNewBlock(NMBB, A);
return NMBB;
}
/// ReplaceTailWithBranchTo - Delete the instruction OldInst and everything
/// after it, replacing it with an unconditional branch to NewDest.
void
TargetInstrInfoImpl::ReplaceTailWithBranchTo(MachineBasicBlock::iterator Tail,
MachineBasicBlock *NewDest) const {
MachineBasicBlock *MBB = Tail->getParent();
// Remove all the old successors of MBB from the CFG.
while (!MBB->succ_empty())
MBB->removeSuccessor(MBB->succ_begin());
// Remove all the dead instructions from the end of MBB.
MBB->erase(Tail, MBB->end());
// If MBB isn't immediately before MBB, insert a branch to it.
if (++MachineFunction::iterator(MBB) != MachineFunction::iterator(NewDest))
InsertBranch(*MBB, NewDest, 0, SmallVector<MachineOperand, 0>(),
Tail->getDebugLoc());
MBB->addSuccessor(NewDest);
}
/// Insert a LOOP marker for a loop starting at MBB (if it's a loop header).
static void PlaceLoopMarker(
MachineBasicBlock &MBB, MachineFunction &MF,
SmallVectorImpl<MachineBasicBlock *> &ScopeTops,
DenseMap<const MachineInstr *, const MachineBasicBlock *> &LoopTops,
const WebAssemblyInstrInfo &TII, const MachineLoopInfo &MLI) {
MachineLoop *Loop = MLI.getLoopFor(&MBB);
if (!Loop || Loop->getHeader() != &MBB)
return;
// The operand of a LOOP is the first block after the loop. If the loop is the
// bottom of the function, insert a dummy block at the end.
MachineBasicBlock *Bottom = LoopBottom(Loop);
auto Iter = next(MachineFunction::iterator(Bottom));
if (Iter == MF.end()) {
MachineBasicBlock *Label = MF.CreateMachineBasicBlock();
// Give it a fake predecessor so that AsmPrinter prints its label.
Label->addSuccessor(Label);
MF.push_back(Label);
Iter = next(MachineFunction::iterator(Bottom));
}
MachineBasicBlock *AfterLoop = &*Iter;
// Mark the beginning of the loop (after the end of any existing loop that
// ends here).
auto InsertPos = MBB.begin();
while (InsertPos != MBB.end() &&
InsertPos->getOpcode() == WebAssembly::END_LOOP)
++InsertPos;
BuildMI(MBB, InsertPos, DebugLoc(), TII.get(WebAssembly::LOOP));
// Mark the end of the loop.
MachineInstr *End = BuildMI(*AfterLoop, AfterLoop->begin(), DebugLoc(),
TII.get(WebAssembly::END_LOOP));
LoopTops[End] = &MBB;
assert((!ScopeTops[AfterLoop->getNumber()] ||
ScopeTops[AfterLoop->getNumber()]->getNumber() < MBB.getNumber()) &&
"With block sorting the outermost loop for a block should be first.");
if (!ScopeTops[AfterLoop->getNumber()])
ScopeTops[AfterLoop->getNumber()] = &MBB;
}
bool BranchRelaxation::fixupUnconditionalBranch(MachineInstr &MI) {
MachineBasicBlock *MBB = MI.getParent();
unsigned OldBrSize = TII->getInstSizeInBytes(MI);
MachineBasicBlock *DestBB = TII->getBranchDestBlock(MI);
int64_t DestOffset = BlockInfo[DestBB->getNumber()].Offset;
int64_t SrcOffset = getInstrOffset(MI);
assert(!TII->isBranchOffsetInRange(MI.getOpcode(), DestOffset - SrcOffset));
BlockInfo[MBB->getNumber()].Size -= OldBrSize;
MachineBasicBlock *BranchBB = MBB;
// If this was an expanded conditional branch, there is already a single
// unconditional branch in a block.
if (!MBB->empty()) {
BranchBB = createNewBlockAfter(*MBB);
// Add live outs.
for (const MachineBasicBlock *Succ : MBB->successors()) {
for (const MachineBasicBlock::RegisterMaskPair &LiveIn : Succ->liveins())
BranchBB->addLiveIn(LiveIn);
}
BranchBB->sortUniqueLiveIns();
BranchBB->addSuccessor(DestBB);
MBB->replaceSuccessor(DestBB, BranchBB);
}
DebugLoc DL = MI.getDebugLoc();
MI.eraseFromParent();
BlockInfo[BranchBB->getNumber()].Size += TII->insertIndirectBranch(
*BranchBB, *DestBB, DL, DestOffset - SrcOffset, RS.get());
adjustBlockOffsets(*MBB);
return true;
}
/// Do expand branches and split the basic blocks if necessary.
/// Returns true if made any change.
bool MSP430BSel::expandBranches(OffsetVector &BlockOffsets) {
// For each conditional branch, if the offset to its destination is larger
// than the offset field allows, transform it into a long branch sequence
// like this:
// short branch:
// bCC MBB
// long branch:
// b!CC $PC+6
// b MBB
//
bool MadeChange = false;
for (auto MBB = MF->begin(), E = MF->end(); MBB != E; ++MBB) {
unsigned MBBStartOffset = 0;
for (auto MI = MBB->begin(), EE = MBB->end(); MI != EE; ++MI) {
MBBStartOffset += TII->getInstSizeInBytes(*MI);
// If this instruction is not a short branch then skip it.
if (MI->getOpcode() != MSP430::JCC && MI->getOpcode() != MSP430::JMP) {
continue;
}
MachineBasicBlock *DestBB = MI->getOperand(0).getMBB();
// Determine the distance from the current branch to the destination
// block. MBBStartOffset already includes the size of the current branch
// instruction.
int BlockDistance =
BlockOffsets[DestBB->getNumber()] - BlockOffsets[MBB->getNumber()];
int BranchDistance = BlockDistance - MBBStartOffset;
// If this branch is in range, ignore it.
if (isInRage(BranchDistance)) {
continue;
}
DEBUG(dbgs() << " Found a branch that needs expanding, BB#"
<< DestBB->getNumber() << ", Distance " << BranchDistance
<< "\n");
// If JCC is not the last instruction we need to split the MBB.
if (MI->getOpcode() == MSP430::JCC && std::next(MI) != EE) {
DEBUG(dbgs() << " Found a basic block that needs to be split, BB#"
<< MBB->getNumber() << "\n");
// Create a new basic block.
MachineBasicBlock *NewBB =
MF->CreateMachineBasicBlock(MBB->getBasicBlock());
MF->insert(std::next(MBB), NewBB);
// Splice the instructions following MI over to the NewBB.
NewBB->splice(NewBB->end(), &*MBB, std::next(MI), MBB->end());
// Update the successor lists.
for (MachineBasicBlock *Succ : MBB->successors()) {
if (Succ == DestBB) {
continue;
}
MBB->replaceSuccessor(Succ, NewBB);
NewBB->addSuccessor(Succ);
}
// We introduced a new MBB so all following blocks should be numbered
// and measured again.
measureFunction(BlockOffsets, &*MBB);
++NumSplit;
// It may be not necessary to start all over at this point, but it's
// safer do this anyway.
return true;
}
MachineInstr &OldBranch = *MI;
DebugLoc dl = OldBranch.getDebugLoc();
int InstrSizeDiff = -TII->getInstSizeInBytes(OldBranch);
if (MI->getOpcode() == MSP430::JCC) {
MachineBasicBlock *NextMBB = &*std::next(MBB);
assert(MBB->isSuccessor(NextMBB) &&
"This block must have a layout successor!");
// The BCC operands are:
// 0. Target MBB
// 1. MSP430 branch predicate
SmallVector<MachineOperand, 1> Cond;
Cond.push_back(MI->getOperand(1));
// Jump over the long branch on the opposite condition
TII->reverseBranchCondition(Cond);
MI = BuildMI(*MBB, MI, dl, TII->get(MSP430::JCC))
.addMBB(NextMBB)
.add(Cond[0]);
InstrSizeDiff += TII->getInstSizeInBytes(*MI);
++MI;
}
// Unconditional branch to the real destination.
MI = BuildMI(*MBB, MI, dl, TII->get(MSP430::Bi)).addMBB(DestBB);
InstrSizeDiff += TII->getInstSizeInBytes(*MI);
// Remove the old branch from the function.
OldBranch.eraseFromParent();
// The size of a new instruction is different from the old one, so we need
// to correct all block offsets.
for (int i = MBB->getNumber() + 1, e = BlockOffsets.size(); i < e; ++i) {
BlockOffsets[i] += InstrSizeDiff;
}
MBBStartOffset += InstrSizeDiff;
++NumExpanded;
MadeChange = true;
}
}
return MadeChange;
}
/// TailDuplicate - If it is profitable, duplicate TailBB's contents in each
/// of its predecessors.
bool
TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF,
SmallVector<MachineBasicBlock*, 8> &TDBBs,
SmallVector<MachineInstr*, 16> &Copies) {
if (!shouldTailDuplicate(MF, *TailBB))
return false;
DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
// Iterate through all the unique predecessors and tail-duplicate this
// block into them, if possible. Copying the list ahead of time also
// avoids trouble with the predecessor list reallocating.
bool Changed = false;
SmallSetVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
DenseSet<unsigned> UsedByPhi;
getRegsUsedByPHIs(*TailBB, &UsedByPhi);
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
assert(TailBB != PredBB &&
"Single-block loop should have been rejected earlier!");
// EH edges are ignored by AnalyzeBranch.
if (PredBB->succ_size() > 1)
continue;
MachineBasicBlock *PredTBB, *PredFBB;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
continue;
if (!PredCond.empty())
continue;
// Don't duplicate into a fall-through predecessor (at least for now).
if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
continue;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From Succ: " << *TailBB);
TDBBs.push_back(PredBB);
// Remove PredBB's unconditional branch.
TII->RemoveBranch(*PredBB);
// Clone the contents of TailBB into PredBB.
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
while (I != TailBB->end()) {
MachineInstr *MI = &*I;
++I;
if (MI->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, true);
} else {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
DuplicateInstruction(MI, TailBB, PredBB, MF, LocalVRMap, UsedByPhi);
}
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
// Simplify
TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true);
NumInstrDups += TailBB->size() - 1; // subtract one for removed branch
// Update the CFG.
PredBB->removeSuccessor(PredBB->succ_begin());
assert(PredBB->succ_empty() &&
"TailDuplicate called on block with multiple successors!");
for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(),
E = TailBB->succ_end(); I != E; ++I)
PredBB->addSuccessor(*I);
Changed = true;
++NumTailDups;
}
// If TailBB was duplicated into all its predecessors except for the prior
// block, which falls through unconditionally, move the contents of this
// block into the prior block.
MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB));
MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0;
SmallVector<MachineOperand, 4> PriorCond;
// This has to check PrevBB->succ_size() because EH edges are ignored by
// AnalyzeBranch.
if (PrevBB->succ_size() == 1 &&
!TII->AnalyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond, true) &&
PriorCond.empty() && !PriorTBB && TailBB->pred_size() == 1 &&
!TailBB->hasAddressTaken()) {
DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
<< "From MBB: " << *TailBB);
if (PreRegAlloc) {
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos, UsedByPhi, true);
if (MI->getParent())
MI->eraseFromParent();
}
// Now copy the non-PHI instructions.
while (I != TailBB->end()) {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
MachineInstr *MI = &*I++;
DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap, UsedByPhi);
MI->eraseFromParent();
}
MachineBasicBlock::iterator Loc = PrevBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PrevBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first)
.addReg(CopyInfos[i].second));
}
} else {
// No PHIs to worry about, just splice the instructions over.
PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
}
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);
TDBBs.push_back(PrevBB);
Changed = true;
}
// If this is after register allocation, there are no phis to fix.
if (!PreRegAlloc)
return Changed;
// If we made no changes so far, we are safe.
if (!Changed)
return Changed;
// Handle the nasty case in that we duplicated a block that is part of a loop
// into some but not all of its predecessors. For example:
// 1 -> 2 <-> 3 |
// \ |
// \---> rest |
// if we duplicate 2 into 1 but not into 3, we end up with
// 12 -> 3 <-> 2 -> rest |
// \ / |
// \----->-----/ |
// If there was a "var = phi(1, 3)" in 2, it has to be ultimately replaced
// with a phi in 3 (which now dominates 2).
// What we do here is introduce a copy in 3 of the register defined by the
// phi, just like when we are duplicating 2 into 3, but we don't copy any
// real instructions or remove the 3 -> 2 edge from the phi in 2.
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (std::find(TDBBs.begin(), TDBBs.end(), PredBB) != TDBBs.end())
continue;
// EH edges
if (PredBB->succ_size() != 1)
continue;
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos, UsedByPhi, false);
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
}
return Changed;
}
/// fixupConditionalBranch - Fix up a conditional branch whose destination is
/// too far away to fit in its displacement field. It is converted to an inverse
/// conditional branch + an unconditional branch to the destination.
bool BranchRelaxation::fixupConditionalBranch(MachineInstr &MI) {
DebugLoc DL = MI.getDebugLoc();
MachineBasicBlock *MBB = MI.getParent();
MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
MachineBasicBlock *NewBB = nullptr;
SmallVector<MachineOperand, 4> Cond;
auto insertUncondBranch = [&](MachineBasicBlock *MBB,
MachineBasicBlock *DestBB) {
unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
int NewBrSize = 0;
TII->insertUnconditionalBranch(*MBB, DestBB, DL, &NewBrSize);
BBSize += NewBrSize;
};
auto insertBranch = [&](MachineBasicBlock *MBB, MachineBasicBlock *TBB,
MachineBasicBlock *FBB,
SmallVectorImpl<MachineOperand>& Cond) {
unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
int NewBrSize = 0;
TII->insertBranch(*MBB, TBB, FBB, Cond, DL, &NewBrSize);
BBSize += NewBrSize;
};
auto removeBranch = [&](MachineBasicBlock *MBB) {
unsigned &BBSize = BlockInfo[MBB->getNumber()].Size;
int RemovedSize = 0;
TII->removeBranch(*MBB, &RemovedSize);
BBSize -= RemovedSize;
};
auto finalizeBlockChanges = [&](MachineBasicBlock *MBB,
MachineBasicBlock *NewBB) {
// Keep the block offsets up to date.
adjustBlockOffsets(*MBB);
// Need to fix live-in lists if we track liveness.
if (NewBB && TRI->trackLivenessAfterRegAlloc(*MF))
computeAndAddLiveIns(LiveRegs, *NewBB);
};
bool Fail = TII->analyzeBranch(*MBB, TBB, FBB, Cond);
assert(!Fail && "branches to be relaxed must be analyzable");
(void)Fail;
// Add an unconditional branch to the destination and invert the branch
// condition to jump over it:
// tbz L1
// =>
// tbnz L2
// b L1
// L2:
bool ReversedCond = !TII->reverseBranchCondition(Cond);
if (ReversedCond) {
if (FBB && isBlockInRange(MI, *FBB)) {
// Last MI in the BB is an unconditional branch. We can simply invert the
// condition and swap destinations:
// beq L1
// b L2
// =>
// bne L2
// b L1
LLVM_DEBUG(dbgs() << " Invert condition and swap "
"its destination with "
<< MBB->back());
removeBranch(MBB);
insertBranch(MBB, FBB, TBB, Cond);
finalizeBlockChanges(MBB, nullptr);
return true;
}
if (FBB) {
// We need to split the basic block here to obtain two long-range
// unconditional branches.
NewBB = createNewBlockAfter(*MBB);
insertUncondBranch(NewBB, FBB);
// Update the succesor lists according to the transformation to follow.
// Do it here since if there's no split, no update is needed.
MBB->replaceSuccessor(FBB, NewBB);
NewBB->addSuccessor(FBB);
}
// We now have an appropriate fall-through block in place (either naturally or
// just created), so we can use the inverted the condition.
MachineBasicBlock &NextBB = *std::next(MachineFunction::iterator(MBB));
LLVM_DEBUG(dbgs() << " Insert B to " << printMBBReference(*TBB)
<< ", invert condition and change dest. to "
<< printMBBReference(NextBB) << '\n');
removeBranch(MBB);
// Insert a new conditional branch and a new unconditional branch.
insertBranch(MBB, &NextBB, TBB, Cond);
finalizeBlockChanges(MBB, NewBB);
return true;
}
// Branch cond can't be inverted.
// In this case we always add a block after the MBB.
LLVM_DEBUG(dbgs() << " The branch condition can't be inverted. "
<< " Insert a new BB after " << MBB->back());
if (!FBB)
FBB = &(*std::next(MachineFunction::iterator(MBB)));
// This is the block with cond. branch and the distance to TBB is too long.
// beq L1
// L2:
// We do the following transformation:
// beq NewBB
// b L2
// NewBB:
// b L1
// L2:
NewBB = createNewBlockAfter(*MBB);
insertUncondBranch(NewBB, TBB);
LLVM_DEBUG(dbgs() << " Insert cond B to the new BB "
<< printMBBReference(*NewBB)
<< " Keep the exiting condition.\n"
<< " Insert B to " << printMBBReference(*FBB) << ".\n"
<< " In the new BB: Insert B to "
<< printMBBReference(*TBB) << ".\n");
// Update the successor lists according to the transformation to follow.
MBB->replaceSuccessor(TBB, NewBB);
NewBB->addSuccessor(TBB);
// Replace branch in the current (MBB) block.
removeBranch(MBB);
insertBranch(MBB, NewBB, FBB, Cond);
finalizeBlockChanges(MBB, NewBB);
return true;
}
MachineBasicBlock *
MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) {
// Splitting the critical edge to a landing pad block is non-trivial. Don't do
// it in this generic function.
if (Succ->isLandingPad())
return nullptr;
MachineFunction *MF = getParent();
DebugLoc dl; // FIXME: this is nowhere
// Performance might be harmed on HW that implements branching using exec mask
// where both sides of the branches are always executed.
if (MF->getTarget().requiresStructuredCFG())
return nullptr;
// We may need to update this's terminator, but we can't do that if
// AnalyzeBranch fails. If this uses a jump table, we won't touch it.
const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo();
MachineBasicBlock *TBB = nullptr, *FBB = nullptr;
SmallVector<MachineOperand, 4> Cond;
if (TII->AnalyzeBranch(*this, TBB, FBB, Cond))
return nullptr;
// Avoid bugpoint weirdness: A block may end with a conditional branch but
// jumps to the same MBB is either case. We have duplicate CFG edges in that
// case that we can't handle. Since this never happens in properly optimized
// code, just skip those edges.
if (TBB && TBB == FBB) {
DEBUG(dbgs() << "Won't split critical edge after degenerate BB#"
<< getNumber() << '\n');
return nullptr;
}
MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock();
MF->insert(std::next(MachineFunction::iterator(this)), NMBB);
DEBUG(dbgs() << "Splitting critical edge:"
" BB#" << getNumber()
<< " -- BB#" << NMBB->getNumber()
<< " -- BB#" << Succ->getNumber() << '\n');
LiveIntervals *LIS = P->getAnalysisIfAvailable<LiveIntervals>();
SlotIndexes *Indexes = P->getAnalysisIfAvailable<SlotIndexes>();
if (LIS)
LIS->insertMBBInMaps(NMBB);
else if (Indexes)
Indexes->insertMBBInMaps(NMBB);
// On some targets like Mips, branches may kill virtual registers. Make sure
// that LiveVariables is properly updated after updateTerminator replaces the
// terminators.
LiveVariables *LV = P->getAnalysisIfAvailable<LiveVariables>();
// Collect a list of virtual registers killed by the terminators.
SmallVector<unsigned, 4> KilledRegs;
if (LV)
for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
I != E; ++I) {
MachineInstr *MI = I;
for (MachineInstr::mop_iterator OI = MI->operands_begin(),
OE = MI->operands_end(); OI != OE; ++OI) {
if (!OI->isReg() || OI->getReg() == 0 ||
!OI->isUse() || !OI->isKill() || OI->isUndef())
continue;
unsigned Reg = OI->getReg();
if (TargetRegisterInfo::isPhysicalRegister(Reg) ||
LV->getVarInfo(Reg).removeKill(MI)) {
KilledRegs.push_back(Reg);
DEBUG(dbgs() << "Removing terminator kill: " << *MI);
OI->setIsKill(false);
}
}
}
SmallVector<unsigned, 4> UsedRegs;
if (LIS) {
for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
I != E; ++I) {
MachineInstr *MI = I;
for (MachineInstr::mop_iterator OI = MI->operands_begin(),
OE = MI->operands_end(); OI != OE; ++OI) {
if (!OI->isReg() || OI->getReg() == 0)
continue;
unsigned Reg = OI->getReg();
if (std::find(UsedRegs.begin(), UsedRegs.end(), Reg) == UsedRegs.end())
UsedRegs.push_back(Reg);
}
}
}
ReplaceUsesOfBlockWith(Succ, NMBB);
// If updateTerminator() removes instructions, we need to remove them from
// SlotIndexes.
SmallVector<MachineInstr*, 4> Terminators;
if (Indexes) {
for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
I != E; ++I)
Terminators.push_back(I);
}
updateTerminator();
if (Indexes) {
SmallVector<MachineInstr*, 4> NewTerminators;
for (instr_iterator I = getFirstInstrTerminator(), E = instr_end();
I != E; ++I)
NewTerminators.push_back(I);
for (SmallVectorImpl<MachineInstr*>::iterator I = Terminators.begin(),
E = Terminators.end(); I != E; ++I) {
if (std::find(NewTerminators.begin(), NewTerminators.end(), *I) ==
NewTerminators.end())
Indexes->removeMachineInstrFromMaps(*I);
}
}
// Insert unconditional "jump Succ" instruction in NMBB if necessary.
NMBB->addSuccessor(Succ);
if (!NMBB->isLayoutSuccessor(Succ)) {
Cond.clear();
MF->getSubtarget().getInstrInfo()->InsertBranch(*NMBB, Succ, nullptr, Cond,
dl);
if (Indexes) {
for (instr_iterator I = NMBB->instr_begin(), E = NMBB->instr_end();
I != E; ++I) {
// Some instructions may have been moved to NMBB by updateTerminator(),
// so we first remove any instruction that already has an index.
if (Indexes->hasIndex(I))
Indexes->removeMachineInstrFromMaps(I);
Indexes->insertMachineInstrInMaps(I);
}
}
}
// Fix PHI nodes in Succ so they refer to NMBB instead of this
for (MachineBasicBlock::instr_iterator
i = Succ->instr_begin(),e = Succ->instr_end();
i != e && i->isPHI(); ++i)
for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2)
if (i->getOperand(ni+1).getMBB() == this)
i->getOperand(ni+1).setMBB(NMBB);
// Inherit live-ins from the successor
for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(),
E = Succ->livein_end(); I != E; ++I)
NMBB->addLiveIn(*I);
// Update LiveVariables.
const TargetRegisterInfo *TRI = MF->getSubtarget().getRegisterInfo();
if (LV) {
// Restore kills of virtual registers that were killed by the terminators.
while (!KilledRegs.empty()) {
unsigned Reg = KilledRegs.pop_back_val();
for (instr_iterator I = instr_end(), E = instr_begin(); I != E;) {
if (!(--I)->addRegisterKilled(Reg, TRI, /* addIfNotFound= */ false))
continue;
if (TargetRegisterInfo::isVirtualRegister(Reg))
LV->getVarInfo(Reg).Kills.push_back(I);
DEBUG(dbgs() << "Restored terminator kill: " << *I);
break;
}
}
// Update relevant live-through information.
LV->addNewBlock(NMBB, this, Succ);
}
if (LIS) {
// After splitting the edge and updating SlotIndexes, live intervals may be
// in one of two situations, depending on whether this block was the last in
// the function. If the original block was the last in the function, all live
// intervals will end prior to the beginning of the new split block. If the
// original block was not at the end of the function, all live intervals will
// extend to the end of the new split block.
bool isLastMBB =
std::next(MachineFunction::iterator(NMBB)) == getParent()->end();
SlotIndex StartIndex = Indexes->getMBBEndIdx(this);
SlotIndex PrevIndex = StartIndex.getPrevSlot();
SlotIndex EndIndex = Indexes->getMBBEndIdx(NMBB);
// Find the registers used from NMBB in PHIs in Succ.
SmallSet<unsigned, 8> PHISrcRegs;
for (MachineBasicBlock::instr_iterator
I = Succ->instr_begin(), E = Succ->instr_end();
I != E && I->isPHI(); ++I) {
for (unsigned ni = 1, ne = I->getNumOperands(); ni != ne; ni += 2) {
if (I->getOperand(ni+1).getMBB() == NMBB) {
MachineOperand &MO = I->getOperand(ni);
unsigned Reg = MO.getReg();
PHISrcRegs.insert(Reg);
if (MO.isUndef())
continue;
LiveInterval &LI = LIS->getInterval(Reg);
VNInfo *VNI = LI.getVNInfoAt(PrevIndex);
assert(VNI && "PHI sources should be live out of their predecessors.");
LI.addSegment(LiveInterval::Segment(StartIndex, EndIndex, VNI));
}
}
}
MachineRegisterInfo *MRI = &getParent()->getRegInfo();
for (unsigned i = 0, e = MRI->getNumVirtRegs(); i != e; ++i) {
unsigned Reg = TargetRegisterInfo::index2VirtReg(i);
if (PHISrcRegs.count(Reg) || !LIS->hasInterval(Reg))
continue;
LiveInterval &LI = LIS->getInterval(Reg);
if (!LI.liveAt(PrevIndex))
continue;
bool isLiveOut = LI.liveAt(LIS->getMBBStartIdx(Succ));
if (isLiveOut && isLastMBB) {
VNInfo *VNI = LI.getVNInfoAt(PrevIndex);
assert(VNI && "LiveInterval should have VNInfo where it is live.");
LI.addSegment(LiveInterval::Segment(StartIndex, EndIndex, VNI));
} else if (!isLiveOut && !isLastMBB) {
LI.removeSegment(StartIndex, EndIndex);
}
}
// Update all intervals for registers whose uses may have been modified by
// updateTerminator().
LIS->repairIntervalsInRange(this, getFirstTerminator(), end(), UsedRegs);
}
if (MachineDominatorTree *MDT =
P->getAnalysisIfAvailable<MachineDominatorTree>())
MDT->recordSplitCriticalEdge(this, Succ, NMBB);
if (MachineLoopInfo *MLI = P->getAnalysisIfAvailable<MachineLoopInfo>())
if (MachineLoop *TIL = MLI->getLoopFor(this)) {
// If one or the other blocks were not in a loop, the new block is not
// either, and thus LI doesn't need to be updated.
if (MachineLoop *DestLoop = MLI->getLoopFor(Succ)) {
if (TIL == DestLoop) {
// Both in the same loop, the NMBB joins loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (TIL->contains(DestLoop)) {
// Edge from an outer loop to an inner loop. Add to the outer loop.
TIL->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (DestLoop->contains(TIL)) {
// Edge from an inner loop to an outer loop. Add to the outer loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else {
// Edge from two loops with no containment relation. Because these
// are natural loops, we know that the destination block must be the
// header of its loop (adding a branch into a loop elsewhere would
// create an irreducible loop).
assert(DestLoop->getHeader() == Succ &&
"Should not create irreducible loops!");
if (MachineLoop *P = DestLoop->getParentLoop())
P->addBasicBlockToLoop(NMBB, MLI->getBase());
}
}
}
return NMBB;
}
void MipsFrameLowering::emitPrologue(MachineFunction &MF) const {
MachineBasicBlock &MBB = MF.front();
MachineFrameInfo *MFI = MF.getFrameInfo();
MipsFunctionInfo *MipsFI = MF.getInfo<MipsFunctionInfo>();
const MipsRegisterInfo *RegInfo =
static_cast<const MipsRegisterInfo*>(MF.getTarget().getRegisterInfo());
const MipsInstrInfo &TII =
*static_cast<const MipsInstrInfo*>(MF.getTarget().getInstrInfo());
MachineBasicBlock::iterator MBBI = MBB.begin();
DebugLoc dl = MBBI != MBB.end() ? MBBI->getDebugLoc() : DebugLoc();
bool isPIC = (MF.getTarget().getRelocationModel() == Reloc::PIC_);
unsigned SP = STI.isABI_N64() ? Mips::SP_64 : Mips::SP;
unsigned FP = STI.isABI_N64() ? Mips::FP_64 : Mips::FP;
unsigned ZERO = STI.isABI_N64() ? Mips::ZERO_64 : Mips::ZERO;
unsigned ADDu = STI.isABI_N64() ? Mips::DADDu : Mips::ADDu;
unsigned ADDiu = STI.isABI_N64() ? Mips::DADDiu : Mips::ADDiu;
// First, compute final stack size.
unsigned RegSize = STI.isGP32bit() ? 4 : 8;
unsigned StackAlign = getStackAlignment();
unsigned LocalVarAreaOffset = MipsFI->needGPSaveRestore() ?
(MFI->getObjectOffset(MipsFI->getGPFI()) + RegSize) :
MipsFI->getMaxCallFrameSize();
uint64_t StackSize = RoundUpToAlignment(LocalVarAreaOffset, StackAlign) +
RoundUpToAlignment(MFI->getStackSize(), StackAlign);
// Update stack size
MFI->setStackSize(StackSize);
// Emit instructions that set the global base register if the target ABI is
// O32.
if (isPIC && MipsFI->globalBaseRegSet() && STI.isABI_O32() &&
!MipsFI->globalBaseRegFixed()) {
// See MipsInstrInfo.td for explanation.
MachineBasicBlock *NewEntry = MF.CreateMachineBasicBlock();
MF.insert(&MBB, NewEntry);
NewEntry->addSuccessor(&MBB);
// Copy live in registers.
for (MachineBasicBlock::livein_iterator R = MBB.livein_begin();
R != MBB.livein_end(); ++R)
NewEntry->addLiveIn(*R);
BuildMI(*NewEntry, NewEntry->begin(), dl, TII.get(Mips:: SETGP01),
Mips::V0);
}
// No need to allocate space on the stack.
if (StackSize == 0 && !MFI->adjustsStack()) return;
MachineModuleInfo &MMI = MF.getMMI();
std::vector<MachineMove> &Moves = MMI.getFrameMoves();
MachineLocation DstML, SrcML;
// Adjust stack.
if (isInt<16>(-StackSize)) // addi sp, sp, (-stacksize)
BuildMI(MBB, MBBI, dl, TII.get(ADDiu), SP).addReg(SP).addImm(-StackSize);
else { // Expand immediate that doesn't fit in 16-bit.
MipsFI->setEmitNOAT();
expandLargeImm(SP, -StackSize, STI.isABI_N64(), TII, MBB, MBBI, dl);
}
// emit ".cfi_def_cfa_offset StackSize"
MCSymbol *AdjustSPLabel = MMI.getContext().CreateTempSymbol();
BuildMI(MBB, MBBI, dl,
TII.get(TargetOpcode::PROLOG_LABEL)).addSym(AdjustSPLabel);
DstML = MachineLocation(MachineLocation::VirtualFP);
SrcML = MachineLocation(MachineLocation::VirtualFP, -StackSize);
Moves.push_back(MachineMove(AdjustSPLabel, DstML, SrcML));
const std::vector<CalleeSavedInfo> &CSI = MFI->getCalleeSavedInfo();
if (CSI.size()) {
// Find the instruction past the last instruction that saves a callee-saved
// register to the stack.
for (unsigned i = 0; i < CSI.size(); ++i)
++MBBI;
// Iterate over list of callee-saved registers and emit .cfi_offset
// directives.
MCSymbol *CSLabel = MMI.getContext().CreateTempSymbol();
BuildMI(MBB, MBBI, dl,
TII.get(TargetOpcode::PROLOG_LABEL)).addSym(CSLabel);
for (std::vector<CalleeSavedInfo>::const_iterator I = CSI.begin(),
E = CSI.end(); I != E; ++I) {
int64_t Offset = MFI->getObjectOffset(I->getFrameIdx());
unsigned Reg = I->getReg();
// If Reg is a double precision register, emit two cfa_offsets,
// one for each of the paired single precision registers.
if (Mips::AFGR64RegisterClass->contains(Reg)) {
const uint16_t *SubRegs = RegInfo->getSubRegisters(Reg);
MachineLocation DstML0(MachineLocation::VirtualFP, Offset);
MachineLocation DstML1(MachineLocation::VirtualFP, Offset + 4);
MachineLocation SrcML0(*SubRegs);
MachineLocation SrcML1(*(SubRegs + 1));
if (!STI.isLittle())
std::swap(SrcML0, SrcML1);
Moves.push_back(MachineMove(CSLabel, DstML0, SrcML0));
Moves.push_back(MachineMove(CSLabel, DstML1, SrcML1));
}
else {
// Reg is either in CPURegs or FGR32.
DstML = MachineLocation(MachineLocation::VirtualFP, Offset);
SrcML = MachineLocation(Reg);
Moves.push_back(MachineMove(CSLabel, DstML, SrcML));
}
}
}
// if framepointer enabled, set it to point to the stack pointer.
if (hasFP(MF)) {
// Insert instruction "move $fp, $sp" at this location.
BuildMI(MBB, MBBI, dl, TII.get(ADDu), FP).addReg(SP).addReg(ZERO);
// emit ".cfi_def_cfa_register $fp"
MCSymbol *SetFPLabel = MMI.getContext().CreateTempSymbol();
BuildMI(MBB, MBBI, dl,
TII.get(TargetOpcode::PROLOG_LABEL)).addSym(SetFPLabel);
DstML = MachineLocation(FP);
SrcML = MachineLocation(MachineLocation::VirtualFP);
Moves.push_back(MachineMove(SetFPLabel, DstML, SrcML));
}
// Restore GP from the saved stack location
if (MipsFI->needGPSaveRestore()) {
unsigned Offset = MFI->getObjectOffset(MipsFI->getGPFI());
BuildMI(MBB, MBBI, dl, TII.get(Mips::CPRESTORE)).addImm(Offset)
.addReg(Mips::GP);
}
}
/// Splits a MachineBasicBlock to branch before \p SplitBefore. The original
/// branch is \p OrigBranch. The target of the new branch can either be the same
/// as the target of the original branch or the fallthrough successor of the
/// original block as determined by \p BranchToFallThrough. The branch
/// conditions will be inverted according to \p InvertNewBranch and
/// \p InvertOrigBranch. If an instruction that previously fed the branch is to
/// be deleted, it is provided in \p MIToDelete and \p NewCond will be used as
/// the branch condition. The branch probabilities will be set if the
/// MachineBranchProbabilityInfo isn't null.
static bool splitMBB(BlockSplitInfo &BSI) {
assert(BSI.allInstrsInSameMBB() &&
"All instructions must be in the same block.");
MachineBasicBlock *ThisMBB = BSI.OrigBranch->getParent();
MachineFunction *MF = ThisMBB->getParent();
MachineRegisterInfo *MRI = &MF->getRegInfo();
assert(MRI->isSSA() && "Can only do this while the function is in SSA form.");
if (ThisMBB->succ_size() != 2) {
LLVM_DEBUG(
dbgs() << "Don't know how to handle blocks that don't have exactly"
<< " two successors.\n");
return false;
}
const PPCInstrInfo *TII = MF->getSubtarget<PPCSubtarget>().getInstrInfo();
unsigned OrigBROpcode = BSI.OrigBranch->getOpcode();
unsigned InvertedOpcode =
OrigBROpcode == PPC::BC
? PPC::BCn
: OrigBROpcode == PPC::BCn
? PPC::BC
: OrigBROpcode == PPC::BCLR ? PPC::BCLRn : PPC::BCLR;
unsigned NewBROpcode = BSI.InvertNewBranch ? InvertedOpcode : OrigBROpcode;
MachineBasicBlock *OrigTarget = BSI.OrigBranch->getOperand(1).getMBB();
MachineBasicBlock *OrigFallThrough = OrigTarget == *ThisMBB->succ_begin()
? *ThisMBB->succ_rbegin()
: *ThisMBB->succ_begin();
MachineBasicBlock *NewBRTarget =
BSI.BranchToFallThrough ? OrigFallThrough : OrigTarget;
BranchProbability ProbToNewTarget =
!BSI.MBPI ? BranchProbability::getUnknown()
: BSI.MBPI->getEdgeProbability(ThisMBB, NewBRTarget);
// Create a new basic block.
MachineBasicBlock::iterator InsertPoint = BSI.SplitBefore;
const BasicBlock *LLVM_BB = ThisMBB->getBasicBlock();
MachineFunction::iterator It = ThisMBB->getIterator();
MachineBasicBlock *NewMBB = MF->CreateMachineBasicBlock(LLVM_BB);
MF->insert(++It, NewMBB);
// Move everything after SplitBefore into the new block.
NewMBB->splice(NewMBB->end(), ThisMBB, InsertPoint, ThisMBB->end());
NewMBB->transferSuccessors(ThisMBB);
// Add the two successors to ThisMBB. The probabilities come from the
// existing blocks if available.
ThisMBB->addSuccessor(NewBRTarget, ProbToNewTarget);
ThisMBB->addSuccessor(NewMBB, ProbToNewTarget.getCompl());
// Add the branches to ThisMBB.
BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(),
TII->get(NewBROpcode))
.addReg(BSI.SplitCond->getOperand(0).getReg())
.addMBB(NewBRTarget);
BuildMI(*ThisMBB, ThisMBB->end(), BSI.SplitBefore->getDebugLoc(),
TII->get(PPC::B))
.addMBB(NewMBB);
if (BSI.MIToDelete)
BSI.MIToDelete->eraseFromParent();
// Change the condition on the original branch and invert it if requested.
auto FirstTerminator = NewMBB->getFirstTerminator();
if (BSI.NewCond) {
assert(FirstTerminator->getOperand(0).isReg() &&
"Can't update condition of unconditional branch.");
FirstTerminator->getOperand(0).setReg(BSI.NewCond->getOperand(0).getReg());
}
if (BSI.InvertOrigBranch)
FirstTerminator->setDesc(TII->get(InvertedOpcode));
// If any of the PHIs in the successors of NewMBB reference values that
// now come from NewMBB, they need to be updated.
for (auto *Succ : NewMBB->successors()) {
updatePHIs(Succ, ThisMBB, NewMBB, MRI);
}
addIncomingValuesToPHIs(NewBRTarget, ThisMBB, NewMBB, MRI);
LLVM_DEBUG(dbgs() << "After splitting, ThisMBB:\n"; ThisMBB->dump());
LLVM_DEBUG(dbgs() << "NewMBB:\n"; NewMBB->dump());
LLVM_DEBUG(dbgs() << "New branch-to block:\n"; NewBRTarget->dump());
return true;
}
// Returns true if a new block was inserted.
bool SILowerControlFlow::loadM0(MachineInstr &MI, MachineInstr *MovRel, int Offset) {
MachineBasicBlock &MBB = *MI.getParent();
DebugLoc DL = MI.getDebugLoc();
MachineBasicBlock::iterator I(&MI);
const MachineOperand *Idx = TII->getNamedOperand(MI, AMDGPU::OpName::idx);
if (AMDGPU::SReg_32RegClass.contains(Idx->getReg())) {
if (Offset) {
BuildMI(MBB, I, DL, TII->get(AMDGPU::S_ADD_I32), AMDGPU::M0)
.addReg(Idx->getReg(), getUndefRegState(Idx->isUndef()))
.addImm(Offset);
} else {
BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B32), AMDGPU::M0)
.addReg(Idx->getReg(), getUndefRegState(Idx->isUndef()));
}
MBB.insert(I, MovRel);
MI.eraseFromParent();
return false;
}
MachineFunction &MF = *MBB.getParent();
MachineOperand *SaveOp = TII->getNamedOperand(MI, AMDGPU::OpName::sdst);
SaveOp->setIsDead(false);
unsigned Save = SaveOp->getReg();
// Reading from a VGPR requires looping over all workitems in the wavefront.
assert(AMDGPU::SReg_64RegClass.contains(Save) &&
AMDGPU::VGPR_32RegClass.contains(Idx->getReg()));
// Save the EXEC mask
BuildMI(MBB, I, DL, TII->get(AMDGPU::S_MOV_B64), Save)
.addReg(AMDGPU::EXEC);
// To insert the loop we need to split the block. Move everything after this
// point to a new block, and insert a new empty block between the two.
MachineBasicBlock *LoopBB = MF.CreateMachineBasicBlock();
MachineBasicBlock *RemainderBB = MF.CreateMachineBasicBlock();
MachineFunction::iterator MBBI(MBB);
++MBBI;
MF.insert(MBBI, LoopBB);
MF.insert(MBBI, RemainderBB);
LoopBB->addSuccessor(LoopBB);
LoopBB->addSuccessor(RemainderBB);
splitBlockLiveIns(MBB, MI, *LoopBB, *RemainderBB, Save, *Idx);
// Move the rest of the block into a new block.
RemainderBB->transferSuccessors(&MBB);
RemainderBB->splice(RemainderBB->begin(), &MBB, I, MBB.end());
emitLoadM0FromVGPRLoop(*LoopBB, DL, MovRel, *Idx, Offset);
MachineBasicBlock::iterator First = RemainderBB->begin();
BuildMI(*RemainderBB, First, DL, TII->get(AMDGPU::S_MOV_B64), AMDGPU::EXEC)
.addReg(Save);
MI.eraseFromParent();
return true;
}
bool
TailDuplicatePass::duplicateSimpleBB(MachineBasicBlock *TailBB,
SmallVectorImpl<MachineBasicBlock *> &TDBBs,
const DenseSet<unsigned> &UsedByPhi,
SmallVectorImpl<MachineInstr *> &Copies) {
SmallPtrSet<MachineBasicBlock*, 8> Succs(TailBB->succ_begin(),
TailBB->succ_end());
SmallVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
bool Changed = false;
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (PredBB->getLandingPadSuccessor())
continue;
if (bothUsedInPHI(*PredBB, Succs))
continue;
MachineBasicBlock *PredTBB = nullptr, *PredFBB = nullptr;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
continue;
Changed = true;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From simple Succ: " << *TailBB);
MachineBasicBlock *NewTarget = *TailBB->succ_begin();
MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(PredBB));
// Make PredFBB explicit.
if (PredCond.empty())
PredFBB = PredTBB;
// Make fall through explicit.
if (!PredTBB)
PredTBB = NextBB;
if (!PredFBB)
PredFBB = NextBB;
// Redirect
if (PredFBB == TailBB)
PredFBB = NewTarget;
if (PredTBB == TailBB)
PredTBB = NewTarget;
// Make the branch unconditional if possible
if (PredTBB == PredFBB) {
PredCond.clear();
PredFBB = nullptr;
}
// Avoid adding fall through branches.
if (PredFBB == NextBB)
PredFBB = nullptr;
if (PredTBB == NextBB && PredFBB == nullptr)
PredTBB = nullptr;
TII->RemoveBranch(*PredBB);
if (PredTBB)
TII->InsertBranch(*PredBB, PredTBB, PredFBB, PredCond, DebugLoc());
uint32_t Weight = MBPI->getEdgeWeight(PredBB, TailBB);
PredBB->removeSuccessor(TailBB);
unsigned NumSuccessors = PredBB->succ_size();
assert(NumSuccessors <= 1);
if (NumSuccessors == 0 || *PredBB->succ_begin() != NewTarget)
PredBB->addSuccessor(NewTarget, Weight);
TDBBs.push_back(PredBB);
}
return Changed;
}
/// TailDuplicate - If it is profitable, duplicate TailBB's contents in each
/// of its predecessors.
bool
TailDuplicatePass::TailDuplicate(MachineBasicBlock *TailBB, MachineFunction &MF,
SmallVector<MachineBasicBlock*, 8> &TDBBs,
SmallVector<MachineInstr*, 16> &Copies) {
// Set the limit on the number of instructions to duplicate, with a default
// of one less than the tail-merge threshold. When optimizing for size,
// duplicate only one, because one branch instruction can be eliminated to
// compensate for the duplication.
unsigned MaxDuplicateCount;
if (TailDuplicateSize.getNumOccurrences() == 0 &&
MF.getFunction()->hasFnAttr(Attribute::OptimizeForSize))
MaxDuplicateCount = 1;
else
MaxDuplicateCount = TailDuplicateSize;
if (PreRegAlloc) {
if (TailBB->empty())
return false;
const TargetInstrDesc &TID = TailBB->back().getDesc();
// Pre-regalloc tail duplication hurts compile time and doesn't help
// much except for indirect branches and returns.
if (!TID.isIndirectBranch() && !TID.isReturn())
return false;
// If the target has hardware branch prediction that can handle indirect
// branches, duplicating them can often make them predictable when there
// are common paths through the code. The limit needs to be high enough
// to allow undoing the effects of tail merging and other optimizations
// that rearrange the predecessors of the indirect branch.
MaxDuplicateCount = 20;
}
// Don't try to tail-duplicate single-block loops.
if (TailBB->isSuccessor(TailBB))
return false;
// Check the instructions in the block to determine whether tail-duplication
// is invalid or unlikely to be profitable.
unsigned InstrCount = 0;
bool HasCall = false;
for (MachineBasicBlock::iterator I = TailBB->begin();
I != TailBB->end(); ++I) {
// Non-duplicable things shouldn't be tail-duplicated.
if (I->getDesc().isNotDuplicable()) return false;
// Do not duplicate 'return' instructions if this is a pre-regalloc run.
// A return may expand into a lot more instructions (e.g. reload of callee
// saved registers) after PEI.
if (PreRegAlloc && I->getDesc().isReturn()) return false;
// Don't duplicate more than the threshold.
if (InstrCount == MaxDuplicateCount) return false;
// Remember if we saw a call.
if (I->getDesc().isCall()) HasCall = true;
if (!I->isPHI() && !I->isDebugValue())
InstrCount += 1;
}
// Don't tail-duplicate calls before register allocation. Calls presents a
// barrier to register allocation so duplicating them may end up increasing
// spills.
if (InstrCount > 1 && (PreRegAlloc && HasCall))
return false;
DEBUG(dbgs() << "\n*** Tail-duplicating BB#" << TailBB->getNumber() << '\n');
// Iterate through all the unique predecessors and tail-duplicate this
// block into them, if possible. Copying the list ahead of time also
// avoids trouble with the predecessor list reallocating.
bool Changed = false;
SmallSetVector<MachineBasicBlock*, 8> Preds(TailBB->pred_begin(),
TailBB->pred_end());
for (SmallSetVector<MachineBasicBlock *, 8>::iterator PI = Preds.begin(),
PE = Preds.end(); PI != PE; ++PI) {
MachineBasicBlock *PredBB = *PI;
assert(TailBB != PredBB &&
"Single-block loop should have been rejected earlier!");
if (PredBB->succ_size() > 1) continue;
MachineBasicBlock *PredTBB, *PredFBB;
SmallVector<MachineOperand, 4> PredCond;
if (TII->AnalyzeBranch(*PredBB, PredTBB, PredFBB, PredCond, true))
continue;
if (!PredCond.empty())
continue;
// EH edges are ignored by AnalyzeBranch.
if (PredBB->succ_size() != 1)
continue;
// Don't duplicate into a fall-through predecessor (at least for now).
if (PredBB->isLayoutSuccessor(TailBB) && PredBB->canFallThrough())
continue;
DEBUG(dbgs() << "\nTail-duplicating into PredBB: " << *PredBB
<< "From Succ: " << *TailBB);
TDBBs.push_back(PredBB);
// Remove PredBB's unconditional branch.
TII->RemoveBranch(*PredBB);
// Clone the contents of TailBB into PredBB.
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
while (I != TailBB->end()) {
MachineInstr *MI = &*I;
++I;
if (MI->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
ProcessPHI(MI, TailBB, PredBB, LocalVRMap, CopyInfos);
} else {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
DuplicateInstruction(MI, TailBB, PredBB, MF, LocalVRMap);
}
}
MachineBasicBlock::iterator Loc = PredBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PredBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first).addReg(CopyInfos[i].second));
}
NumInstrDups += TailBB->size() - 1; // subtract one for removed branch
// Update the CFG.
PredBB->removeSuccessor(PredBB->succ_begin());
assert(PredBB->succ_empty() &&
"TailDuplicate called on block with multiple successors!");
for (MachineBasicBlock::succ_iterator I = TailBB->succ_begin(),
E = TailBB->succ_end(); I != E; ++I)
PredBB->addSuccessor(*I);
Changed = true;
++NumTailDups;
}
// If TailBB was duplicated into all its predecessors except for the prior
// block, which falls through unconditionally, move the contents of this
// block into the prior block.
MachineBasicBlock *PrevBB = prior(MachineFunction::iterator(TailBB));
MachineBasicBlock *PriorTBB = 0, *PriorFBB = 0;
SmallVector<MachineOperand, 4> PriorCond;
bool PriorUnAnalyzable =
TII->AnalyzeBranch(*PrevBB, PriorTBB, PriorFBB, PriorCond, true);
// This has to check PrevBB->succ_size() because EH edges are ignored by
// AnalyzeBranch.
if (!PriorUnAnalyzable && PriorCond.empty() && !PriorTBB &&
TailBB->pred_size() == 1 && PrevBB->succ_size() == 1 &&
!TailBB->hasAddressTaken()) {
DEBUG(dbgs() << "\nMerging into block: " << *PrevBB
<< "From MBB: " << *TailBB);
if (PreRegAlloc) {
DenseMap<unsigned, unsigned> LocalVRMap;
SmallVector<std::pair<unsigned,unsigned>, 4> CopyInfos;
MachineBasicBlock::iterator I = TailBB->begin();
// Process PHI instructions first.
while (I != TailBB->end() && I->isPHI()) {
// Replace the uses of the def of the PHI with the register coming
// from PredBB.
MachineInstr *MI = &*I++;
ProcessPHI(MI, TailBB, PrevBB, LocalVRMap, CopyInfos);
if (MI->getParent())
MI->eraseFromParent();
}
// Now copy the non-PHI instructions.
while (I != TailBB->end()) {
// Replace def of virtual registers with new registers, and update
// uses with PHI source register or the new registers.
MachineInstr *MI = &*I++;
DuplicateInstruction(MI, TailBB, PrevBB, MF, LocalVRMap);
MI->eraseFromParent();
}
MachineBasicBlock::iterator Loc = PrevBB->getFirstTerminator();
for (unsigned i = 0, e = CopyInfos.size(); i != e; ++i) {
Copies.push_back(BuildMI(*PrevBB, Loc, DebugLoc(),
TII->get(TargetOpcode::COPY),
CopyInfos[i].first)
.addReg(CopyInfos[i].second));
}
} else {
// No PHIs to worry about, just splice the instructions over.
PrevBB->splice(PrevBB->end(), TailBB, TailBB->begin(), TailBB->end());
}
PrevBB->removeSuccessor(PrevBB->succ_begin());
assert(PrevBB->succ_empty());
PrevBB->transferSuccessors(TailBB);
TDBBs.push_back(PrevBB);
Changed = true;
}
return Changed;
}
// Expand branch instructions to long branches.
// TODO: This function has to be fixed for beqz16 and bnez16, because it
// currently assumes that all branches have 16-bit offsets, and will produce
// wrong code if branches whose allowed offsets are [-128, -126, ..., 126]
// are present.
void MipsLongBranch::expandToLongBranch(MBBInfo &I) {
MachineBasicBlock::iterator Pos;
MachineBasicBlock *MBB = I.Br->getParent(), *TgtMBB = getTargetMBB(*I.Br);
DebugLoc DL = I.Br->getDebugLoc();
const BasicBlock *BB = MBB->getBasicBlock();
MachineFunction::iterator FallThroughMBB = ++MachineFunction::iterator(MBB);
MachineBasicBlock *LongBrMBB = MF->CreateMachineBasicBlock(BB);
const MipsSubtarget &Subtarget =
static_cast<const MipsSubtarget &>(MF->getSubtarget());
const MipsInstrInfo *TII =
static_cast<const MipsInstrInfo *>(Subtarget.getInstrInfo());
MF->insert(FallThroughMBB, LongBrMBB);
MBB->replaceSuccessor(TgtMBB, LongBrMBB);
if (IsPIC) {
MachineBasicBlock *BalTgtMBB = MF->CreateMachineBasicBlock(BB);
MF->insert(FallThroughMBB, BalTgtMBB);
LongBrMBB->addSuccessor(BalTgtMBB);
BalTgtMBB->addSuccessor(TgtMBB);
// We must select between the MIPS32r6/MIPS64r6 BAL (which is a normal
// instruction) and the pre-MIPS32r6/MIPS64r6 definition (which is an
// pseudo-instruction wrapping BGEZAL).
unsigned BalOp = Subtarget.hasMips32r6() ? Mips::BAL : Mips::BAL_BR;
if (!ABI.IsN64()) {
// $longbr:
// addiu $sp, $sp, -8
// sw $ra, 0($sp)
// lui $at, %hi($tgt - $baltgt)
// bal $baltgt
// addiu $at, $at, %lo($tgt - $baltgt)
// $baltgt:
// addu $at, $ra, $at
// lw $ra, 0($sp)
// jr $at
// addiu $sp, $sp, 8
// $fallthrough:
//
Pos = LongBrMBB->begin();
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::ADDiu), Mips::SP)
.addReg(Mips::SP).addImm(-8);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::SW)).addReg(Mips::RA)
.addReg(Mips::SP).addImm(0);
// LUi and ADDiu instructions create 32-bit offset of the target basic
// block from the target of BAL instruction. We cannot use immediate
// value for this offset because it cannot be determined accurately when
// the program has inline assembly statements. We therefore use the
// relocation expressions %hi($tgt-$baltgt) and %lo($tgt-$baltgt) which
// are resolved during the fixup, so the values will always be correct.
//
// Since we cannot create %hi($tgt-$baltgt) and %lo($tgt-$baltgt)
// expressions at this point (it is possible only at the MC layer),
// we replace LUi and ADDiu with pseudo instructions
// LONG_BRANCH_LUi and LONG_BRANCH_ADDiu, and add both basic
// blocks as operands to these instructions. When lowering these pseudo
// instructions to LUi and ADDiu in the MC layer, we will create
// %hi($tgt-$baltgt) and %lo($tgt-$baltgt) expressions and add them as
// operands to lowered instructions.
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::LONG_BRANCH_LUi), Mips::AT)
.addMBB(TgtMBB).addMBB(BalTgtMBB);
MIBundleBuilder(*LongBrMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(BalOp)).addMBB(BalTgtMBB))
.append(BuildMI(*MF, DL, TII->get(Mips::LONG_BRANCH_ADDiu), Mips::AT)
.addReg(Mips::AT)
.addMBB(TgtMBB)
.addMBB(BalTgtMBB));
Pos = BalTgtMBB->begin();
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::ADDu), Mips::AT)
.addReg(Mips::RA).addReg(Mips::AT);
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::LW), Mips::RA)
.addReg(Mips::SP).addImm(0);
if (!Subtarget.isTargetNaCl()) {
MIBundleBuilder(*BalTgtMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(Mips::JR)).addReg(Mips::AT))
.append(BuildMI(*MF, DL, TII->get(Mips::ADDiu), Mips::SP)
.addReg(Mips::SP).addImm(8));
} else {
// In NaCl, modifying the sp is not allowed in branch delay slot.
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::ADDiu), Mips::SP)
.addReg(Mips::SP).addImm(8);
MIBundleBuilder(*BalTgtMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(Mips::JR)).addReg(Mips::AT))
.append(BuildMI(*MF, DL, TII->get(Mips::NOP)));
// Bundle-align the target of indirect branch JR.
TgtMBB->setAlignment(MIPS_NACL_BUNDLE_ALIGN);
}
} else {
// $longbr:
// daddiu $sp, $sp, -16
// sd $ra, 0($sp)
// daddiu $at, $zero, %hi($tgt - $baltgt)
// dsll $at, $at, 16
// bal $baltgt
// daddiu $at, $at, %lo($tgt - $baltgt)
// $baltgt:
// daddu $at, $ra, $at
// ld $ra, 0($sp)
// jr64 $at
// daddiu $sp, $sp, 16
// $fallthrough:
//
// We assume the branch is within-function, and that offset is within
// +/- 2GB. High 32 bits will therefore always be zero.
// Note that this will work even if the offset is negative, because
// of the +1 modification that's added in that case. For example, if the
// offset is -1MB (0xFFFFFFFFFFF00000), the computation for %higher is
//
// 0xFFFFFFFFFFF00000 + 0x80008000 = 0x000000007FF08000
//
// and the bits [47:32] are zero. For %highest
//
// 0xFFFFFFFFFFF00000 + 0x800080008000 = 0x000080007FF08000
//
// and the bits [63:48] are zero.
Pos = LongBrMBB->begin();
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::SP_64)
.addReg(Mips::SP_64).addImm(-16);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::SD)).addReg(Mips::RA_64)
.addReg(Mips::SP_64).addImm(0);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::LONG_BRANCH_DADDiu),
Mips::AT_64).addReg(Mips::ZERO_64)
.addMBB(TgtMBB, MipsII::MO_ABS_HI).addMBB(BalTgtMBB);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DSLL), Mips::AT_64)
.addReg(Mips::AT_64).addImm(16);
MIBundleBuilder(*LongBrMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(BalOp)).addMBB(BalTgtMBB))
.append(
BuildMI(*MF, DL, TII->get(Mips::LONG_BRANCH_DADDiu), Mips::AT_64)
.addReg(Mips::AT_64)
.addMBB(TgtMBB, MipsII::MO_ABS_LO)
.addMBB(BalTgtMBB));
Pos = BalTgtMBB->begin();
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::DADDu), Mips::AT_64)
.addReg(Mips::RA_64).addReg(Mips::AT_64);
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::LD), Mips::RA_64)
.addReg(Mips::SP_64).addImm(0);
MIBundleBuilder(*BalTgtMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(Mips::JR64)).addReg(Mips::AT_64))
.append(BuildMI(*MF, DL, TII->get(Mips::DADDiu), Mips::SP_64)
.addReg(Mips::SP_64).addImm(16));
}
assert(LongBrMBB->size() + BalTgtMBB->size() == LongBranchSeqSize);
} else {
// $longbr:
// j $tgt
// nop
// $fallthrough:
//
Pos = LongBrMBB->begin();
LongBrMBB->addSuccessor(TgtMBB);
MIBundleBuilder(*LongBrMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(Mips::J)).addMBB(TgtMBB))
.append(BuildMI(*MF, DL, TII->get(Mips::NOP)));
assert(LongBrMBB->size() == LongBranchSeqSize);
}
if (I.Br->isUnconditionalBranch()) {
// Change branch destination.
assert(I.Br->getDesc().getNumOperands() == 1);
I.Br->RemoveOperand(0);
I.Br->addOperand(MachineOperand::CreateMBB(LongBrMBB));
} else
// Change branch destination and reverse condition.
replaceBranch(*MBB, I.Br, DL, &*FallThroughMBB);
}
void X86CmovConverterPass::convertCmovInstsToBranches(
SmallVectorImpl<MachineInstr *> &Group) const {
assert(!Group.empty() && "No CMOV instructions to convert");
++NumOfOptimizedCmovGroups;
// If the CMOV group is not packed, e.g., there are debug instructions between
// first CMOV and last CMOV, then pack the group and make the CMOV instruction
// consecutive by moving the debug instructions to after the last CMOV.
packCmovGroup(Group.front(), Group.back());
// To convert a CMOVcc instruction, we actually have to insert the diamond
// control-flow pattern. The incoming instruction knows the destination vreg
// to set, the condition code register to branch on, the true/false values to
// select between, and a branch opcode to use.
// Before
// -----
// MBB:
// cond = cmp ...
// v1 = CMOVge t1, f1, cond
// v2 = CMOVlt t2, f2, cond
// v3 = CMOVge v1, f3, cond
//
// After
// -----
// MBB:
// cond = cmp ...
// jge %SinkMBB
//
// FalseMBB:
// jmp %SinkMBB
//
// SinkMBB:
// %v1 = phi[%f1, %FalseMBB], [%t1, %MBB]
// %v2 = phi[%t2, %FalseMBB], [%f2, %MBB] ; For CMOV with OppCC switch
// ; true-value with false-value
// %v3 = phi[%f3, %FalseMBB], [%t1, %MBB] ; Phi instruction cannot use
// ; previous Phi instruction result
MachineInstr &MI = *Group.front();
MachineInstr *LastCMOV = Group.back();
DebugLoc DL = MI.getDebugLoc();
X86::CondCode CC = X86::CondCode(X86::getCondFromCMovOpc(MI.getOpcode()));
X86::CondCode OppCC = X86::GetOppositeBranchCondition(CC);
// Potentially swap the condition codes so that any memory operand to a CMOV
// is in the *false* position instead of the *true* position. We can invert
// any non-memory operand CMOV instructions to cope with this and we ensure
// memory operand CMOVs are only included with a single condition code.
if (llvm::any_of(Group, [&](MachineInstr *I) {
return I->mayLoad() && X86::getCondFromCMovOpc(I->getOpcode()) == CC;
}))
std::swap(CC, OppCC);
MachineBasicBlock *MBB = MI.getParent();
MachineFunction::iterator It = ++MBB->getIterator();
MachineFunction *F = MBB->getParent();
const BasicBlock *BB = MBB->getBasicBlock();
MachineBasicBlock *FalseMBB = F->CreateMachineBasicBlock(BB);
MachineBasicBlock *SinkMBB = F->CreateMachineBasicBlock(BB);
F->insert(It, FalseMBB);
F->insert(It, SinkMBB);
// If the EFLAGS register isn't dead in the terminator, then claim that it's
// live into the sink and copy blocks.
if (checkEFLAGSLive(LastCMOV)) {
FalseMBB->addLiveIn(X86::EFLAGS);
SinkMBB->addLiveIn(X86::EFLAGS);
}
// Transfer the remainder of BB and its successor edges to SinkMBB.
SinkMBB->splice(SinkMBB->begin(), MBB,
std::next(MachineBasicBlock::iterator(LastCMOV)), MBB->end());
SinkMBB->transferSuccessorsAndUpdatePHIs(MBB);
// Add the false and sink blocks as its successors.
MBB->addSuccessor(FalseMBB);
MBB->addSuccessor(SinkMBB);
// Create the conditional branch instruction.
BuildMI(MBB, DL, TII->get(X86::GetCondBranchFromCond(CC))).addMBB(SinkMBB);
// Add the sink block to the false block successors.
FalseMBB->addSuccessor(SinkMBB);
MachineInstrBuilder MIB;
MachineBasicBlock::iterator MIItBegin = MachineBasicBlock::iterator(MI);
MachineBasicBlock::iterator MIItEnd =
std::next(MachineBasicBlock::iterator(LastCMOV));
MachineBasicBlock::iterator FalseInsertionPoint = FalseMBB->begin();
MachineBasicBlock::iterator SinkInsertionPoint = SinkMBB->begin();
// First we need to insert an explicit load on the false path for any memory
// operand. We also need to potentially do register rewriting here, but it is
// simpler as the memory operands are always on the false path so we can
// simply take that input, whatever it is.
DenseMap<unsigned, unsigned> FalseBBRegRewriteTable;
for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd;) {
auto &MI = *MIIt++;
// Skip any CMOVs in this group which don't load from memory.
if (!MI.mayLoad()) {
// Remember the false-side register input.
unsigned FalseReg =
MI.getOperand(X86::getCondFromCMovOpc(MI.getOpcode()) == CC ? 1 : 2)
.getReg();
// Walk back through any intermediate cmovs referenced.
while (true) {
auto FRIt = FalseBBRegRewriteTable.find(FalseReg);
if (FRIt == FalseBBRegRewriteTable.end())
break;
FalseReg = FRIt->second;
}
FalseBBRegRewriteTable[MI.getOperand(0).getReg()] = FalseReg;
continue;
}
// The condition must be the *opposite* of the one we've decided to branch
// on as the branch will go *around* the load and the load should happen
// when the CMOV condition is false.
assert(X86::getCondFromCMovOpc(MI.getOpcode()) == OppCC &&
"Can only handle memory-operand cmov instructions with a condition "
"opposite to the selected branch direction.");
// The goal is to rewrite the cmov from:
//
// MBB:
// %A = CMOVcc %B (tied), (mem)
//
// to
//
// MBB:
// %A = CMOVcc %B (tied), %C
// FalseMBB:
// %C = MOV (mem)
//
// Which will allow the next loop to rewrite the CMOV in terms of a PHI:
//
// MBB:
// JMP!cc SinkMBB
// FalseMBB:
// %C = MOV (mem)
// SinkMBB:
// %A = PHI [ %C, FalseMBB ], [ %B, MBB]
// Get a fresh register to use as the destination of the MOV.
const TargetRegisterClass *RC = MRI->getRegClass(MI.getOperand(0).getReg());
unsigned TmpReg = MRI->createVirtualRegister(RC);
SmallVector<MachineInstr *, 4> NewMIs;
bool Unfolded = TII->unfoldMemoryOperand(*MBB->getParent(), MI, TmpReg,
/*UnfoldLoad*/ true,
/*UnfoldStore*/ false, NewMIs);
(void)Unfolded;
assert(Unfolded && "Should never fail to unfold a loading cmov!");
// Move the new CMOV to just before the old one and reset any impacted
// iterator.
auto *NewCMOV = NewMIs.pop_back_val();
assert(X86::getCondFromCMovOpc(NewCMOV->getOpcode()) == OppCC &&
"Last new instruction isn't the expected CMOV!");
LLVM_DEBUG(dbgs() << "\tRewritten cmov: "; NewCMOV->dump());
MBB->insert(MachineBasicBlock::iterator(MI), NewCMOV);
if (&*MIItBegin == &MI)
MIItBegin = MachineBasicBlock::iterator(NewCMOV);
// Sink whatever instructions were needed to produce the unfolded operand
// into the false block.
for (auto *NewMI : NewMIs) {
LLVM_DEBUG(dbgs() << "\tRewritten load instr: "; NewMI->dump());
FalseMBB->insert(FalseInsertionPoint, NewMI);
// Re-map any operands that are from other cmovs to the inputs for this block.
for (auto &MOp : NewMI->uses()) {
if (!MOp.isReg())
continue;
auto It = FalseBBRegRewriteTable.find(MOp.getReg());
if (It == FalseBBRegRewriteTable.end())
continue;
MOp.setReg(It->second);
// This might have been a kill when it referenced the cmov result, but
// it won't necessarily be once rewritten.
// FIXME: We could potentially improve this by tracking whether the
// operand to the cmov was also a kill, and then skipping the PHI node
// construction below.
MOp.setIsKill(false);
}
}
MBB->erase(MachineBasicBlock::iterator(MI),
std::next(MachineBasicBlock::iterator(MI)));
// Add this PHI to the rewrite table.
FalseBBRegRewriteTable[NewCMOV->getOperand(0).getReg()] = TmpReg;
}
// As we are creating the PHIs, we have to be careful if there is more than
// one. Later CMOVs may reference the results of earlier CMOVs, but later
// PHIs have to reference the individual true/false inputs from earlier PHIs.
// That also means that PHI construction must work forward from earlier to
// later, and that the code must maintain a mapping from earlier PHI's
// destination registers, and the registers that went into the PHI.
DenseMap<unsigned, std::pair<unsigned, unsigned>> RegRewriteTable;
for (MachineBasicBlock::iterator MIIt = MIItBegin; MIIt != MIItEnd; ++MIIt) {
unsigned DestReg = MIIt->getOperand(0).getReg();
unsigned Op1Reg = MIIt->getOperand(1).getReg();
unsigned Op2Reg = MIIt->getOperand(2).getReg();
// If this CMOV we are processing is the opposite condition from the jump we
// generated, then we have to swap the operands for the PHI that is going to
// be generated.
if (X86::getCondFromCMovOpc(MIIt->getOpcode()) == OppCC)
std::swap(Op1Reg, Op2Reg);
auto Op1Itr = RegRewriteTable.find(Op1Reg);
if (Op1Itr != RegRewriteTable.end())
Op1Reg = Op1Itr->second.first;
auto Op2Itr = RegRewriteTable.find(Op2Reg);
if (Op2Itr != RegRewriteTable.end())
Op2Reg = Op2Itr->second.second;
// SinkMBB:
// %Result = phi [ %FalseValue, FalseMBB ], [ %TrueValue, MBB ]
// ...
MIB = BuildMI(*SinkMBB, SinkInsertionPoint, DL, TII->get(X86::PHI), DestReg)
.addReg(Op1Reg)
.addMBB(FalseMBB)
.addReg(Op2Reg)
.addMBB(MBB);
(void)MIB;
LLVM_DEBUG(dbgs() << "\tFrom: "; MIIt->dump());
LLVM_DEBUG(dbgs() << "\tTo: "; MIB->dump());
// Add this PHI to the rewrite table.
RegRewriteTable[DestReg] = std::make_pair(Op1Reg, Op2Reg);
}
// Now remove the CMOV(s).
MBB->erase(MIItBegin, MIItEnd);
}
// Expand branch instructions to long branches.
void MipsLongBranch::expandToLongBranch(MBBInfo &I) {
MachineBasicBlock::iterator Pos;
MachineBasicBlock *MBB = I.Br->getParent(), *TgtMBB = getTargetMBB(*I.Br);
DebugLoc DL = I.Br->getDebugLoc();
const BasicBlock *BB = MBB->getBasicBlock();
MachineFunction::iterator FallThroughMBB = ++MachineFunction::iterator(MBB);
MachineBasicBlock *LongBrMBB = MF->CreateMachineBasicBlock(BB);
const MipsInstrInfo *TII =
static_cast<const MipsInstrInfo*>(TM.getInstrInfo());
MF->insert(FallThroughMBB, LongBrMBB);
MBB->removeSuccessor(TgtMBB);
MBB->addSuccessor(LongBrMBB);
if (IsPIC) {
MachineBasicBlock *BalTgtMBB = MF->CreateMachineBasicBlock(BB);
MF->insert(FallThroughMBB, BalTgtMBB);
LongBrMBB->addSuccessor(BalTgtMBB);
BalTgtMBB->addSuccessor(TgtMBB);
int64_t TgtAddress = MBBInfos[TgtMBB->getNumber()].Address;
unsigned BalTgtMBBSize = 5;
int64_t Offset = TgtAddress - (I.Address + I.Size - BalTgtMBBSize * 4);
int64_t Lo = SignExtend64<16>(Offset & 0xffff);
int64_t Hi = SignExtend64<16>(((Offset + 0x8000) >> 16) & 0xffff);
if (ABI != MipsSubtarget::N64) {
// $longbr:
// addiu $sp, $sp, -8
// sw $ra, 0($sp)
// bal $baltgt
// lui $at, %hi($tgt - $baltgt)
// $baltgt:
// addiu $at, $at, %lo($tgt - $baltgt)
// addu $at, $ra, $at
// lw $ra, 0($sp)
// jr $at
// addiu $sp, $sp, 8
// $fallthrough:
//
Pos = LongBrMBB->begin();
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::ADDiu), Mips::SP)
.addReg(Mips::SP).addImm(-8);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::SW)).addReg(Mips::RA)
.addReg(Mips::SP).addImm(0);
MIBundleBuilder(*LongBrMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(Mips::BAL_BR)).addMBB(BalTgtMBB))
.append(BuildMI(*MF, DL, TII->get(Mips::LUi), Mips::AT).addImm(Hi));
Pos = BalTgtMBB->begin();
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::ADDiu), Mips::AT)
.addReg(Mips::AT).addImm(Lo);
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::ADDu), Mips::AT)
.addReg(Mips::RA).addReg(Mips::AT);
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::LW), Mips::RA)
.addReg(Mips::SP).addImm(0);
MIBundleBuilder(*BalTgtMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(Mips::JR)).addReg(Mips::AT))
.append(BuildMI(*MF, DL, TII->get(Mips::ADDiu), Mips::SP)
.addReg(Mips::SP).addImm(8));
} else {
// $longbr:
// daddiu $sp, $sp, -16
// sd $ra, 0($sp)
// lui64 $at, %highest($tgt - $baltgt)
// daddiu $at, $at, %higher($tgt - $baltgt)
// dsll $at, $at, 16
// daddiu $at, $at, %hi($tgt - $baltgt)
// bal $baltgt
// dsll $at, $at, 16
// $baltgt:
// daddiu $at, $at, %lo($tgt - $baltgt)
// daddu $at, $ra, $at
// ld $ra, 0($sp)
// jr64 $at
// daddiu $sp, $sp, 16
// $fallthrough:
//
int64_t Higher = SignExtend64<16>(((Offset + 0x80008000) >> 32) & 0xffff);
int64_t Highest =
SignExtend64<16>(((Offset + 0x800080008000LL) >> 48) & 0xffff);
Pos = LongBrMBB->begin();
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::SP_64)
.addReg(Mips::SP_64).addImm(-16);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::SD)).addReg(Mips::RA_64)
.addReg(Mips::SP_64).addImm(0);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::LUi64), Mips::AT_64)
.addImm(Highest);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::AT_64)
.addReg(Mips::AT_64).addImm(Higher);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DSLL), Mips::AT_64)
.addReg(Mips::AT_64).addImm(16);
BuildMI(*LongBrMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::AT_64)
.addReg(Mips::AT_64).addImm(Hi);
MIBundleBuilder(*LongBrMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(Mips::BAL_BR)).addMBB(BalTgtMBB))
.append(BuildMI(*MF, DL, TII->get(Mips::DSLL), Mips::AT_64)
.addReg(Mips::AT_64).addImm(16));
Pos = BalTgtMBB->begin();
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::DADDiu), Mips::AT_64)
.addReg(Mips::AT_64).addImm(Lo);
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::DADDu), Mips::AT_64)
.addReg(Mips::RA_64).addReg(Mips::AT_64);
BuildMI(*BalTgtMBB, Pos, DL, TII->get(Mips::LD), Mips::RA_64)
.addReg(Mips::SP_64).addImm(0);
MIBundleBuilder(*BalTgtMBB, Pos)
.append(BuildMI(*MF, DL, TII->get(Mips::JR64)).addReg(Mips::AT_64))
.append(BuildMI(*MF, DL, TII->get(Mips::DADDiu), Mips::SP_64)
.addReg(Mips::SP_64).addImm(16));
}
assert(BalTgtMBBSize == BalTgtMBB->size());
assert(LongBrMBB->size() + BalTgtMBBSize == LongBranchSeqSize);
} else {
bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
MachineLoopInfo &MLI,
MachineLoop *Loop) {
MachineBasicBlock *Header = Loop ? Loop->getHeader() : &*MF.begin();
SetVector<MachineBasicBlock *> RewriteSuccs;
// DFS through Loop's body, looking for for irreducible control flow. Loop is
// natural, and we stay in its body, and we treat any nested loops
// monolithically, so any cycles we encounter indicate irreducibility.
SmallPtrSet<MachineBasicBlock *, 8> OnStack;
SmallPtrSet<MachineBasicBlock *, 8> Visited;
SmallVector<SuccessorList, 4> LoopWorklist;
LoopWorklist.push_back(SuccessorList(Header));
OnStack.insert(Header);
Visited.insert(Header);
while (!LoopWorklist.empty()) {
SuccessorList &Top = LoopWorklist.back();
if (Top.HasNext()) {
MachineBasicBlock *Next = Top.Next();
if (Next == Header || (Loop && !Loop->contains(Next)))
continue;
if (LLVM_LIKELY(OnStack.insert(Next).second)) {
if (!Visited.insert(Next).second) {
OnStack.erase(Next);
continue;
}
MachineLoop *InnerLoop = MLI.getLoopFor(Next);
if (InnerLoop != Loop)
LoopWorklist.push_back(SuccessorList(InnerLoop));
else
LoopWorklist.push_back(SuccessorList(Next));
} else {
RewriteSuccs.insert(Top.getBlock());
}
continue;
}
OnStack.erase(Top.getBlock());
LoopWorklist.pop_back();
}
// Most likely, we didn't find any irreducible control flow.
if (LLVM_LIKELY(RewriteSuccs.empty()))
return false;
DEBUG(dbgs() << "Irreducible control flow detected!\n");
// Ok. We have irreducible control flow! Create a dispatch block which will
// contains a jump table to any block in the problematic set of blocks.
MachineBasicBlock *Dispatch = MF.CreateMachineBasicBlock();
MF.insert(MF.end(), Dispatch);
MLI.changeLoopFor(Dispatch, Loop);
// Add the jump table.
const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
MachineInstrBuilder MIB = BuildMI(*Dispatch, Dispatch->end(), DebugLoc(),
TII.get(WebAssembly::BR_TABLE_I32));
// Add the register which will be used to tell the jump table which block to
// jump to.
MachineRegisterInfo &MRI = MF.getRegInfo();
unsigned Reg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
MIB.addReg(Reg);
// Collect all the blocks which need to have their successors rewritten,
// add the successors to the jump table, and remember their index.
DenseMap<MachineBasicBlock *, unsigned> Indices;
SmallVector<MachineBasicBlock *, 4> SuccWorklist(RewriteSuccs.begin(),
RewriteSuccs.end());
while (!SuccWorklist.empty()) {
MachineBasicBlock *MBB = SuccWorklist.pop_back_val();
auto Pair = Indices.insert(std::make_pair(MBB, 0));
if (!Pair.second)
continue;
unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1;
DEBUG(dbgs() << printMBBReference(*MBB) << " has index " << Index << "\n");
Pair.first->second = Index;
for (auto Pred : MBB->predecessors())
RewriteSuccs.insert(Pred);
MIB.addMBB(MBB);
Dispatch->addSuccessor(MBB);
MetaBlock Meta(MBB);
for (auto *Succ : Meta.successors())
if (Succ != Header && (!Loop || Loop->contains(Succ)))
SuccWorklist.push_back(Succ);
}
// Rewrite the problematic successors for every block in RewriteSuccs.
// For simplicity, we just introduce a new block for every edge we need to
// rewrite. Fancier things are possible.
for (MachineBasicBlock *MBB : RewriteSuccs) {
DenseMap<MachineBasicBlock *, MachineBasicBlock *> Map;
for (auto *Succ : MBB->successors()) {
if (!Indices.count(Succ))
continue;
MachineBasicBlock *Split = MF.CreateMachineBasicBlock();
MF.insert(MBB->isLayoutSuccessor(Succ) ? MachineFunction::iterator(Succ)
: MF.end(),
Split);
MLI.changeLoopFor(Split, Loop);
// Set the jump table's register of the index of the block we wish to
// jump to, and jump to the jump table.
BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::CONST_I32),
Reg)
.addImm(Indices[Succ]);
BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::BR))
.addMBB(Dispatch);
Split->addSuccessor(Dispatch);
Map[Succ] = Split;
}
// Remap the terminator operands and the successor list.
for (MachineInstr &Term : MBB->terminators())
for (auto &Op : Term.explicit_uses())
if (Op.isMBB() && Indices.count(Op.getMBB()))
Op.setMBB(Map[Op.getMBB()]);
for (auto Rewrite : Map)
MBB->replaceSuccessor(Rewrite.first, Rewrite.second);
}
// Create a fake default label, because br_table requires one.
MIB.addMBB(MIB.getInstr()
->getOperand(MIB.getInstr()->getNumExplicitOperands() - 1)
.getMBB());
return true;
}
/// fixupConditionalBranch - Fix up a conditional branch whose destination is
/// too far away to fit in its displacement field. It is converted to an inverse
/// conditional branch + an unconditional branch to the destination.
bool AArch64BranchRelaxation::fixupConditionalBranch(MachineInstr *MI) {
MachineBasicBlock *DestBB = getDestBlock(MI);
// Add an unconditional branch to the destination and invert the branch
// condition to jump over it:
// tbz L1
// =>
// tbnz L2
// b L1
// L2:
// If the branch is at the end of its MBB and that has a fall-through block,
// direct the updated conditional branch to the fall-through block. Otherwise,
// split the MBB before the next instruction.
MachineBasicBlock *MBB = MI->getParent();
MachineInstr *BMI = &MBB->back();
bool NeedSplit = (BMI != MI) || !BBHasFallthrough(MBB);
if (BMI != MI) {
if (std::next(MachineBasicBlock::iterator(MI)) ==
std::prev(MBB->getLastNonDebugInstr()) &&
BMI->getOpcode() == AArch64::B) {
// Last MI in the BB is an unconditional branch. Can we simply invert the
// condition and swap destinations:
// beq L1
// b L2
// =>
// bne L2
// b L1
MachineBasicBlock *NewDest = BMI->getOperand(0).getMBB();
if (isBlockInRange(MI, NewDest,
getBranchDisplacementBits(MI->getOpcode()))) {
DEBUG(dbgs() << " Invert condition and swap its destination with "
<< *BMI);
BMI->getOperand(0).setMBB(DestBB);
unsigned OpNum = (MI->getOpcode() == AArch64::TBZW ||
MI->getOpcode() == AArch64::TBNZW ||
MI->getOpcode() == AArch64::TBZX ||
MI->getOpcode() == AArch64::TBNZX)
? 2
: 1;
MI->getOperand(OpNum).setMBB(NewDest);
MI->setDesc(TII->get(getOppositeConditionOpcode(MI->getOpcode())));
if (MI->getOpcode() == AArch64::Bcc)
invertBccCondition(MI);
return true;
}
}
}
if (NeedSplit) {
// Analyze the branch so we know how to update the successor lists.
MachineBasicBlock *TBB, *FBB;
SmallVector<MachineOperand, 2> Cond;
TII->AnalyzeBranch(*MBB, TBB, FBB, Cond, false);
MachineBasicBlock *NewBB = splitBlockBeforeInstr(MI);
// No need for the branch to the next block. We're adding an unconditional
// branch to the destination.
int delta = TII->GetInstSizeInBytes(&MBB->back());
BlockInfo[MBB->getNumber()].Size -= delta;
MBB->back().eraseFromParent();
// BlockInfo[SplitBB].Offset is wrong temporarily, fixed below
// Update the successor lists according to the transformation to follow.
// Do it here since if there's no split, no update is needed.
MBB->replaceSuccessor(FBB, NewBB);
NewBB->addSuccessor(FBB);
}
MachineBasicBlock *NextBB = std::next(MachineFunction::iterator(MBB));
DEBUG(dbgs() << " Insert B to BB#" << DestBB->getNumber()
<< ", invert condition and change dest. to BB#"
<< NextBB->getNumber() << "\n");
// Insert a new conditional branch and a new unconditional branch.
MachineInstrBuilder MIB = BuildMI(
MBB, DebugLoc(), TII->get(getOppositeConditionOpcode(MI->getOpcode())))
.addOperand(MI->getOperand(0));
if (MI->getOpcode() == AArch64::TBZW || MI->getOpcode() == AArch64::TBNZW ||
MI->getOpcode() == AArch64::TBZX || MI->getOpcode() == AArch64::TBNZX)
MIB.addOperand(MI->getOperand(1));
if (MI->getOpcode() == AArch64::Bcc)
invertBccCondition(MIB);
MIB.addMBB(NextBB);
BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back());
BuildMI(MBB, DebugLoc(), TII->get(AArch64::B)).addMBB(DestBB);
BlockInfo[MBB->getNumber()].Size += TII->GetInstSizeInBytes(&MBB->back());
// Remove the old conditional branch. It may or may not still be in MBB.
BlockInfo[MI->getParent()->getNumber()].Size -= TII->GetInstSizeInBytes(MI);
MI->eraseFromParent();
// Finally, keep the block offsets up to date.
adjustBlockOffsets(*MBB);
return true;
}
MachineBasicBlock *
AlphaTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
MachineBasicBlock *BB) const {
const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
assert((MI->getOpcode() == Alpha::CAS32 ||
MI->getOpcode() == Alpha::CAS64 ||
MI->getOpcode() == Alpha::LAS32 ||
MI->getOpcode() == Alpha::LAS64 ||
MI->getOpcode() == Alpha::SWAP32 ||
MI->getOpcode() == Alpha::SWAP64) &&
"Unexpected instr type to insert");
bool is32 = MI->getOpcode() == Alpha::CAS32 ||
MI->getOpcode() == Alpha::LAS32 ||
MI->getOpcode() == Alpha::SWAP32;
//Load locked store conditional for atomic ops take on the same form
//start:
//ll
//do stuff (maybe branch to exit)
//sc
//test sc and maybe branck to start
//exit:
const BasicBlock *LLVM_BB = BB->getBasicBlock();
DebugLoc dl = MI->getDebugLoc();
MachineFunction::iterator It = BB;
++It;
MachineBasicBlock *thisMBB = BB;
MachineFunction *F = BB->getParent();
MachineBasicBlock *llscMBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *sinkMBB = F->CreateMachineBasicBlock(LLVM_BB);
sinkMBB->transferSuccessors(thisMBB);
F->insert(It, llscMBB);
F->insert(It, sinkMBB);
BuildMI(thisMBB, dl, TII->get(Alpha::BR)).addMBB(llscMBB);
unsigned reg_res = MI->getOperand(0).getReg(),
reg_ptr = MI->getOperand(1).getReg(),
reg_v2 = MI->getOperand(2).getReg(),
reg_store = F->getRegInfo().createVirtualRegister(&Alpha::GPRCRegClass);
BuildMI(llscMBB, dl, TII->get(is32 ? Alpha::LDL_L : Alpha::LDQ_L),
reg_res).addImm(0).addReg(reg_ptr);
switch (MI->getOpcode()) {
case Alpha::CAS32:
case Alpha::CAS64: {
unsigned reg_cmp
= F->getRegInfo().createVirtualRegister(&Alpha::GPRCRegClass);
BuildMI(llscMBB, dl, TII->get(Alpha::CMPEQ), reg_cmp)
.addReg(reg_v2).addReg(reg_res);
BuildMI(llscMBB, dl, TII->get(Alpha::BEQ))
.addImm(0).addReg(reg_cmp).addMBB(sinkMBB);
BuildMI(llscMBB, dl, TII->get(Alpha::BISr), reg_store)
.addReg(Alpha::R31).addReg(MI->getOperand(3).getReg());
break;
}
case Alpha::LAS32:
case Alpha::LAS64: {
BuildMI(llscMBB, dl,TII->get(is32 ? Alpha::ADDLr : Alpha::ADDQr), reg_store)
.addReg(reg_res).addReg(reg_v2);
break;
}
case Alpha::SWAP32:
case Alpha::SWAP64: {
BuildMI(llscMBB, dl, TII->get(Alpha::BISr), reg_store)
.addReg(reg_v2).addReg(reg_v2);
break;
}
}
BuildMI(llscMBB, dl, TII->get(is32 ? Alpha::STL_C : Alpha::STQ_C), reg_store)
.addReg(reg_store).addImm(0).addReg(reg_ptr);
BuildMI(llscMBB, dl, TII->get(Alpha::BEQ))
.addImm(0).addReg(reg_store).addMBB(llscMBB);
BuildMI(llscMBB, dl, TII->get(Alpha::BR)).addMBB(sinkMBB);
thisMBB->addSuccessor(llscMBB);
llscMBB->addSuccessor(llscMBB);
llscMBB->addSuccessor(sinkMBB);
F->DeleteMachineInstr(MI); // The pseudo instruction is gone now.
return sinkMBB;
}
//===----------------------------------------------------------------------===//
// Lower helper functions
//===----------------------------------------------------------------------===//
MachineBasicBlock*
MBlazeTargetLowering::EmitInstrWithCustomInserter(MachineInstr *MI,
MachineBasicBlock *BB) const {
const TargetInstrInfo *TII = getTargetMachine().getInstrInfo();
DebugLoc dl = MI->getDebugLoc();
switch (MI->getOpcode()) {
default:
assert(false && "Unexpected instr type to insert");
case MBlaze::ShiftRL:
case MBlaze::ShiftRA:
case MBlaze::ShiftL: {
// To "insert" a shift left instruction, we actually have to insert a
// simple loop. The incoming instruction knows the destination vreg to
// set, the source vreg to operate over and the shift amount.
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator It = BB;
++It;
// start:
// andi samt, samt, 31
// beqid samt, finish
// add dst, src, r0
// loop:
// addik samt, samt, -1
// sra dst, dst
// bneid samt, loop
// nop
// finish:
MachineFunction *F = BB->getParent();
MachineRegisterInfo &R = F->getRegInfo();
MachineBasicBlock *loop = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *finish = F->CreateMachineBasicBlock(LLVM_BB);
F->insert(It, loop);
F->insert(It, finish);
// Update machine-CFG edges by transfering adding all successors and
// remaining instructions from the current block to the new block which
// will contain the Phi node for the select.
finish->splice(finish->begin(), BB,
llvm::next(MachineBasicBlock::iterator(MI)),
BB->end());
finish->transferSuccessorsAndUpdatePHIs(BB);
// Add the true and fallthrough blocks as its successors.
BB->addSuccessor(loop);
BB->addSuccessor(finish);
// Next, add the finish block as a successor of the loop block
loop->addSuccessor(finish);
loop->addSuccessor(loop);
unsigned IAMT = R.createVirtualRegister(MBlaze::GPRRegisterClass);
BuildMI(BB, dl, TII->get(MBlaze::ANDI), IAMT)
.addReg(MI->getOperand(2).getReg())
.addImm(31);
unsigned IVAL = R.createVirtualRegister(MBlaze::GPRRegisterClass);
BuildMI(BB, dl, TII->get(MBlaze::ADDI), IVAL)
.addReg(MI->getOperand(1).getReg())
.addImm(0);
BuildMI(BB, dl, TII->get(MBlaze::BEQID))
.addReg(IAMT)
.addMBB(finish);
unsigned DST = R.createVirtualRegister(MBlaze::GPRRegisterClass);
unsigned NDST = R.createVirtualRegister(MBlaze::GPRRegisterClass);
BuildMI(loop, dl, TII->get(MBlaze::PHI), DST)
.addReg(IVAL).addMBB(BB)
.addReg(NDST).addMBB(loop);
unsigned SAMT = R.createVirtualRegister(MBlaze::GPRRegisterClass);
unsigned NAMT = R.createVirtualRegister(MBlaze::GPRRegisterClass);
BuildMI(loop, dl, TII->get(MBlaze::PHI), SAMT)
.addReg(IAMT).addMBB(BB)
.addReg(NAMT).addMBB(loop);
if (MI->getOpcode() == MBlaze::ShiftL)
BuildMI(loop, dl, TII->get(MBlaze::ADD), NDST).addReg(DST).addReg(DST);
else if (MI->getOpcode() == MBlaze::ShiftRA)
BuildMI(loop, dl, TII->get(MBlaze::SRA), NDST).addReg(DST);
else if (MI->getOpcode() == MBlaze::ShiftRL)
BuildMI(loop, dl, TII->get(MBlaze::SRL), NDST).addReg(DST);
else
llvm_unreachable("Cannot lower unknown shift instruction");
BuildMI(loop, dl, TII->get(MBlaze::ADDI), NAMT)
.addReg(SAMT)
.addImm(-1);
BuildMI(loop, dl, TII->get(MBlaze::BNEID))
.addReg(NAMT)
.addMBB(loop);
BuildMI(*finish, finish->begin(), dl,
TII->get(MBlaze::PHI), MI->getOperand(0).getReg())
.addReg(IVAL).addMBB(BB)
.addReg(NDST).addMBB(loop);
// The pseudo instruction is no longer needed so remove it
MI->eraseFromParent();
return finish;
}
case MBlaze::Select_FCC:
case MBlaze::Select_CC: {
// To "insert" a SELECT_CC instruction, we actually have to insert the
// diamond control-flow pattern. The incoming instruction knows the
// destination vreg to set, the condition code register to branch on, the
// true/false values to select between, and a branch opcode to use.
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator It = BB;
++It;
// thisMBB:
// ...
// TrueVal = ...
// setcc r1, r2, r3
// bNE r1, r0, copy1MBB
// fallthrough --> copy0MBB
MachineFunction *F = BB->getParent();
MachineBasicBlock *flsBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *dneBB = F->CreateMachineBasicBlock(LLVM_BB);
unsigned Opc;
switch (MI->getOperand(4).getImm()) {
default:
llvm_unreachable("Unknown branch condition");
case MBlazeCC::EQ:
Opc = MBlaze::BEQID;
break;
case MBlazeCC::NE:
Opc = MBlaze::BNEID;
break;
case MBlazeCC::GT:
Opc = MBlaze::BGTID;
break;
case MBlazeCC::LT:
Opc = MBlaze::BLTID;
break;
case MBlazeCC::GE:
Opc = MBlaze::BGEID;
break;
case MBlazeCC::LE:
Opc = MBlaze::BLEID;
break;
}
F->insert(It, flsBB);
F->insert(It, dneBB);
// Transfer the remainder of BB and its successor edges to dneBB.
dneBB->splice(dneBB->begin(), BB,
llvm::next(MachineBasicBlock::iterator(MI)),
BB->end());
dneBB->transferSuccessorsAndUpdatePHIs(BB);
BB->addSuccessor(flsBB);
BB->addSuccessor(dneBB);
flsBB->addSuccessor(dneBB);
BuildMI(BB, dl, TII->get(Opc))
.addReg(MI->getOperand(3).getReg())
.addMBB(dneBB);
// sinkMBB:
// %Result = phi [ %FalseValue, copy0MBB ], [ %TrueValue, thisMBB ]
// ...
//BuildMI(dneBB, dl, TII->get(MBlaze::PHI), MI->getOperand(0).getReg())
// .addReg(MI->getOperand(1).getReg()).addMBB(flsBB)
// .addReg(MI->getOperand(2).getReg()).addMBB(BB);
BuildMI(*dneBB, dneBB->begin(), dl,
TII->get(MBlaze::PHI), MI->getOperand(0).getReg())
.addReg(MI->getOperand(2).getReg()).addMBB(flsBB)
.addReg(MI->getOperand(1).getReg()).addMBB(BB);
MI->eraseFromParent(); // The pseudo instruction is gone now.
return dneBB;
}
}
}
MachineBasicBlock*
MSP430TargetLowering::EmitShiftInstr(MachineInstr *MI,
MachineBasicBlock *BB) const {
MachineFunction *F = BB->getParent();
MachineRegisterInfo &RI = F->getRegInfo();
DebugLoc dl = MI->getDebugLoc();
const TargetInstrInfo &TII = *getTargetMachine().getInstrInfo();
unsigned Opc;
const TargetRegisterClass * RC;
switch (MI->getOpcode()) {
default:
assert(0 && "Invalid shift opcode!");
case MSP430::Shl8:
Opc = MSP430::SHL8r1;
RC = MSP430::GR8RegisterClass;
break;
case MSP430::Shl16:
Opc = MSP430::SHL16r1;
RC = MSP430::GR16RegisterClass;
break;
case MSP430::Sra8:
Opc = MSP430::SAR8r1;
RC = MSP430::GR8RegisterClass;
break;
case MSP430::Sra16:
Opc = MSP430::SAR16r1;
RC = MSP430::GR16RegisterClass;
break;
case MSP430::Srl8:
Opc = MSP430::SAR8r1c;
RC = MSP430::GR8RegisterClass;
break;
case MSP430::Srl16:
Opc = MSP430::SAR16r1c;
RC = MSP430::GR16RegisterClass;
break;
}
const BasicBlock *LLVM_BB = BB->getBasicBlock();
MachineFunction::iterator I = BB;
++I;
// Create loop block
MachineBasicBlock *LoopBB = F->CreateMachineBasicBlock(LLVM_BB);
MachineBasicBlock *RemBB = F->CreateMachineBasicBlock(LLVM_BB);
F->insert(I, LoopBB);
F->insert(I, RemBB);
// Update machine-CFG edges by transferring all successors of the current
// block to the block containing instructions after shift.
RemBB->splice(RemBB->begin(), BB,
llvm::next(MachineBasicBlock::iterator(MI)),
BB->end());
RemBB->transferSuccessorsAndUpdatePHIs(BB);
// Add adges BB => LoopBB => RemBB, BB => RemBB, LoopBB => LoopBB
BB->addSuccessor(LoopBB);
BB->addSuccessor(RemBB);
LoopBB->addSuccessor(RemBB);
LoopBB->addSuccessor(LoopBB);
unsigned ShiftAmtReg = RI.createVirtualRegister(MSP430::GR8RegisterClass);
unsigned ShiftAmtReg2 = RI.createVirtualRegister(MSP430::GR8RegisterClass);
unsigned ShiftReg = RI.createVirtualRegister(RC);
unsigned ShiftReg2 = RI.createVirtualRegister(RC);
unsigned ShiftAmtSrcReg = MI->getOperand(2).getReg();
unsigned SrcReg = MI->getOperand(1).getReg();
unsigned DstReg = MI->getOperand(0).getReg();
// BB:
// cmp 0, N
// je RemBB
BuildMI(BB, dl, TII.get(MSP430::CMP8ri))
.addReg(ShiftAmtSrcReg).addImm(0);
BuildMI(BB, dl, TII.get(MSP430::JCC))
.addMBB(RemBB)
.addImm(MSP430CC::COND_E);
// LoopBB:
// ShiftReg = phi [%SrcReg, BB], [%ShiftReg2, LoopBB]
// ShiftAmt = phi [%N, BB], [%ShiftAmt2, LoopBB]
// ShiftReg2 = shift ShiftReg
// ShiftAmt2 = ShiftAmt - 1;
BuildMI(LoopBB, dl, TII.get(MSP430::PHI), ShiftReg)
.addReg(SrcReg).addMBB(BB)
.addReg(ShiftReg2).addMBB(LoopBB);
BuildMI(LoopBB, dl, TII.get(MSP430::PHI), ShiftAmtReg)
.addReg(ShiftAmtSrcReg).addMBB(BB)
.addReg(ShiftAmtReg2).addMBB(LoopBB);
BuildMI(LoopBB, dl, TII.get(Opc), ShiftReg2)
.addReg(ShiftReg);
BuildMI(LoopBB, dl, TII.get(MSP430::SUB8ri), ShiftAmtReg2)
.addReg(ShiftAmtReg).addImm(1);
BuildMI(LoopBB, dl, TII.get(MSP430::JCC))
.addMBB(LoopBB)
.addImm(MSP430CC::COND_NE);
// RemBB:
// DestReg = phi [%SrcReg, BB], [%ShiftReg, LoopBB]
BuildMI(*RemBB, RemBB->begin(), dl, TII.get(MSP430::PHI), DstReg)
.addReg(SrcReg).addMBB(BB)
.addReg(ShiftReg2).addMBB(LoopBB);
MI->eraseFromParent(); // The pseudo instruction is gone now.
return RemBB;
}
MachineBasicBlock *
MachineBasicBlock::SplitCriticalEdge(MachineBasicBlock *Succ, Pass *P) {
MachineFunction *MF = getParent();
DebugLoc dl; // FIXME: this is nowhere
// We may need to update this's terminator, but we can't do that if
// AnalyzeBranch fails. If this uses a jump table, we won't touch it.
const TargetInstrInfo *TII = MF->getTarget().getInstrInfo();
MachineBasicBlock *TBB = 0, *FBB = 0;
SmallVector<MachineOperand, 4> Cond;
if (TII->AnalyzeBranch(*this, TBB, FBB, Cond))
return NULL;
// Avoid bugpoint weirdness: A block may end with a conditional branch but
// jumps to the same MBB is either case. We have duplicate CFG edges in that
// case that we can't handle. Since this never happens in properly optimized
// code, just skip those edges.
if (TBB && TBB == FBB) {
DEBUG(dbgs() << "Won't split critical edge after degenerate BB#"
<< getNumber() << '\n');
return NULL;
}
MachineBasicBlock *NMBB = MF->CreateMachineBasicBlock();
MF->insert(llvm::next(MachineFunction::iterator(this)), NMBB);
DEBUG(dbgs() << "Splitting critical edge:"
" BB#" << getNumber()
<< " -- BB#" << NMBB->getNumber()
<< " -- BB#" << Succ->getNumber() << '\n');
// On some targets like Mips, branches may kill virtual registers. Make sure
// that LiveVariables is properly updated after updateTerminator replaces the
// terminators.
LiveVariables *LV = P->getAnalysisIfAvailable<LiveVariables>();
// Collect a list of virtual registers killed by the terminators.
SmallVector<unsigned, 4> KilledRegs;
if (LV)
for (iterator I = getFirstTerminator(), E = end(); I != E; ++I) {
MachineInstr *MI = I;
for (MachineInstr::mop_iterator OI = MI->operands_begin(),
OE = MI->operands_end(); OI != OE; ++OI) {
if (!OI->isReg() || !OI->isUse() || !OI->isKill() || OI->isUndef())
continue;
unsigned Reg = OI->getReg();
if (TargetRegisterInfo::isVirtualRegister(Reg) &&
LV->getVarInfo(Reg).removeKill(MI)) {
KilledRegs.push_back(Reg);
DEBUG(dbgs() << "Removing terminator kill: " << *MI);
OI->setIsKill(false);
}
}
}
ReplaceUsesOfBlockWith(Succ, NMBB);
updateTerminator();
// Insert unconditional "jump Succ" instruction in NMBB if necessary.
NMBB->addSuccessor(Succ);
if (!NMBB->isLayoutSuccessor(Succ)) {
Cond.clear();
MF->getTarget().getInstrInfo()->InsertBranch(*NMBB, Succ, NULL, Cond, dl);
}
// Fix PHI nodes in Succ so they refer to NMBB instead of this
for (MachineBasicBlock::iterator i = Succ->begin(), e = Succ->end();
i != e && i->isPHI(); ++i)
for (unsigned ni = 1, ne = i->getNumOperands(); ni != ne; ni += 2)
if (i->getOperand(ni+1).getMBB() == this)
i->getOperand(ni+1).setMBB(NMBB);
// Inherit live-ins from the successor
for (MachineBasicBlock::livein_iterator I = Succ->livein_begin(),
E = Succ->livein_end(); I != E; ++I)
NMBB->addLiveIn(*I);
// Update LiveVariables.
if (LV) {
// Restore kills of virtual registers that were killed by the terminators.
while (!KilledRegs.empty()) {
unsigned Reg = KilledRegs.pop_back_val();
for (iterator I = end(), E = begin(); I != E;) {
if (!(--I)->addRegisterKilled(Reg, NULL, /* addIfNotFound= */ false))
continue;
LV->getVarInfo(Reg).Kills.push_back(I);
DEBUG(dbgs() << "Restored terminator kill: " << *I);
break;
}
}
// Update relevant live-through information.
LV->addNewBlock(NMBB, this, Succ);
}
if (MachineDominatorTree *MDT =
P->getAnalysisIfAvailable<MachineDominatorTree>()) {
// Update dominator information.
MachineDomTreeNode *SucccDTNode = MDT->getNode(Succ);
bool IsNewIDom = true;
for (const_pred_iterator PI = Succ->pred_begin(), E = Succ->pred_end();
PI != E; ++PI) {
MachineBasicBlock *PredBB = *PI;
if (PredBB == NMBB)
continue;
if (!MDT->dominates(SucccDTNode, MDT->getNode(PredBB))) {
IsNewIDom = false;
break;
}
}
// We know "this" dominates the newly created basic block.
MachineDomTreeNode *NewDTNode = MDT->addNewBlock(NMBB, this);
// If all the other predecessors of "Succ" are dominated by "Succ" itself
// then the new block is the new immediate dominator of "Succ". Otherwise,
// the new block doesn't dominate anything.
if (IsNewIDom)
MDT->changeImmediateDominator(SucccDTNode, NewDTNode);
}
if (MachineLoopInfo *MLI = P->getAnalysisIfAvailable<MachineLoopInfo>())
if (MachineLoop *TIL = MLI->getLoopFor(this)) {
// If one or the other blocks were not in a loop, the new block is not
// either, and thus LI doesn't need to be updated.
if (MachineLoop *DestLoop = MLI->getLoopFor(Succ)) {
if (TIL == DestLoop) {
// Both in the same loop, the NMBB joins loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (TIL->contains(DestLoop)) {
// Edge from an outer loop to an inner loop. Add to the outer loop.
TIL->addBasicBlockToLoop(NMBB, MLI->getBase());
} else if (DestLoop->contains(TIL)) {
// Edge from an inner loop to an outer loop. Add to the outer loop.
DestLoop->addBasicBlockToLoop(NMBB, MLI->getBase());
} else {
// Edge from two loops with no containment relation. Because these
// are natural loops, we know that the destination block must be the
// header of its loop (adding a branch into a loop elsewhere would
// create an irreducible loop).
assert(DestLoop->getHeader() == Succ &&
"Should not create irreducible loops!");
if (MachineLoop *P = DestLoop->getParentLoop())
P->addBasicBlockToLoop(NMBB, MLI->getBase());
}
}
}
return NMBB;
}
