/// Collect all the loop live out values in the map that maps original live out
/// value to live out value in the cloned loop.
void LoopCloner::collectLoopLiveOutValues(
DenseMap<SILValue, SmallVector<SILValue, 8>> &LoopLiveOutValues) {
for (auto *Block : Loop->getBlocks()) {
// Look at block arguments.
for (auto *Arg : Block->getArguments()) {
for (auto *Op : Arg->getUses()) {
// Is this use outside the loop?
if (!Loop->contains(Op->getUser())) {
auto ArgumentValue = SILValue(Arg);
if (!LoopLiveOutValues.count(ArgumentValue))
LoopLiveOutValues[ArgumentValue].push_back(
getMappedValue(ArgumentValue));
}
}
}
// And the instructions.
for (auto &Inst : *Block) {
for (SILValue result : Inst.getResults()) {
for (auto *Op : result->getUses()) {
// Ignore uses inside the loop.
if (Loop->contains(Op->getUser()))
continue;
auto UsedValue = Op->get();
assert(UsedValue == result && "Instructions must match");
if (!LoopLiveOutValues.count(UsedValue))
LoopLiveOutValues[UsedValue].push_back(getMappedValue(result));
}
}
}
}
}
void StackColoring::expungeSlotMap(DenseMap<int, int> &SlotRemap,
unsigned NumSlots) {
// Expunge slot remap map.
for (unsigned i=0; i < NumSlots; ++i) {
// If we are remapping i
if (SlotRemap.count(i)) {
int Target = SlotRemap[i];
// As long as our target is mapped to something else, follow it.
while (SlotRemap.count(Target)) {
Target = SlotRemap[Target];
SlotRemap[i] = Target;
}
}
}
}
// Clone OldFunc into NewFunc, transforming the old arguments into references to
// ArgMap values.
//
void llvm::CloneFunctionInto(Function *NewFunc, const Function *OldFunc,
DenseMap<const Value*, Value*> &ValueMap,
std::vector<ReturnInst*> &Returns,
const char *NameSuffix, ClonedCodeInfo *CodeInfo) {
assert(NameSuffix && "NameSuffix cannot be null!");
#ifndef NDEBUG
for (Function::const_arg_iterator I = OldFunc->arg_begin(),
E = OldFunc->arg_end(); I != E; ++I)
assert(ValueMap.count(I) && "No mapping from source argument specified!");
#endif
// Clone any attributes.
if (NewFunc->arg_size() == OldFunc->arg_size())
NewFunc->copyAttributesFrom(OldFunc);
else {
//Some arguments were deleted with the ValueMap. Copy arguments one by one
for (Function::const_arg_iterator I = OldFunc->arg_begin(),
E = OldFunc->arg_end(); I != E; ++I)
if (Argument* Anew = dyn_cast<Argument>(ValueMap[I]))
Anew->addAttr( OldFunc->getAttributes()
.getParamAttributes(I->getArgNo() + 1));
NewFunc->setAttributes(NewFunc->getAttributes()
.addAttr(0, OldFunc->getAttributes()
.getRetAttributes()));
NewFunc->setAttributes(NewFunc->getAttributes()
.addAttr(~0, OldFunc->getAttributes()
.getFnAttributes()));
}
// Loop over all of the basic blocks in the function, cloning them as
// appropriate. Note that we save BE this way in order to handle cloning of
// recursive functions into themselves.
//
for (Function::const_iterator BI = OldFunc->begin(), BE = OldFunc->end();
BI != BE; ++BI) {
const BasicBlock &BB = *BI;
// Create a new basic block and copy instructions into it!
BasicBlock *CBB = CloneBasicBlock(&BB, ValueMap, NameSuffix, NewFunc,
CodeInfo);
ValueMap[&BB] = CBB; // Add basic block mapping.
if (ReturnInst *RI = dyn_cast<ReturnInst>(CBB->getTerminator()))
Returns.push_back(RI);
}
// Loop over all of the instructions in the function, fixing up operand
// references as we go. This uses ValueMap to do all the hard work.
//
for (Function::iterator BB = cast<BasicBlock>(ValueMap[OldFunc->begin()]),
BE = NewFunc->end(); BB != BE; ++BB)
// Loop over all instructions, fixing each one as we find it...
for (BasicBlock::iterator II = BB->begin(); II != BB->end(); ++II)
RemapInstruction(II, ValueMap);
}
bool RecurrenceDescriptor::isFirstOrderRecurrence(
PHINode *Phi, Loop *TheLoop,
DenseMap<Instruction *, Instruction *> &SinkAfter, DominatorTree *DT) {
// Ensure the phi node is in the loop header and has two incoming values.
if (Phi->getParent() != TheLoop->getHeader() ||
Phi->getNumIncomingValues() != 2)
return false;
// Ensure the loop has a preheader and a single latch block. The loop
// vectorizer will need the latch to set up the next iteration of the loop.
auto *Preheader = TheLoop->getLoopPreheader();
auto *Latch = TheLoop->getLoopLatch();
if (!Preheader || !Latch)
return false;
// Ensure the phi node's incoming blocks are the loop preheader and latch.
if (Phi->getBasicBlockIndex(Preheader) < 0 ||
Phi->getBasicBlockIndex(Latch) < 0)
return false;
// Get the previous value. The previous value comes from the latch edge while
// the initial value comes form the preheader edge.
auto *Previous = dyn_cast<Instruction>(Phi->getIncomingValueForBlock(Latch));
if (!Previous || !TheLoop->contains(Previous) || isa<PHINode>(Previous) ||
SinkAfter.count(Previous)) // Cannot rely on dominance due to motion.
return false;
// Ensure every user of the phi node is dominated by the previous value.
// The dominance requirement ensures the loop vectorizer will not need to
// vectorize the initial value prior to the first iteration of the loop.
// TODO: Consider extending this sinking to handle other kinds of instructions
// and expressions, beyond sinking a single cast past Previous.
if (Phi->hasOneUse()) {
auto *I = Phi->user_back();
if (I->isCast() && (I->getParent() == Phi->getParent()) && I->hasOneUse() &&
DT->dominates(Previous, I->user_back())) {
if (!DT->dominates(Previous, I)) // Otherwise we're good w/o sinking.
SinkAfter[I] = Previous;
return true;
}
}
for (User *U : Phi->users())
if (auto *I = dyn_cast<Instruction>(U)) {
if (!DT->dominates(Previous, I))
return false;
}
return true;
}
/// Collect all the loop live out values in the map that maps original live out
/// value to live out value in the cloned loop.
static void collectLoopLiveOutValues(
DenseMap<SILValue, SmallVector<SILValue, 8>> &LoopLiveOutValues,
SILLoop *Loop, DenseMap<SILValue, SILValue> &ClonedValues,
DenseMap<SILInstruction *, SILInstruction *> &ClonedInstructions) {
for (auto *Block : Loop->getBlocks()) {
// Look at block arguments.
for (auto *Arg : Block->getBBArgs()) {
for (auto *Op : Arg->getUses()) {
// Is this use outside the loop?
if (!Loop->contains(Op->getUser())) {
auto ArgumentValue = SILValue(Arg);
assert(ClonedValues.count(ArgumentValue) && "Unmapped Argument!");
if (!LoopLiveOutValues.count(ArgumentValue))
LoopLiveOutValues[ArgumentValue].push_back(
ClonedValues[ArgumentValue]);
}
}
}
// And the instructions.
for (auto &Inst : *Block) {
for (auto *Op : Inst.getUses()) {
// Is this use outside the loop.
if (!Loop->contains(Op->getUser())) {
auto UsedValue = Op->get();
assert(UsedValue.getDef() == &Inst && "Instructions must match");
assert(ClonedInstructions.count(&Inst) && "Unmapped instruction!");
if (!LoopLiveOutValues.count(UsedValue))
LoopLiveOutValues[UsedValue].push_back(SILValue(
ClonedInstructions[&Inst], UsedValue.getResultNumber()));
}
}
}
}
}
void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) {
// For each function there is a separate subsection
// which holds the PC to file:line table.
const MCSymbol *Fn = Asm->getSymbol(GV);
assert(Fn);
const FunctionInfo &FI = FnDebugInfo[GV];
if (FI.Instrs.empty())
return;
assert(FI.End && "Don't know where the function ends?");
StringRef GVName = GV->getName();
StringRef FuncName;
if (DISubprogram SP = getDISubprogram(GV))
FuncName = SP.getDisplayName();
// FIXME Clang currently sets DisplayName to "bar" for a C++
// "namespace_foo::bar" function, see PR21528. Luckily, dbghelp.dll is trying
// to demangle display names anyways, so let's just put a mangled name into
// the symbols subsection until Clang gives us what we need.
if (GVName.startswith("\01?"))
FuncName = GVName.substr(1);
// Emit a symbol subsection, required by VS2012+ to find function boundaries.
MCSymbol *SymbolsBegin = Asm->MMI->getContext().CreateTempSymbol(),
*SymbolsEnd = Asm->MMI->getContext().CreateTempSymbol();
Asm->OutStreamer.AddComment("Symbol subsection for " + Twine(FuncName));
Asm->EmitInt32(COFF::DEBUG_SYMBOL_SUBSECTION);
EmitLabelDiff(Asm->OutStreamer, SymbolsBegin, SymbolsEnd);
Asm->OutStreamer.EmitLabel(SymbolsBegin);
{
MCSymbol *ProcSegmentBegin = Asm->MMI->getContext().CreateTempSymbol(),
*ProcSegmentEnd = Asm->MMI->getContext().CreateTempSymbol();
EmitLabelDiff(Asm->OutStreamer, ProcSegmentBegin, ProcSegmentEnd, 2);
Asm->OutStreamer.EmitLabel(ProcSegmentBegin);
Asm->EmitInt16(COFF::DEBUG_SYMBOL_TYPE_PROC_START);
// Some bytes of this segment don't seem to be required for basic debugging,
// so just fill them with zeroes.
Asm->OutStreamer.EmitFill(12, 0);
// This is the important bit that tells the debugger where the function
// code is located and what's its size:
EmitLabelDiff(Asm->OutStreamer, Fn, FI.End);
Asm->OutStreamer.EmitFill(12, 0);
Asm->OutStreamer.EmitCOFFSecRel32(Fn);
Asm->OutStreamer.EmitCOFFSectionIndex(Fn);
Asm->EmitInt8(0);
// Emit the function display name as a null-terminated string.
Asm->OutStreamer.EmitBytes(FuncName);
Asm->EmitInt8(0);
Asm->OutStreamer.EmitLabel(ProcSegmentEnd);
// We're done with this function.
Asm->EmitInt16(0x0002);
Asm->EmitInt16(COFF::DEBUG_SYMBOL_TYPE_PROC_END);
}
Asm->OutStreamer.EmitLabel(SymbolsEnd);
// Every subsection must be aligned to a 4-byte boundary.
Asm->OutStreamer.EmitFill((-FuncName.size()) % 4, 0);
// PCs/Instructions are grouped into segments sharing the same filename.
// Pre-calculate the lengths (in instructions) of these segments and store
// them in a map for convenience. Each index in the map is the sequential
// number of the respective instruction that starts a new segment.
DenseMap<size_t, size_t> FilenameSegmentLengths;
size_t LastSegmentEnd = 0;
StringRef PrevFilename = InstrInfo[FI.Instrs[0]].Filename;
for (size_t J = 1, F = FI.Instrs.size(); J != F; ++J) {
if (PrevFilename == InstrInfo[FI.Instrs[J]].Filename)
continue;
FilenameSegmentLengths[LastSegmentEnd] = J - LastSegmentEnd;
LastSegmentEnd = J;
PrevFilename = InstrInfo[FI.Instrs[J]].Filename;
}
FilenameSegmentLengths[LastSegmentEnd] = FI.Instrs.size() - LastSegmentEnd;
// Emit a line table subsection, requred to do PC-to-file:line lookup.
Asm->OutStreamer.AddComment("Line table subsection for " + Twine(FuncName));
Asm->EmitInt32(COFF::DEBUG_LINE_TABLE_SUBSECTION);
MCSymbol *LineTableBegin = Asm->MMI->getContext().CreateTempSymbol(),
*LineTableEnd = Asm->MMI->getContext().CreateTempSymbol();
EmitLabelDiff(Asm->OutStreamer, LineTableBegin, LineTableEnd);
Asm->OutStreamer.EmitLabel(LineTableBegin);
// Identify the function this subsection is for.
Asm->OutStreamer.EmitCOFFSecRel32(Fn);
Asm->OutStreamer.EmitCOFFSectionIndex(Fn);
// Insert padding after a 16-bit section index.
Asm->EmitInt16(0);
// Length of the function's code, in bytes.
EmitLabelDiff(Asm->OutStreamer, Fn, FI.End);
// PC-to-linenumber lookup table:
MCSymbol *FileSegmentEnd = nullptr;
for (size_t J = 0, F = FI.Instrs.size(); J != F; ++J) {
MCSymbol *Instr = FI.Instrs[J];
assert(InstrInfo.count(Instr));
if (FilenameSegmentLengths.count(J)) {
// We came to a beginning of a new filename segment.
if (FileSegmentEnd)
Asm->OutStreamer.EmitLabel(FileSegmentEnd);
StringRef CurFilename = InstrInfo[FI.Instrs[J]].Filename;
assert(FileNameRegistry.Infos.count(CurFilename));
size_t IndexInStringTable =
FileNameRegistry.Infos[CurFilename].FilenameID;
// Each segment starts with the offset of the filename
// in the string table.
Asm->OutStreamer.AddComment(
"Segment for file '" + Twine(CurFilename) + "' begins");
MCSymbol *FileSegmentBegin = Asm->MMI->getContext().CreateTempSymbol();
Asm->OutStreamer.EmitLabel(FileSegmentBegin);
Asm->EmitInt32(8 * IndexInStringTable);
// Number of PC records in the lookup table.
size_t SegmentLength = FilenameSegmentLengths[J];
Asm->EmitInt32(SegmentLength);
// Full size of the segment for this filename, including the prev two
// records.
FileSegmentEnd = Asm->MMI->getContext().CreateTempSymbol();
EmitLabelDiff(Asm->OutStreamer, FileSegmentBegin, FileSegmentEnd);
}
// The first PC with the given linenumber and the linenumber itself.
EmitLabelDiff(Asm->OutStreamer, Fn, Instr);
Asm->EmitInt32(InstrInfo[Instr].LineNumber);
}
if (FileSegmentEnd)
Asm->OutStreamer.EmitLabel(FileSegmentEnd);
Asm->OutStreamer.EmitLabel(LineTableEnd);
}
bool PartialInlinerImpl::tryPartialInline(FunctionCloner &Cloner) {
int NonWeightedRcost;
int SizeCost;
if (Cloner.OutlinedFunc == nullptr)
return false;
std::tie(SizeCost, NonWeightedRcost) = computeOutliningCosts(Cloner);
auto RelativeToEntryFreq = getOutliningCallBBRelativeFreq(Cloner);
auto WeightedRcost = BlockFrequency(NonWeightedRcost) * RelativeToEntryFreq;
// The call sequence to the outlined function is larger than the original
// outlined region size, it does not increase the chances of inlining
// the function with outlining (The inliner uses the size increase to
// model the cost of inlining a callee).
if (!SkipCostAnalysis && Cloner.OutlinedRegionCost < SizeCost) {
OptimizationRemarkEmitter ORE(Cloner.OrigFunc);
DebugLoc DLoc;
BasicBlock *Block;
std::tie(DLoc, Block) = getOneDebugLoc(Cloner.ClonedFunc);
ORE.emit(OptimizationRemarkAnalysis(DEBUG_TYPE, "OutlineRegionTooSmall",
DLoc, Block)
<< ore::NV("Function", Cloner.OrigFunc)
<< " not partially inlined into callers (Original Size = "
<< ore::NV("OutlinedRegionOriginalSize", Cloner.OutlinedRegionCost)
<< ", Size of call sequence to outlined function = "
<< ore::NV("NewSize", SizeCost) << ")");
return false;
}
assert(Cloner.OrigFunc->user_begin() == Cloner.OrigFunc->user_end() &&
"F's users should all be replaced!");
std::vector<User *> Users(Cloner.ClonedFunc->user_begin(),
Cloner.ClonedFunc->user_end());
DenseMap<User *, uint64_t> CallSiteToProfCountMap;
if (Cloner.OrigFunc->getEntryCount())
computeCallsiteToProfCountMap(Cloner.ClonedFunc, CallSiteToProfCountMap);
auto CalleeEntryCount = Cloner.OrigFunc->getEntryCount();
uint64_t CalleeEntryCountV = (CalleeEntryCount ? *CalleeEntryCount : 0);
bool AnyInline = false;
for (User *User : Users) {
CallSite CS = getCallSite(User);
if (IsLimitReached())
continue;
OptimizationRemarkEmitter ORE(CS.getCaller());
if (!shouldPartialInline(CS, Cloner, WeightedRcost, ORE))
continue;
ORE.emit(
OptimizationRemark(DEBUG_TYPE, "PartiallyInlined", CS.getInstruction())
<< ore::NV("Callee", Cloner.OrigFunc) << " partially inlined into "
<< ore::NV("Caller", CS.getCaller()));
InlineFunctionInfo IFI(nullptr, GetAssumptionCache, PSI);
InlineFunction(CS, IFI);
// Now update the entry count:
if (CalleeEntryCountV && CallSiteToProfCountMap.count(User)) {
uint64_t CallSiteCount = CallSiteToProfCountMap[User];
CalleeEntryCountV -= std::min(CalleeEntryCountV, CallSiteCount);
}
AnyInline = true;
NumPartialInlining++;
// Update the stats
NumPartialInlined++;
}
if (AnyInline) {
Cloner.IsFunctionInlined = true;
if (CalleeEntryCount)
Cloner.OrigFunc->setEntryCount(CalleeEntryCountV);
}
return AnyInline;
}
void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
unsigned FixedInstr = 0;
unsigned FixedMemOp = 0;
unsigned FixedDbg = 0;
MachineModuleInfo *MMI = &MF->getMMI();
// Remap debug information that refers to stack slots.
for (auto &VI : MMI->getVariableDbgInfo()) {
if (!VI.Var)
continue;
if (SlotRemap.count(VI.Slot)) {
DEBUG(dbgs() << "Remapping debug info for ["
<< cast<DILocalVariable>(VI.Var)->getName() << "].\n");
VI.Slot = SlotRemap[VI.Slot];
FixedDbg++;
}
}
// Keep a list of *allocas* which need to be remapped.
DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
for (const std::pair<int, int> &SI : SlotRemap) {
const AllocaInst *From = MFI->getObjectAllocation(SI.first);
const AllocaInst *To = MFI->getObjectAllocation(SI.second);
assert(To && From && "Invalid allocation object");
Allocas[From] = To;
// AA might be used later for instruction scheduling, and we need it to be
// able to deduce the correct aliasing releationships between pointers
// derived from the alloca being remapped and the target of that remapping.
// The only safe way, without directly informing AA about the remapping
// somehow, is to directly update the IR to reflect the change being made
// here.
Instruction *Inst = const_cast<AllocaInst *>(To);
if (From->getType() != To->getType()) {
BitCastInst *Cast = new BitCastInst(Inst, From->getType());
Cast->insertAfter(Inst);
Inst = Cast;
}
// Allow the stack protector to adjust its value map to account for the
// upcoming replacement.
SP->adjustForColoring(From, To);
// The new alloca might not be valid in a llvm.dbg.declare for this
// variable, so undef out the use to make the verifier happy.
AllocaInst *FromAI = const_cast<AllocaInst *>(From);
if (FromAI->isUsedByMetadata())
ValueAsMetadata::handleRAUW(FromAI, UndefValue::get(FromAI->getType()));
for (auto &Use : FromAI->uses()) {
if (BitCastInst *BCI = dyn_cast<BitCastInst>(Use.get()))
if (BCI->isUsedByMetadata())
ValueAsMetadata::handleRAUW(BCI, UndefValue::get(BCI->getType()));
}
// Note that this will not replace uses in MMOs (which we'll update below),
// or anywhere else (which is why we won't delete the original
// instruction).
FromAI->replaceAllUsesWith(Inst);
}
// Remap all instructions to the new stack slots.
for (MachineBasicBlock &BB : *MF)
for (MachineInstr &I : BB) {
// Skip lifetime markers. We'll remove them soon.
if (I.getOpcode() == TargetOpcode::LIFETIME_START ||
I.getOpcode() == TargetOpcode::LIFETIME_END)
continue;
// Update the MachineMemOperand to use the new alloca.
for (MachineMemOperand *MMO : I.memoperands()) {
// FIXME: In order to enable the use of TBAA when using AA in CodeGen,
// we'll also need to update the TBAA nodes in MMOs with values
// derived from the merged allocas. When doing this, we'll need to use
// the same variant of GetUnderlyingObjects that is used by the
// instruction scheduler (that can look through ptrtoint/inttoptr
// pairs).
// We've replaced IR-level uses of the remapped allocas, so we only
// need to replace direct uses here.
const AllocaInst *AI = dyn_cast_or_null<AllocaInst>(MMO->getValue());
if (!AI)
continue;
if (!Allocas.count(AI))
continue;
MMO->setValue(Allocas[AI]);
FixedMemOp++;
}
// Update all of the machine instruction operands.
for (MachineOperand &MO : I.operands()) {
if (!MO.isFI())
continue;
int FromSlot = MO.getIndex();
// Don't touch arguments.
if (FromSlot<0)
continue;
// Only look at mapped slots.
if (!SlotRemap.count(FromSlot))
continue;
// In a debug build, check that the instruction that we are modifying is
// inside the expected live range. If the instruction is not inside
// the calculated range then it means that the alloca usage moved
// outside of the lifetime markers, or that the user has a bug.
// NOTE: Alloca address calculations which happen outside the lifetime
// zone are are okay, despite the fact that we don't have a good way
// for validating all of the usages of the calculation.
#ifndef NDEBUG
bool TouchesMemory = I.mayLoad() || I.mayStore();
// If we *don't* protect the user from escaped allocas, don't bother
// validating the instructions.
if (!I.isDebugValue() && TouchesMemory && ProtectFromEscapedAllocas) {
SlotIndex Index = Indexes->getInstructionIndex(I);
const LiveInterval *Interval = &*Intervals[FromSlot];
assert(Interval->find(Index) != Interval->end() &&
"Found instruction usage outside of live range.");
}
#endif
// Fix the machine instructions.
int ToSlot = SlotRemap[FromSlot];
MO.setIndex(ToSlot);
FixedInstr++;
}
}
// Update the location of C++ catch objects for the MSVC personality routine.
if (WinEHFuncInfo *EHInfo = MF->getWinEHFuncInfo())
for (WinEHTryBlockMapEntry &TBME : EHInfo->TryBlockMap)
for (WinEHHandlerType &H : TBME.HandlerArray)
if (H.CatchObj.FrameIndex != INT_MAX &&
SlotRemap.count(H.CatchObj.FrameIndex))
H.CatchObj.FrameIndex = SlotRemap[H.CatchObj.FrameIndex];
DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
}
static void EmitDebugLocInfo(ObjectWriter *OW, const char *FunctionName,
int FunctionSize, int NumLocInfos,
DebugLocInfo LocInfos[]) {
assert(OW && "ObjWriter is null");
auto *AsmPrinter = &OW->getAsmPrinter();
auto &OST = static_cast<MCObjectStreamer &>(*AsmPrinter->OutStreamer);
MCContext &OutContext = OST.getContext();
const MCObjectFileInfo *MOFI = OutContext.getObjectFileInfo();
// TODO: Should convert this for non-Windows.
if (MOFI->getObjectFileType() != MOFI->IsCOFF) {
return;
}
MCSection *Section = MOFI->getCOFFDebugSymbolsSection();
OST.SwitchSection(Section);
MCSymbol *Fn = OutContext.getOrCreateSymbol(Twine(FunctionName));
// Emit a symbol subsection, required by VS2012+ to find function boundaries.
MCSymbol *SymbolsBegin = OutContext.createTempSymbol(),
*SymbolsEnd = OutContext.createTempSymbol();
OST.EmitIntValue(unsigned(ModuleSubstreamKind::Symbols), 4);
EmitLabelDiff(OST, SymbolsBegin, SymbolsEnd);
OST.EmitLabel(SymbolsBegin);
{
MCSymbol *ProcSegmentBegin = OutContext.createTempSymbol(),
*ProcSegmentEnd = OutContext.createTempSymbol();
EmitLabelDiff(OST, ProcSegmentBegin, ProcSegmentEnd, 2);
OST.EmitLabel(ProcSegmentBegin);
OST.EmitIntValue(unsigned(SymbolRecordKind::S_GPROC32_ID), 2);
// Some bytes of this segment don't seem to be required for basic debugging,
// so just fill them with zeroes.
OST.EmitFill(12, 0);
// This is the important bit that tells the debugger where the function
// code is located and what's its size:
OST.EmitIntValue(FunctionSize, 4);
OST.EmitFill(12, 0);
OST.EmitCOFFSecRel32(Fn);
OST.EmitCOFFSectionIndex(Fn);
OST.EmitIntValue(0, 1);
// Emit the function display name as a null-terminated string.
OST.EmitBytes(FunctionName);
OST.EmitIntValue(0, 1);
OST.EmitLabel(ProcSegmentEnd);
// We're done with this function.
OST.EmitIntValue(0x0002, 2);
OST.EmitIntValue(unsigned(SymbolRecordKind::S_PROC_ID_END), 2);
}
OST.EmitLabel(SymbolsEnd);
// Every subsection must be aligned to a 4-byte boundary.
OST.EmitValueToAlignment(4);
// PCs/Instructions are grouped into segments sharing the same filename.
// Pre-calculate the lengths (in instructions) of these segments and store
// them in a map for convenience. Each index in the map is the sequential
// number of the respective instruction that starts a new segment.
DenseMap<size_t, size_t> FilenameSegmentLengths;
size_t LastSegmentEnd = 0;
int PrevFileId = LocInfos[0].FileId;
for (int J = 1; J < NumLocInfos; ++J) {
if (PrevFileId == LocInfos[J].FileId)
continue;
FilenameSegmentLengths[LastSegmentEnd] = J - LastSegmentEnd;
LastSegmentEnd = J;
PrevFileId = LocInfos[J].FileId;
}
FilenameSegmentLengths[LastSegmentEnd] = NumLocInfos - LastSegmentEnd;
// Emit a line table subsection, required to do PC-to-file:line lookup.
OST.EmitIntValue(unsigned(ModuleSubstreamKind::Lines), 4);
MCSymbol *LineTableBegin = OutContext.createTempSymbol(),
*LineTableEnd = OutContext.createTempSymbol();
EmitLabelDiff(OST, LineTableBegin, LineTableEnd);
OST.EmitLabel(LineTableBegin);
// Identify the function this subsection is for.
OST.EmitCOFFSecRel32(Fn);
OST.EmitCOFFSectionIndex(Fn);
// Insert flags after a 16-bit section index.
OST.EmitIntValue(codeview::LineFlags::HaveColumns, 2);
// Length of the function's code, in bytes.
OST.EmitIntValue(FunctionSize, 4);
// PC-to-linenumber lookup table:
MCSymbol *FileSegmentEnd = nullptr;
// The start of the last segment:
size_t LastSegmentStart = 0;
auto FinishPreviousChunk = [&] {
if (!FileSegmentEnd)
return;
for (size_t ColSegI = LastSegmentStart,
ColSegEnd = ColSegI + FilenameSegmentLengths[LastSegmentStart];
ColSegI != ColSegEnd; ++ColSegI) {
unsigned ColumnNumber = LocInfos[ColSegI].ColNumber;
OST.EmitIntValue(ColumnNumber, 2); // Start column
OST.EmitIntValue(ColumnNumber, 2); // End column
}
OST.EmitLabel(FileSegmentEnd);
};
for (int J = 0; J < NumLocInfos; ++J) {
DebugLocInfo Loc = LocInfos[J];
if (FilenameSegmentLengths.count(J)) {
// We came to a beginning of a new filename segment.
FinishPreviousChunk();
MCSymbol *FileSegmentBegin = OutContext.createTempSymbol();
OST.EmitLabel(FileSegmentBegin);
// Each entry in string index table is 8 byte.
OST.EmitIntValue(8 * Loc.FileId, 4);
// Number of PC records in the lookup table.
size_t SegmentLength = FilenameSegmentLengths[J];
OST.EmitIntValue(SegmentLength, 4);
// Full size of the segment for this filename, including the prev two
// records.
FileSegmentEnd = OutContext.createTempSymbol();
EmitLabelDiff(OST, FileSegmentBegin, FileSegmentEnd);
LastSegmentStart = J;
}
// The first PC with the given linenumber and the linenumber itself.
OST.EmitIntValue(Loc.NativeOffset, 4);
OST.EmitIntValue(Loc.LineNumber, 4);
}
FinishPreviousChunk();
OST.EmitLabel(LineTableEnd);
}
bool BDCE::runOnFunction(Function& F) {
if (skipOptnoneFunction(F))
return false;
AC = &getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
DT = &getAnalysis<DominatorTreeWrapperPass>().getDomTree();
DenseMap<Instruction *, APInt> AliveBits;
SmallVector<Instruction*, 128> Worklist;
// The set of visited instructions (non-integer-typed only).
SmallPtrSet<Instruction*, 128> Visited;
// Collect the set of "root" instructions that are known live.
for (Instruction &I : inst_range(F)) {
if (!isAlwaysLive(&I))
continue;
DEBUG(dbgs() << "BDCE: Root: " << I << "\n");
// For integer-valued instructions, set up an initial empty set of alive
// bits and add the instruction to the work list. For other instructions
// add their operands to the work list (for integer values operands, mark
// all bits as live).
if (IntegerType *IT = dyn_cast<IntegerType>(I.getType())) {
if (!AliveBits.count(&I)) {
AliveBits[&I] = APInt(IT->getBitWidth(), 0);
Worklist.push_back(&I);
}
continue;
}
// Non-integer-typed instructions...
for (Use &OI : I.operands()) {
if (Instruction *J = dyn_cast<Instruction>(OI)) {
if (IntegerType *IT = dyn_cast<IntegerType>(J->getType()))
AliveBits[J] = APInt::getAllOnesValue(IT->getBitWidth());
Worklist.push_back(J);
}
}
// To save memory, we don't add I to the Visited set here. Instead, we
// check isAlwaysLive on every instruction when searching for dead
// instructions later (we need to check isAlwaysLive for the
// integer-typed instructions anyway).
}
// Propagate liveness backwards to operands.
while (!Worklist.empty()) {
Instruction *UserI = Worklist.pop_back_val();
DEBUG(dbgs() << "BDCE: Visiting: " << *UserI);
APInt AOut;
if (UserI->getType()->isIntegerTy()) {
AOut = AliveBits[UserI];
DEBUG(dbgs() << " Alive Out: " << AOut);
}
DEBUG(dbgs() << "\n");
if (!UserI->getType()->isIntegerTy())
Visited.insert(UserI);
APInt KnownZero, KnownOne, KnownZero2, KnownOne2;
// Compute the set of alive bits for each operand. These are anded into the
// existing set, if any, and if that changes the set of alive bits, the
// operand is added to the work-list.
for (Use &OI : UserI->operands()) {
if (Instruction *I = dyn_cast<Instruction>(OI)) {
if (IntegerType *IT = dyn_cast<IntegerType>(I->getType())) {
unsigned BitWidth = IT->getBitWidth();
APInt AB = APInt::getAllOnesValue(BitWidth);
if (UserI->getType()->isIntegerTy() && !AOut &&
!isAlwaysLive(UserI)) {
AB = APInt(BitWidth, 0);
} else {
// If all bits of the output are dead, then all bits of the input
// Bits of each operand that are used to compute alive bits of the
// output are alive, all others are dead.
determineLiveOperandBits(UserI, I, OI.getOperandNo(), AOut, AB,
KnownZero, KnownOne,
KnownZero2, KnownOne2);
}
// If we've added to the set of alive bits (or the operand has not
// been previously visited), then re-queue the operand to be visited
// again.
APInt ABPrev(BitWidth, 0);
auto ABI = AliveBits.find(I);
if (ABI != AliveBits.end())
ABPrev = ABI->second;
APInt ABNew = AB | ABPrev;
if (ABNew != ABPrev || ABI == AliveBits.end()) {
AliveBits[I] = std::move(ABNew);
Worklist.push_back(I);
}
} else if (!Visited.count(I)) {
Worklist.push_back(I);
}
}
}
}
bool Changed = false;
// The inverse of the live set is the dead set. These are those instructions
// which have no side effects and do not influence the control flow or return
// value of the function, and may therefore be deleted safely.
// NOTE: We reuse the Worklist vector here for memory efficiency.
for (Instruction &I : inst_range(F)) {
// For live instructions that have all dead bits, first make them dead by
// replacing all uses with something else. Then, if they don't need to
// remain live (because they have side effects, etc.) we can remove them.
if (I.getType()->isIntegerTy()) {
auto ABI = AliveBits.find(&I);
if (ABI != AliveBits.end()) {
if (ABI->second.getBoolValue())
continue;
DEBUG(dbgs() << "BDCE: Trivializing: " << I << " (all bits dead)\n");
// FIXME: In theory we could substitute undef here instead of zero.
// This should be reconsidered once we settle on the semantics of
// undef, poison, etc.
Value *Zero = ConstantInt::get(I.getType(), 0);
++NumSimplified;
I.replaceAllUsesWith(Zero);
Changed = true;
}
} else if (Visited.count(&I)) {
continue;
}
if (isAlwaysLive(&I))
continue;
Worklist.push_back(&I);
I.dropAllReferences();
Changed = true;
}
for (Instruction *&I : Worklist) {
++NumRemoved;
I->eraseFromParent();
}
return Changed;
}
static bool convertFunction(Function *Func) {
bool Changed = false;
IntegerType *I32 = Type::getInt32Ty(Func->getContext());
// Skip zero in case programs treat a null pointer as special.
uint32_t NextNum = 1;
DenseMap<BasicBlock *, ConstantInt *> LabelNums;
BasicBlock *DefaultBB = NULL;
// Replace each indirectbr with a switch.
//
// If there are multiple indirectbr instructions in the function,
// this could be expensive. While an indirectbr is usually
// converted to O(1) machine instructions, the switch we generate
// here will be O(n) in the number of target labels.
//
// However, Clang usually generates just a single indirectbr per
// function anyway when compiling C computed gotos.
//
// We could try to generate one switch to handle all the indirectbr
// instructions in the function, but that would be complicated to
// implement given that variables that are live at one indirectbr
// might not be live at others.
for (llvm::Function::iterator BB = Func->begin(), E = Func->end();
BB != E; ++BB) {
if (IndirectBrInst *Br = dyn_cast<IndirectBrInst>(BB->getTerminator())) {
Changed = true;
if (!DefaultBB) {
DefaultBB = BasicBlock::Create(Func->getContext(),
"indirectbr_default", Func);
new UnreachableInst(Func->getContext(), DefaultBB);
}
// An indirectbr can list the same target block multiple times.
// Keep track of the basic blocks we've handled to avoid adding
// the same case multiple times.
DenseSet<BasicBlock *> BlocksSeen;
Value *Cast = new PtrToIntInst(Br->getAddress(), I32,
"indirectbr_cast", Br);
unsigned Count = Br->getNumSuccessors();
SwitchInst *Switch = SwitchInst::Create(Cast, DefaultBB, Count, Br);
for (unsigned I = 0; I < Count; ++I) {
BasicBlock *Dest = Br->getSuccessor(I);
if (!BlocksSeen.insert(Dest).second) {
// Remove duplicated entries from phi nodes.
for (BasicBlock::iterator Inst = Dest->begin(); ; ++Inst) {
PHINode *Phi = dyn_cast<PHINode>(Inst);
if (!Phi)
break;
Phi->removeIncomingValue(Br->getParent());
}
continue;
}
ConstantInt *Val;
if (LabelNums.count(Dest) == 0) {
Val = ConstantInt::get(I32, NextNum++);
LabelNums[Dest] = Val;
BlockAddress *BA = BlockAddress::get(Func, Dest);
Value *ValAsPtr = ConstantExpr::getIntToPtr(Val, BA->getType());
BA->replaceAllUsesWith(ValAsPtr);
BA->destroyConstant();
} else {
Val = LabelNums[Dest];
}
Switch->addCase(Val, Br->getSuccessor(I));
}
Br->eraseFromParent();
}
}
// If there are any blockaddresses that are never used by an
// indirectbr, replace them with dummy values.
SmallVector<Value *, 20> Users(Func->user_begin(), Func->user_end());
for (auto U : Users) {
if (BlockAddress *BA = dyn_cast<BlockAddress>(U)) {
Changed = true;
Value *DummyVal = ConstantExpr::getIntToPtr(ConstantInt::get(I32, ~0L),
BA->getType());
BA->replaceAllUsesWith(DummyVal);
BA->destroyConstant();
}
}
return Changed;
}
/// gold informs us that all symbols have been read. At this point, we use
/// get_symbols to see if any of our definitions have been overridden by a
/// native object file. Then, perform optimization and codegen.
static ld_plugin_status allSymbolsReadHook() {
if (Modules.empty())
return LDPS_OK;
if (unsigned NumOpts = options::extra.size())
cl::ParseCommandLineOptions(NumOpts, &options::extra[0]);
// Map to own RAII objects that manage the file opening and releasing
// interfaces with gold. This is needed only for ThinLTO mode, since
// unlike regular LTO, where addModule will result in the opened file
// being merged into a new combined module, we need to keep these files open
// through Lto->run().
DenseMap<void *, std::unique_ptr<PluginInputFile>> HandleToInputFile;
std::unique_ptr<LTO> Lto = createLTO();
std::string OldPrefix, NewPrefix;
if (options::thinlto_index_only)
getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
for (claimed_file &F : Modules) {
if (options::thinlto && !HandleToInputFile.count(F.leader_handle))
HandleToInputFile.insert(std::make_pair(
F.leader_handle, llvm::make_unique<PluginInputFile>(F.handle)));
const void *View = getSymbolsAndView(F);
if (!View) {
if (options::thinlto_index_only)
// Write empty output files that may be expected by the distributed
// build system.
writeEmptyDistributedBuildOutputs(F.name, OldPrefix, NewPrefix);
continue;
}
addModule(*Lto, F, View);
}
SmallString<128> Filename;
// Note that getOutputFileName will append a unique ID for each task
if (!options::obj_path.empty())
Filename = options::obj_path;
else if (options::TheOutputType == options::OT_SAVE_TEMPS)
Filename = output_name + ".o";
bool SaveTemps = !Filename.empty();
MaxTasks = Lto->getMaxTasks();
std::vector<uintptr_t> IsTemporary(MaxTasks);
std::vector<SmallString<128>> Filenames(MaxTasks);
auto AddStream =
[&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
IsTemporary[Task] = !SaveTemps;
getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps, Filenames[Task],
MaxTasks > 1 ? Task : -1);
int FD;
std::error_code EC =
sys::fs::openFileForWrite(Filenames[Task], FD, sys::fs::F_None);
if (EC)
message(LDPL_FATAL, "Could not open file: %s", EC.message().c_str());
return llvm::make_unique<lto::NativeObjectStream>(
llvm::make_unique<llvm::raw_fd_ostream>(FD, true));
};
auto AddFile = [&](size_t Task, StringRef Path) { Filenames[Task] = Path; };
NativeObjectCache Cache;
if (!options::cache_dir.empty())
Cache = localCache(options::cache_dir, AddFile);
check(Lto->run(AddStream, Cache));
if (options::TheOutputType == options::OT_DISABLE ||
options::TheOutputType == options::OT_BC_ONLY)
return LDPS_OK;
if (options::thinlto_index_only) {
cleanup_hook();
exit(0);
}
for (unsigned I = 0; I != MaxTasks; ++I)
if (!Filenames[I].empty())
recordFile(Filenames[I].str(), IsTemporary[I]);
if (!options::extra_library_path.empty() &&
set_extra_library_path(options::extra_library_path.c_str()) != LDPS_OK)
message(LDPL_FATAL, "Unable to set the extra library path.");
return LDPS_OK;
}
/// gold informs us that all symbols have been read. At this point, we use
/// get_symbols to see if any of our definitions have been overridden by a
/// native object file. Then, perform optimization and codegen.
static ld_plugin_status allSymbolsReadHook() {
if (Modules.empty())
return LDPS_OK;
if (unsigned NumOpts = options::extra.size())
cl::ParseCommandLineOptions(NumOpts, &options::extra[0]);
// Map to own RAII objects that manage the file opening and releasing
// interfaces with gold. This is needed only for ThinLTO mode, since
// unlike regular LTO, where addModule will result in the opened file
// being merged into a new combined module, we need to keep these files open
// through Lto->run().
DenseMap<void *, std::unique_ptr<PluginInputFile>> HandleToInputFile;
std::unique_ptr<LTO> Lto = createLTO();
std::string OldPrefix, NewPrefix;
if (options::thinlto_index_only)
getThinLTOOldAndNewPrefix(OldPrefix, NewPrefix);
std::string OldSuffix, NewSuffix;
getThinLTOOldAndNewSuffix(OldSuffix, NewSuffix);
// Set for owning string objects used as buffer identifiers.
StringSet<> ObjectFilenames;
for (claimed_file &F : Modules) {
if (options::thinlto && !HandleToInputFile.count(F.leader_handle))
HandleToInputFile.insert(std::make_pair(
F.leader_handle, llvm::make_unique<PluginInputFile>(F.handle)));
const void *View = getSymbolsAndView(F);
// In case we are thin linking with a minimized bitcode file, ensure
// the module paths encoded in the index reflect where the backends
// will locate the full bitcode files for compiling/importing.
std::string Identifier =
getThinLTOObjectFileName(F.name, OldSuffix, NewSuffix);
auto ObjFilename = ObjectFilenames.insert(Identifier);
assert(ObjFilename.second);
if (!View) {
if (options::thinlto_index_only)
// Write empty output files that may be expected by the distributed
// build system.
writeEmptyDistributedBuildOutputs(Identifier, OldPrefix, NewPrefix);
continue;
}
addModule(*Lto, F, View, ObjFilename.first->first());
}
SmallString<128> Filename;
// Note that getOutputFileName will append a unique ID for each task
if (!options::obj_path.empty())
Filename = options::obj_path;
else if (options::TheOutputType == options::OT_SAVE_TEMPS)
Filename = output_name + ".o";
bool SaveTemps = !Filename.empty();
size_t MaxTasks = Lto->getMaxTasks();
std::vector<uintptr_t> IsTemporary(MaxTasks);
std::vector<SmallString<128>> Filenames(MaxTasks);
auto AddStream =
[&](size_t Task) -> std::unique_ptr<lto::NativeObjectStream> {
IsTemporary[Task] = !SaveTemps;
getOutputFileName(Filename, /*TempOutFile=*/!SaveTemps, Filenames[Task],
Task);
int FD;
std::error_code EC =
sys::fs::openFileForWrite(Filenames[Task], FD, sys::fs::F_None);
if (EC)
message(LDPL_FATAL, "Could not open file %s: %s", Filenames[Task].c_str(),
EC.message().c_str());
return llvm::make_unique<lto::NativeObjectStream>(
llvm::make_unique<llvm::raw_fd_ostream>(FD, true));
};
auto AddBuffer = [&](size_t Task, std::unique_ptr<MemoryBuffer> MB) {
// Note that this requires that the memory buffers provided to AddBuffer are
// backed by a file.
Filenames[Task] = MB->getBufferIdentifier();
};
NativeObjectCache Cache;
if (!options::cache_dir.empty())
Cache = check(localCache(options::cache_dir, AddBuffer));
check(Lto->run(AddStream, Cache));
if (options::TheOutputType == options::OT_DISABLE ||
options::TheOutputType == options::OT_BC_ONLY)
return LDPS_OK;
if (options::thinlto_index_only) {
if (llvm::AreStatisticsEnabled())
llvm::PrintStatistics();
cleanup_hook();
exit(0);
}
for (unsigned I = 0; I != MaxTasks; ++I)
if (!Filenames[I].empty())
recordFile(Filenames[I].str(), IsTemporary[I]);
if (!options::extra_library_path.empty() &&
set_extra_library_path(options::extra_library_path.c_str()) != LDPS_OK)
message(LDPL_FATAL, "Unable to set the extra library path.");
return LDPS_OK;
}
void WinCodeViewLineTables::emitDebugInfoForFunction(const Function *GV) {
// For each function there is a separate subsection
// which holds the PC to file:line table.
const MCSymbol *Fn = Asm->getSymbol(GV);
assert(Fn);
const FunctionInfo &FI = FnDebugInfo[GV];
if (FI.Instrs.empty())
return;
assert(FI.End && "Don't know where the function ends?");
// PCs/Instructions are grouped into segments sharing the same filename.
// Pre-calculate the lengths (in instructions) of these segments and store
// them in a map for convenience. Each index in the map is the sequential
// number of the respective instruction that starts a new segment.
DenseMap<size_t, size_t> FilenameSegmentLengths;
size_t LastSegmentEnd = 0;
StringRef PrevFilename = InstrInfo[FI.Instrs[0]].Filename;
for (size_t J = 1, F = FI.Instrs.size(); J != F; ++J) {
if (PrevFilename == InstrInfo[FI.Instrs[J]].Filename)
continue;
FilenameSegmentLengths[LastSegmentEnd] = J - LastSegmentEnd;
LastSegmentEnd = J;
PrevFilename = InstrInfo[FI.Instrs[J]].Filename;
}
FilenameSegmentLengths[LastSegmentEnd] = FI.Instrs.size() - LastSegmentEnd;
// Emit the control code of the subsection followed by the payload size.
Asm->OutStreamer.AddComment(
"Linetable subsection for " + Twine(Fn->getName()));
Asm->EmitInt32(COFF::DEBUG_LINE_TABLE_SUBSECTION);
MCSymbol *SubsectionBegin = Asm->MMI->getContext().CreateTempSymbol(),
*SubsectionEnd = Asm->MMI->getContext().CreateTempSymbol();
EmitLabelDiff(Asm->OutStreamer, SubsectionBegin, SubsectionEnd);
Asm->OutStreamer.EmitLabel(SubsectionBegin);
// Identify the function this subsection is for.
Asm->OutStreamer.EmitCOFFSecRel32(Fn);
Asm->OutStreamer.EmitCOFFSectionIndex(Fn);
// Length of the function's code, in bytes.
EmitLabelDiff(Asm->OutStreamer, Fn, FI.End);
// PC-to-linenumber lookup table:
MCSymbol *FileSegmentEnd = nullptr;
for (size_t J = 0, F = FI.Instrs.size(); J != F; ++J) {
MCSymbol *Instr = FI.Instrs[J];
assert(InstrInfo.count(Instr));
if (FilenameSegmentLengths.count(J)) {
// We came to a beginning of a new filename segment.
if (FileSegmentEnd)
Asm->OutStreamer.EmitLabel(FileSegmentEnd);
StringRef CurFilename = InstrInfo[FI.Instrs[J]].Filename;
assert(FileNameRegistry.Infos.count(CurFilename));
size_t IndexInStringTable =
FileNameRegistry.Infos[CurFilename].FilenameID;
// Each segment starts with the offset of the filename
// in the string table.
Asm->OutStreamer.AddComment(
"Segment for file '" + Twine(CurFilename) + "' begins");
MCSymbol *FileSegmentBegin = Asm->MMI->getContext().CreateTempSymbol();
Asm->OutStreamer.EmitLabel(FileSegmentBegin);
Asm->EmitInt32(8 * IndexInStringTable);
// Number of PC records in the lookup table.
size_t SegmentLength = FilenameSegmentLengths[J];
Asm->EmitInt32(SegmentLength);
// Full size of the segment for this filename, including the prev two
// records.
FileSegmentEnd = Asm->MMI->getContext().CreateTempSymbol();
EmitLabelDiff(Asm->OutStreamer, FileSegmentBegin, FileSegmentEnd);
}
// The first PC with the given linenumber and the linenumber itself.
EmitLabelDiff(Asm->OutStreamer, Fn, Instr);
Asm->EmitInt32(InstrInfo[Instr].LineNumber);
}
if (FileSegmentEnd)
Asm->OutStreamer.EmitLabel(FileSegmentEnd);
Asm->OutStreamer.EmitLabel(SubsectionEnd);
}
/*
* Insert sigmas for the value V;
* When sigmas are created, the creation of phis may happen too.
*/
void vSSA::insertSigmas(TerminatorInst *TI, Value *V)
{
DEBUG(dbgs() << "VSSA: Inserting sigma for " << *TI << " and " << *V << "\n");
static DenseMap<TerminatorInst*, SetVector<Value*>*> TreatedTI;
if (!TreatedTI.count(TI))
TreatedTI[TI] = new SetVector<Value*>();
if (TreatedTI[TI]->count(V))
return;
TreatedTI[TI]->insert(V);
// Basic Block of the Terminator Instruction
//errs() << "Creating sigma for: " << *V << " at " << *TI << "\n";
DEBUG(dbgs() << "VSSA: Really inserting sigma for " << *TI << " and " << *V << "\n");
BasicBlock *BB = TI->getParent();
// Vector that contains all vSSA_PHI nodes created in the process of creating sigmas for V
SmallVector<PHINode*, 25> vssaphi_created;
bool firstSigma = true;
// Iterate over all successors of BB, checking if a sigma is needed
for (unsigned i = 0, e = TI->getNumSuccessors(); i < e; ++i) {
// Next Basic Block
BasicBlock *BB_next = TI->getSuccessor(i);
// If the successor is not BB itself and BB dominates the successor
if (BB_next != BB && BB_next->getSinglePredecessor() != NULL && dominateOrHasInFrontier(BB, BB_next, V)) {
// Create the sigma function (but before, verify if there is already an identical sigma function)
DEBUG(dbgs() << "Succ: " << BB_next->getName() << "\n");
if (verifySigmaExistance(V, BB_next, BB)) {
DEBUG(dbgs() << "Sigma exists!\n");
continue;
}
DEBUG(dbgs() << "Sigma does not exists!\n");
PHINode *sigma = PHINode::Create(V->getType(), 1, Twine(vSSA_SIG), &(BB_next->front()));
if (ValueMap.count(V))
ValueMap[sigma] = ValueMap[V];
else
ValueMap[sigma] = V;
DomTreeNode *Node = DT_->getNode(BB_next);
Value *Base = ValueMap[sigma];
if (!SigmaMap.count(Base))
SigmaMap[Base] = new BlockMapTy();
for (po_iterator<DomTreeNode*> It = po_begin(Node), E = po_end(Node); It != E; ++It) {
DEBUG(dbgs() << "SigmaMap: " << *Base << " ==> " << It->getBlock()->getName() << " = " << *sigma << "\n");
(*SigmaMap[Base])[It->getBlock()] = sigma;
}
sigma->addIncoming(V, BB);
//errs() << "VSSA: Created sigma node " << *sigma << "\n";
++numsigmas;
// Rename uses of V to sigma(V)
renameUsesToSigma(V, sigma);
// Get the dominance frontier of the successor
DominanceFrontier::iterator DF_BB = DF_->find(BB_next);
/*
* Creation (or operand insertion) of the SSI_PHI
*/
// If BB_next has no dominance frontier, there is no need of SSI_PHI
if (DF_BB == DF_->end()) {
DEBUG(dbgs() << "VSSA: No dominance frontier\n");
continue;
}
// If it was the first sigma created for V, phis may be needed, so we verify and create them
if (firstSigma) {
// Insert vSSA_phi, if necessary
vssaphi_created = insertPhisForSigma(V, sigma);
}
else {
// If it was not the first sigma created for V, then the phis already have been created and are stored in the deque, so we need only to insert the sigma as operand of them
insertSigmaAsOperandOfPhis(vssaphi_created, sigma);
}
}
// If phis were created, no more phis will be created, only operands will be inserted in them
if (firstSigma && !vssaphi_created.empty())
firstSigma = false;
}
// Populate the phi functions created
populatePhis(vssaphi_created, V);
// Insertion of vSSA_phi functions may require the creation of vSSA_phi functions too, so we call this function recursively
for (SmallVectorImpl<PHINode*>::iterator vit = vssaphi_created.begin(), vend = vssaphi_created.end(); vit != vend; ++vit) {
// Rename uses of V to phi
renameUsesToPhi(V, *vit);
insertPhisForPhi(V, *vit);
}
}
void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
unsigned FixedInstr = 0;
unsigned FixedMemOp = 0;
unsigned FixedDbg = 0;
MachineModuleInfo *MMI = &MF->getMMI();
// Remap debug information that refers to stack slots.
MachineModuleInfo::VariableDbgInfoMapTy &VMap = MMI->getVariableDbgInfo();
for (MachineModuleInfo::VariableDbgInfoMapTy::iterator VI = VMap.begin(),
VE = VMap.end(); VI != VE; ++VI) {
const MDNode *Var = VI->first;
if (!Var) continue;
std::pair<unsigned, DebugLoc> &VP = VI->second;
if (SlotRemap.count(VP.first)) {
DEBUG(dbgs()<<"Remapping debug info for ["<<Var->getName()<<"].\n");
VP.first = SlotRemap[VP.first];
FixedDbg++;
}
}
// Keep a list of *allocas* which need to be remapped.
DenseMap<const Value*, const Value*> Allocas;
for (DenseMap<int, int>::iterator it = SlotRemap.begin(),
e = SlotRemap.end(); it != e; ++it) {
const Value *From = MFI->getObjectAllocation(it->first);
const Value *To = MFI->getObjectAllocation(it->second);
assert(To && From && "Invalid allocation object");
Allocas[From] = To;
}
// Remap all instructions to the new stack slots.
MachineFunction::iterator BB, BBE;
MachineBasicBlock::iterator I, IE;
for (BB = MF->begin(), BBE = MF->end(); BB != BBE; ++BB)
for (I = BB->begin(), IE = BB->end(); I != IE; ++I) {
// Skip lifetime markers. We'll remove them soon.
if (I->getOpcode() == TargetOpcode::LIFETIME_START ||
I->getOpcode() == TargetOpcode::LIFETIME_END)
continue;
// Update the MachineMemOperand to use the new alloca.
for (MachineInstr::mmo_iterator MM = I->memoperands_begin(),
E = I->memoperands_end(); MM != E; ++MM) {
MachineMemOperand *MMO = *MM;
const Value *V = MMO->getValue();
if (!V)
continue;
// Climb up and find the original alloca.
V = GetUnderlyingObject(V);
// If we did not find one, or if the one that we found is not in our
// map, then move on.
if (!V || !Allocas.count(V))
continue;
MMO->setValue(Allocas[V]);
FixedMemOp++;
}
// Update all of the machine instruction operands.
for (unsigned i = 0 ; i < I->getNumOperands(); ++i) {
MachineOperand &MO = I->getOperand(i);
if (!MO.isFI())
continue;
int FromSlot = MO.getIndex();
// Don't touch arguments.
if (FromSlot<0)
continue;
// Only look at mapped slots.
if (!SlotRemap.count(FromSlot))
continue;
// In a debug build, check that the instruction that we are modifying is
// inside the expected live range. If the instruction is not inside
// the calculated range then it means that the alloca usage moved
// outside of the lifetime markers.
#ifndef NDEBUG
SlotIndex Index = Indexes->getInstructionIndex(I);
LiveInterval* Interval = Intervals[FromSlot];
assert(Interval->find(Index) != Interval->end() &&
"Found instruction usage outside of live range.");
#endif
// Fix the machine instructions.
int ToSlot = SlotRemap[FromSlot];
MO.setIndex(ToSlot);
FixedInstr++;
}
}
DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
}
bool WebAssemblyFixIrreducibleControlFlow::VisitLoop(MachineFunction &MF,
MachineLoopInfo &MLI,
MachineLoop *Loop) {
MachineBasicBlock *Header = Loop ? Loop->getHeader() : &*MF.begin();
SetVector<MachineBasicBlock *> RewriteSuccs;
// DFS through Loop's body, looking for for irreducible control flow. Loop is
// natural, and we stay in its body, and we treat any nested loops
// monolithically, so any cycles we encounter indicate irreducibility.
SmallPtrSet<MachineBasicBlock *, 8> OnStack;
SmallPtrSet<MachineBasicBlock *, 8> Visited;
SmallVector<SuccessorList, 4> LoopWorklist;
LoopWorklist.push_back(SuccessorList(Header));
OnStack.insert(Header);
Visited.insert(Header);
while (!LoopWorklist.empty()) {
SuccessorList &Top = LoopWorklist.back();
if (Top.HasNext()) {
MachineBasicBlock *Next = Top.Next();
if (Next == Header || (Loop && !Loop->contains(Next)))
continue;
if (LLVM_LIKELY(OnStack.insert(Next).second)) {
if (!Visited.insert(Next).second) {
OnStack.erase(Next);
continue;
}
MachineLoop *InnerLoop = MLI.getLoopFor(Next);
if (InnerLoop != Loop)
LoopWorklist.push_back(SuccessorList(InnerLoop));
else
LoopWorklist.push_back(SuccessorList(Next));
} else {
RewriteSuccs.insert(Top.getBlock());
}
continue;
}
OnStack.erase(Top.getBlock());
LoopWorklist.pop_back();
}
// Most likely, we didn't find any irreducible control flow.
if (LLVM_LIKELY(RewriteSuccs.empty()))
return false;
DEBUG(dbgs() << "Irreducible control flow detected!\n");
// Ok. We have irreducible control flow! Create a dispatch block which will
// contains a jump table to any block in the problematic set of blocks.
MachineBasicBlock *Dispatch = MF.CreateMachineBasicBlock();
MF.insert(MF.end(), Dispatch);
MLI.changeLoopFor(Dispatch, Loop);
// Add the jump table.
const auto &TII = *MF.getSubtarget<WebAssemblySubtarget>().getInstrInfo();
MachineInstrBuilder MIB = BuildMI(*Dispatch, Dispatch->end(), DebugLoc(),
TII.get(WebAssembly::BR_TABLE_I32));
// Add the register which will be used to tell the jump table which block to
// jump to.
MachineRegisterInfo &MRI = MF.getRegInfo();
unsigned Reg = MRI.createVirtualRegister(&WebAssembly::I32RegClass);
MIB.addReg(Reg);
// Collect all the blocks which need to have their successors rewritten,
// add the successors to the jump table, and remember their index.
DenseMap<MachineBasicBlock *, unsigned> Indices;
SmallVector<MachineBasicBlock *, 4> SuccWorklist(RewriteSuccs.begin(),
RewriteSuccs.end());
while (!SuccWorklist.empty()) {
MachineBasicBlock *MBB = SuccWorklist.pop_back_val();
auto Pair = Indices.insert(std::make_pair(MBB, 0));
if (!Pair.second)
continue;
unsigned Index = MIB.getInstr()->getNumExplicitOperands() - 1;
DEBUG(dbgs() << printMBBReference(*MBB) << " has index " << Index << "\n");
Pair.first->second = Index;
for (auto Pred : MBB->predecessors())
RewriteSuccs.insert(Pred);
MIB.addMBB(MBB);
Dispatch->addSuccessor(MBB);
MetaBlock Meta(MBB);
for (auto *Succ : Meta.successors())
if (Succ != Header && (!Loop || Loop->contains(Succ)))
SuccWorklist.push_back(Succ);
}
// Rewrite the problematic successors for every block in RewriteSuccs.
// For simplicity, we just introduce a new block for every edge we need to
// rewrite. Fancier things are possible.
for (MachineBasicBlock *MBB : RewriteSuccs) {
DenseMap<MachineBasicBlock *, MachineBasicBlock *> Map;
for (auto *Succ : MBB->successors()) {
if (!Indices.count(Succ))
continue;
MachineBasicBlock *Split = MF.CreateMachineBasicBlock();
MF.insert(MBB->isLayoutSuccessor(Succ) ? MachineFunction::iterator(Succ)
: MF.end(),
Split);
MLI.changeLoopFor(Split, Loop);
// Set the jump table's register of the index of the block we wish to
// jump to, and jump to the jump table.
BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::CONST_I32),
Reg)
.addImm(Indices[Succ]);
BuildMI(*Split, Split->end(), DebugLoc(), TII.get(WebAssembly::BR))
.addMBB(Dispatch);
Split->addSuccessor(Dispatch);
Map[Succ] = Split;
}
// Remap the terminator operands and the successor list.
for (MachineInstr &Term : MBB->terminators())
for (auto &Op : Term.explicit_uses())
if (Op.isMBB() && Indices.count(Op.getMBB()))
Op.setMBB(Map[Op.getMBB()]);
for (auto Rewrite : Map)
MBB->replaceSuccessor(Rewrite.first, Rewrite.second);
}
// Create a fake default label, because br_table requires one.
MIB.addMBB(MIB.getInstr()
->getOperand(MIB.getInstr()->getNumExplicitOperands() - 1)
.getMBB());
return true;
}
/// CloneAndPruneFunctionInto - This works exactly like CloneFunctionInto,
/// except that it does some simple constant prop and DCE on the fly. The
/// effect of this is to copy significantly less code in cases where (for
/// example) a function call with constant arguments is inlined, and those
/// constant arguments cause a significant amount of code in the callee to be
/// dead. Since this doesn't produce an exact copy of the input, it can't be
/// used for things like CloneFunction or CloneModule.
void llvm::CloneAndPruneFunctionInto(Function *NewFunc, const Function *OldFunc,
DenseMap<const Value*, Value*> &ValueMap,
SmallVectorImpl<ReturnInst*> &Returns,
const char *NameSuffix,
ClonedCodeInfo *CodeInfo,
const TargetData *TD,
Instruction *TheCall) {
assert(NameSuffix && "NameSuffix cannot be null!");
#ifndef NDEBUG
for (Function::const_arg_iterator II = OldFunc->arg_begin(),
E = OldFunc->arg_end(); II != E; ++II)
assert(ValueMap.count(II) && "No mapping from source argument specified!");
#endif
PruningFunctionCloner PFC(NewFunc, OldFunc, ValueMap, Returns,
NameSuffix, CodeInfo, TD);
// Clone the entry block, and anything recursively reachable from it.
std::vector<const BasicBlock*> CloneWorklist;
CloneWorklist.push_back(&OldFunc->getEntryBlock());
while (!CloneWorklist.empty()) {
const BasicBlock *BB = CloneWorklist.back();
CloneWorklist.pop_back();
PFC.CloneBlock(BB, CloneWorklist);
}
// Loop over all of the basic blocks in the old function. If the block was
// reachable, we have cloned it and the old block is now in the value map:
// insert it into the new function in the right order. If not, ignore it.
//
// Defer PHI resolution until rest of function is resolved.
SmallVector<const PHINode*, 16> PHIToResolve;
for (Function::const_iterator BI = OldFunc->begin(), BE = OldFunc->end();
BI != BE; ++BI) {
BasicBlock *NewBB = cast_or_null<BasicBlock>(ValueMap[BI]);
if (NewBB == 0) continue; // Dead block.
// Add the new block to the new function.
NewFunc->getBasicBlockList().push_back(NewBB);
// Loop over all of the instructions in the block, fixing up operand
// references as we go. This uses ValueMap to do all the hard work.
//
BasicBlock::iterator I = NewBB->begin();
LLVMContext &Context = OldFunc->getContext();
unsigned DbgKind = Context.getMetadata().getMDKind("dbg");
MDNode *TheCallMD = NULL;
SmallVector<Value *, 4> MDVs;
if (TheCall && TheCall->hasMetadata())
TheCallMD = Context.getMetadata().getMD(DbgKind, TheCall);
// Handle PHI nodes specially, as we have to remove references to dead
// blocks.
if (PHINode *PN = dyn_cast<PHINode>(I)) {
// Skip over all PHI nodes, remembering them for later.
BasicBlock::const_iterator OldI = BI->begin();
for (; (PN = dyn_cast<PHINode>(I)); ++I, ++OldI) {
if (I->hasMetadata()) {
if (TheCallMD) {
if (MDNode *IMD = Context.getMetadata().getMD(DbgKind, I)) {
MDNode *NewMD = UpdateInlinedAtInfo(IMD, TheCallMD, Context);
Context.getMetadata().addMD(DbgKind, NewMD, I);
}
} else {
// The cloned instruction has dbg info but the call instruction
// does not have dbg info. Remove dbg info from cloned instruction.
Context.getMetadata().removeMD(DbgKind, I);
}
}
PHIToResolve.push_back(cast<PHINode>(OldI));
}
}
// Otherwise, remap the rest of the instructions normally.
for (; I != NewBB->end(); ++I) {
if (I->hasMetadata()) {
if (TheCallMD) {
if (MDNode *IMD = Context.getMetadata().getMD(DbgKind, I)) {
MDNode *NewMD = UpdateInlinedAtInfo(IMD, TheCallMD, Context);
Context.getMetadata().addMD(DbgKind, NewMD, I);
}
} else {
// The cloned instruction has dbg info but the call instruction
// does not have dbg info. Remove dbg info from cloned instruction.
Context.getMetadata().removeMD(DbgKind, I);
}
}
RemapInstruction(I, ValueMap);
}
}
// Defer PHI resolution until rest of function is resolved, PHI resolution
// requires the CFG to be up-to-date.
for (unsigned phino = 0, e = PHIToResolve.size(); phino != e; ) {
const PHINode *OPN = PHIToResolve[phino];
unsigned NumPreds = OPN->getNumIncomingValues();
const BasicBlock *OldBB = OPN->getParent();
BasicBlock *NewBB = cast<BasicBlock>(ValueMap[OldBB]);
// Map operands for blocks that are live and remove operands for blocks
// that are dead.
for (; phino != PHIToResolve.size() &&
PHIToResolve[phino]->getParent() == OldBB; ++phino) {
OPN = PHIToResolve[phino];
PHINode *PN = cast<PHINode>(ValueMap[OPN]);
for (unsigned pred = 0, e = NumPreds; pred != e; ++pred) {
if (BasicBlock *MappedBlock =
cast_or_null<BasicBlock>(ValueMap[PN->getIncomingBlock(pred)])) {
Value *InVal = MapValue(PN->getIncomingValue(pred),
ValueMap);
assert(InVal && "Unknown input value?");
PN->setIncomingValue(pred, InVal);
PN->setIncomingBlock(pred, MappedBlock);
} else {
PN->removeIncomingValue(pred, false);
--pred, --e; // Revisit the next entry.
}
}
}
// The loop above has removed PHI entries for those blocks that are dead
// and has updated others. However, if a block is live (i.e. copied over)
// but its terminator has been changed to not go to this block, then our
// phi nodes will have invalid entries. Update the PHI nodes in this
// case.
PHINode *PN = cast<PHINode>(NewBB->begin());
NumPreds = std::distance(pred_begin(NewBB), pred_end(NewBB));
if (NumPreds != PN->getNumIncomingValues()) {
assert(NumPreds < PN->getNumIncomingValues());
// Count how many times each predecessor comes to this block.
std::map<BasicBlock*, unsigned> PredCount;
for (pred_iterator PI = pred_begin(NewBB), E = pred_end(NewBB);
PI != E; ++PI)
--PredCount[*PI];
// Figure out how many entries to remove from each PHI.
for (unsigned i = 0, e = PN->getNumIncomingValues(); i != e; ++i)
++PredCount[PN->getIncomingBlock(i)];
// At this point, the excess predecessor entries are positive in the
// map. Loop over all of the PHIs and remove excess predecessor
// entries.
BasicBlock::iterator I = NewBB->begin();
for (; (PN = dyn_cast<PHINode>(I)); ++I) {
for (std::map<BasicBlock*, unsigned>::iterator PCI =PredCount.begin(),
E = PredCount.end(); PCI != E; ++PCI) {
BasicBlock *Pred = PCI->first;
for (unsigned NumToRemove = PCI->second; NumToRemove; --NumToRemove)
PN->removeIncomingValue(Pred, false);
}
}
}
// If the loops above have made these phi nodes have 0 or 1 operand,
// replace them with undef or the input value. We must do this for
// correctness, because 0-operand phis are not valid.
PN = cast<PHINode>(NewBB->begin());
if (PN->getNumIncomingValues() == 0) {
BasicBlock::iterator I = NewBB->begin();
BasicBlock::const_iterator OldI = OldBB->begin();
while ((PN = dyn_cast<PHINode>(I++))) {
Value *NV = UndefValue::get(PN->getType());
PN->replaceAllUsesWith(NV);
assert(ValueMap[OldI] == PN && "ValueMap mismatch");
ValueMap[OldI] = NV;
PN->eraseFromParent();
++OldI;
}
}
// NOTE: We cannot eliminate single entry phi nodes here, because of
// ValueMap. Single entry phi nodes can have multiple ValueMap entries
// pointing at them. Thus, deleting one would require scanning the ValueMap
// to update any entries in it that would require that. This would be
// really slow.
}
// Now that the inlined function body has been fully constructed, go through
// and zap unconditional fall-through branches. This happen all the time when
// specializing code: code specialization turns conditional branches into
// uncond branches, and this code folds them.
Function::iterator I = cast<BasicBlock>(ValueMap[&OldFunc->getEntryBlock()]);
while (I != NewFunc->end()) {
BranchInst *BI = dyn_cast<BranchInst>(I->getTerminator());
if (!BI || BI->isConditional()) { ++I; continue; }
// Note that we can't eliminate uncond branches if the destination has
// single-entry PHI nodes. Eliminating the single-entry phi nodes would
// require scanning the ValueMap to update any entries that point to the phi
// node.
BasicBlock *Dest = BI->getSuccessor(0);
if (!Dest->getSinglePredecessor() || isa<PHINode>(Dest->begin())) {
++I; continue;
}
// We know all single-entry PHI nodes in the inlined function have been
// removed, so we just need to splice the blocks.
BI->eraseFromParent();
// Move all the instructions in the succ to the pred.
I->getInstList().splice(I->end(), Dest->getInstList());
// Make all PHI nodes that referred to Dest now refer to I as their source.
Dest->replaceAllUsesWith(I);
// Remove the dest block.
Dest->eraseFromParent();
// Do not increment I, iteratively merge all things this block branches to.
}
}
MapVector<Instruction *, uint64_t>
llvm::computeMinimumValueSizes(ArrayRef<BasicBlock *> Blocks, DemandedBits &DB,
const TargetTransformInfo *TTI) {
// DemandedBits will give us every value's live-out bits. But we want
// to ensure no extra casts would need to be inserted, so every DAG
// of connected values must have the same minimum bitwidth.
EquivalenceClasses<Value *> ECs;
SmallVector<Value *, 16> Worklist;
SmallPtrSet<Value *, 4> Roots;
SmallPtrSet<Value *, 16> Visited;
DenseMap<Value *, uint64_t> DBits;
SmallPtrSet<Instruction *, 4> InstructionSet;
MapVector<Instruction *, uint64_t> MinBWs;
// Determine the roots. We work bottom-up, from truncs or icmps.
bool SeenExtFromIllegalType = false;
for (auto *BB : Blocks)
for (auto &I : *BB) {
InstructionSet.insert(&I);
if (TTI && (isa<ZExtInst>(&I) || isa<SExtInst>(&I)) &&
!TTI->isTypeLegal(I.getOperand(0)->getType()))
SeenExtFromIllegalType = true;
// Only deal with non-vector integers up to 64-bits wide.
if ((isa<TruncInst>(&I) || isa<ICmpInst>(&I)) &&
!I.getType()->isVectorTy() &&
I.getOperand(0)->getType()->getScalarSizeInBits() <= 64) {
// Don't make work for ourselves. If we know the loaded type is legal,
// don't add it to the worklist.
if (TTI && isa<TruncInst>(&I) && TTI->isTypeLegal(I.getType()))
continue;
Worklist.push_back(&I);
Roots.insert(&I);
}
}
// Early exit.
if (Worklist.empty() || (TTI && !SeenExtFromIllegalType))
return MinBWs;
// Now proceed breadth-first, unioning values together.
while (!Worklist.empty()) {
Value *Val = Worklist.pop_back_val();
Value *Leader = ECs.getOrInsertLeaderValue(Val);
if (Visited.count(Val))
continue;
Visited.insert(Val);
// Non-instructions terminate a chain successfully.
if (!isa<Instruction>(Val))
continue;
Instruction *I = cast<Instruction>(Val);
// If we encounter a type that is larger than 64 bits, we can't represent
// it so bail out.
if (DB.getDemandedBits(I).getBitWidth() > 64)
return MapVector<Instruction *, uint64_t>();
uint64_t V = DB.getDemandedBits(I).getZExtValue();
DBits[Leader] |= V;
DBits[I] = V;
// Casts, loads and instructions outside of our range terminate a chain
// successfully.
if (isa<SExtInst>(I) || isa<ZExtInst>(I) || isa<LoadInst>(I) ||
!InstructionSet.count(I))
continue;
// Unsafe casts terminate a chain unsuccessfully. We can't do anything
// useful with bitcasts, ptrtoints or inttoptrs and it'd be unsafe to
// transform anything that relies on them.
if (isa<BitCastInst>(I) || isa<PtrToIntInst>(I) || isa<IntToPtrInst>(I) ||
!I->getType()->isIntegerTy()) {
DBits[Leader] |= ~0ULL;
continue;
}
// We don't modify the types of PHIs. Reductions will already have been
// truncated if possible, and inductions' sizes will have been chosen by
// indvars.
if (isa<PHINode>(I))
continue;
if (DBits[Leader] == ~0ULL)
// All bits demanded, no point continuing.
continue;
for (Value *O : cast<User>(I)->operands()) {
ECs.unionSets(Leader, O);
Worklist.push_back(O);
}
}
// Now we've discovered all values, walk them to see if there are
// any users we didn't see. If there are, we can't optimize that
// chain.
for (auto &I : DBits)
for (auto *U : I.first->users())
if (U->getType()->isIntegerTy() && DBits.count(U) == 0)
DBits[ECs.getOrInsertLeaderValue(I.first)] |= ~0ULL;
for (auto I = ECs.begin(), E = ECs.end(); I != E; ++I) {
uint64_t LeaderDemandedBits = 0;
for (auto MI = ECs.member_begin(I), ME = ECs.member_end(); MI != ME; ++MI)
LeaderDemandedBits |= DBits[*MI];
uint64_t MinBW = (sizeof(LeaderDemandedBits) * 8) -
llvm::countLeadingZeros(LeaderDemandedBits);
// Round up to a power of 2
if (!isPowerOf2_64((uint64_t)MinBW))
MinBW = NextPowerOf2(MinBW);
// We don't modify the types of PHIs. Reductions will already have been
// truncated if possible, and inductions' sizes will have been chosen by
// indvars.
// If we are required to shrink a PHI, abandon this entire equivalence class.
bool Abort = false;
for (auto MI = ECs.member_begin(I), ME = ECs.member_end(); MI != ME; ++MI)
if (isa<PHINode>(*MI) && MinBW < (*MI)->getType()->getScalarSizeInBits()) {
Abort = true;
break;
}
if (Abort)
continue;
for (auto MI = ECs.member_begin(I), ME = ECs.member_end(); MI != ME; ++MI) {
if (!isa<Instruction>(*MI))
continue;
Type *Ty = (*MI)->getType();
if (Roots.count(*MI))
Ty = cast<Instruction>(*MI)->getOperand(0)->getType();
if (MinBW < Ty->getScalarSizeInBits())
MinBWs[cast<Instruction>(*MI)] = MinBW;
}
}
return MinBWs;
}
Value *PropagateJuliaAddrspaces::LiftPointer(Value *V, Type *LocTy, Instruction *InsertPt) {
SmallVector<Value *, 4> Stack;
Value *CurrentV = V;
// Follow pointer casts back, see if we're based on a pointer in
// an untracked address space, in which case we're allowed to drop
// intermediate addrspace casts.
while (true) {
Stack.push_back(CurrentV);
if (isa<BitCastInst>(CurrentV))
CurrentV = cast<BitCastInst>(CurrentV)->getOperand(0);
else if (isa<AddrSpaceCastInst>(CurrentV)) {
CurrentV = cast<AddrSpaceCastInst>(CurrentV)->getOperand(0);
if (!isSpecialAS(getValueAddrSpace(CurrentV)))
break;
}
else if (isa<GetElementPtrInst>(CurrentV)) {
if (LiftingMap.count(CurrentV)) {
CurrentV = LiftingMap[CurrentV];
break;
} else if (Visited.count(CurrentV)) {
return nullptr;
}
Visited.insert(CurrentV);
CurrentV = cast<GetElementPtrInst>(CurrentV)->getOperand(0);
} else
break;
}
if (!CurrentV->getType()->isPointerTy())
return nullptr;
if (isSpecialAS(getValueAddrSpace(CurrentV)))
return nullptr;
// Ok, we're allowed to change the address space of this load, go back and
// reconstitute any GEPs in the new address space.
for (Value *V : llvm::reverse(Stack)) {
GetElementPtrInst *GEP = dyn_cast<GetElementPtrInst>(V);
if (!GEP)
continue;
if (LiftingMap.count(GEP)) {
CurrentV = LiftingMap[GEP];
continue;
}
GetElementPtrInst *NewGEP = cast<GetElementPtrInst>(GEP->clone());
ToInsert.push_back(std::make_pair(NewGEP, GEP));
Type *GEPTy = GEP->getSourceElementType();
Type *NewRetTy = cast<PointerType>(GEP->getType())->getElementType()->getPointerTo(getValueAddrSpace(CurrentV));
NewGEP->mutateType(NewRetTy);
if (cast<PointerType>(CurrentV->getType())->getElementType() != GEPTy) {
auto *BCI = new BitCastInst(CurrentV, GEPTy->getPointerTo());
ToInsert.push_back(std::make_pair(BCI, NewGEP));
CurrentV = BCI;
}
NewGEP->setOperand(GetElementPtrInst::getPointerOperandIndex(), CurrentV);
LiftingMap[GEP] = NewGEP;
CurrentV = NewGEP;
}
if (LocTy && cast<PointerType>(CurrentV->getType())->getElementType() != LocTy) {
auto *BCI = new BitCastInst(CurrentV, LocTy->getPointerTo());
ToInsert.push_back(std::make_pair(BCI, InsertPt));
CurrentV = BCI;
}
return CurrentV;
}
/// getNonLocalPointerDepFromBB - Perform a dependency query based on
/// pointer/pointeesize starting at the end of StartBB. Add any clobber/def
/// results to the results vector and keep track of which blocks are visited in
/// 'Visited'.
///
/// This has special behavior for the first block queries (when SkipFirstBlock
/// is true). In this special case, it ignores the contents of the specified
/// block and starts returning dependence info for its predecessors.
///
/// This function returns false on success, or true to indicate that it could
/// not compute dependence information for some reason. This should be treated
/// as a clobber dependence on the first instruction in the predecessor block.
bool MemoryDependenceAnalysis::
getNonLocalPointerDepFromBB(Value *Pointer, uint64_t PointeeSize,
bool isLoad, BasicBlock *StartBB,
SmallVectorImpl<NonLocalDepEntry> &Result,
DenseMap<BasicBlock*, Value*> &Visited,
bool SkipFirstBlock) {
// Look up the cached info for Pointer.
ValueIsLoadPair CacheKey(Pointer, isLoad);
std::pair<BBSkipFirstBlockPair, NonLocalDepInfo> *CacheInfo =
&NonLocalPointerDeps[CacheKey];
NonLocalDepInfo *Cache = &CacheInfo->second;
// If we have valid cached information for exactly the block we are
// investigating, just return it with no recomputation.
if (CacheInfo->first == BBSkipFirstBlockPair(StartBB, SkipFirstBlock)) {
// We have a fully cached result for this query then we can just return the
// cached results and populate the visited set. However, we have to verify
// that we don't already have conflicting results for these blocks. Check
// to ensure that if a block in the results set is in the visited set that
// it was for the same pointer query.
if (!Visited.empty()) {
for (NonLocalDepInfo::iterator I = Cache->begin(), E = Cache->end();
I != E; ++I) {
DenseMap<BasicBlock*, Value*>::iterator VI = Visited.find(I->first);
if (VI == Visited.end() || VI->second == Pointer) continue;
// We have a pointer mismatch in a block. Just return clobber, saying
// that something was clobbered in this result. We could also do a
// non-fully cached query, but there is little point in doing this.
return true;
}
}
for (NonLocalDepInfo::iterator I = Cache->begin(), E = Cache->end();
I != E; ++I) {
Visited.insert(std::make_pair(I->first, Pointer));
if (!I->second.isNonLocal())
Result.push_back(*I);
}
++NumCacheCompleteNonLocalPtr;
return false;
}
// Otherwise, either this is a new block, a block with an invalid cache
// pointer or one that we're about to invalidate by putting more info into it
// than its valid cache info. If empty, the result will be valid cache info,
// otherwise it isn't.
if (Cache->empty())
CacheInfo->first = BBSkipFirstBlockPair(StartBB, SkipFirstBlock);
else
CacheInfo->first = BBSkipFirstBlockPair();
SmallVector<BasicBlock*, 32> Worklist;
Worklist.push_back(StartBB);
// Keep track of the entries that we know are sorted. Previously cached
// entries will all be sorted. The entries we add we only sort on demand (we
// don't insert every element into its sorted position). We know that we
// won't get any reuse from currently inserted values, because we don't
// revisit blocks after we insert info for them.
unsigned NumSortedEntries = Cache->size();
DEBUG(AssertSorted(*Cache));
while (!Worklist.empty()) {
BasicBlock *BB = Worklist.pop_back_val();
// Skip the first block if we have it.
if (!SkipFirstBlock) {
// Analyze the dependency of *Pointer in FromBB. See if we already have
// been here.
assert(Visited.count(BB) && "Should check 'visited' before adding to WL");
// Get the dependency info for Pointer in BB. If we have cached
// information, we will use it, otherwise we compute it.
DEBUG(AssertSorted(*Cache, NumSortedEntries));
MemDepResult Dep = GetNonLocalInfoForBlock(Pointer, PointeeSize, isLoad,
BB, Cache, NumSortedEntries);
// If we got a Def or Clobber, add this to the list of results.
if (!Dep.isNonLocal()) {
Result.push_back(NonLocalDepEntry(BB, Dep));
continue;
}
}
// If 'Pointer' is an instruction defined in this block, then we need to do
// phi translation to change it into a value live in the predecessor block.
// If phi translation fails, then we can't continue dependence analysis.
Instruction *PtrInst = dyn_cast<Instruction>(Pointer);
bool NeedsPHITranslation = PtrInst && PtrInst->getParent() == BB;
// If no PHI translation is needed, just add all the predecessors of this
// block to scan them as well.
if (!NeedsPHITranslation) {
SkipFirstBlock = false;
for (BasicBlock **PI = PredCache->GetPreds(BB); *PI; ++PI) {
// Verify that we haven't looked at this block yet.
std::pair<DenseMap<BasicBlock*,Value*>::iterator, bool>
InsertRes = Visited.insert(std::make_pair(*PI, Pointer));
if (InsertRes.second) {
// First time we've looked at *PI.
Worklist.push_back(*PI);
continue;
}
// If we have seen this block before, but it was with a different
// pointer then we have a phi translation failure and we have to treat
// this as a clobber.
if (InsertRes.first->second != Pointer)
goto PredTranslationFailure;
}
continue;
}
// If we do need to do phi translation, then there are a bunch of different
// cases, because we have to find a Value* live in the predecessor block. We
// know that PtrInst is defined in this block at least.
// If this is directly a PHI node, just use the incoming values for each
// pred as the phi translated version.
if (PHINode *PtrPHI = dyn_cast<PHINode>(PtrInst)) {
for (BasicBlock **PI = PredCache->GetPreds(BB); *PI; ++PI) {
BasicBlock *Pred = *PI;
Value *PredPtr = PtrPHI->getIncomingValueForBlock(Pred);
// Check to see if we have already visited this pred block with another
// pointer. If so, we can't do this lookup. This failure can occur
// with PHI translation when a critical edge exists and the PHI node in
// the successor translates to a pointer value different than the
// pointer the block was first analyzed with.
std::pair<DenseMap<BasicBlock*,Value*>::iterator, bool>
InsertRes = Visited.insert(std::make_pair(Pred, PredPtr));
if (!InsertRes.second) {
// If the predecessor was visited with PredPtr, then we already did
// the analysis and can ignore it.
if (InsertRes.first->second == PredPtr)
continue;
// Otherwise, the block was previously analyzed with a different
// pointer. We can't represent the result of this case, so we just
// treat this as a phi translation failure.
goto PredTranslationFailure;
}
// We may have added values to the cache list before this PHI
// translation. If so, we haven't done anything to ensure that the
// cache remains sorted. Sort it now (if needed) so that recursive
// invocations of getNonLocalPointerDepFromBB that could reuse the cache
// value will only see properly sorted cache arrays.
if (Cache && NumSortedEntries != Cache->size())
std::sort(Cache->begin(), Cache->end());
Cache = 0;
// FIXME: it is entirely possible that PHI translating will end up with
// the same value. Consider PHI translating something like:
// X = phi [x, bb1], [y, bb2]. PHI translating for bb1 doesn't *need*
// to recurse here, pedantically speaking.
// If we have a problem phi translating, fall through to the code below
// to handle the failure condition.
if (getNonLocalPointerDepFromBB(PredPtr, PointeeSize, isLoad, Pred,
Result, Visited))
goto PredTranslationFailure;
}
// Refresh the CacheInfo/Cache pointer so that it isn't invalidated.
CacheInfo = &NonLocalPointerDeps[CacheKey];
Cache = &CacheInfo->second;
NumSortedEntries = Cache->size();
// Since we did phi translation, the "Cache" set won't contain all of the
// results for the query. This is ok (we can still use it to accelerate
// specific block queries) but we can't do the fastpath "return all
// results from the set" Clear out the indicator for this.
CacheInfo->first = BBSkipFirstBlockPair();
SkipFirstBlock = false;
continue;
}
// TODO: BITCAST, GEP.
// cerr << "MEMDEP: Could not PHI translate: " << *Pointer;
// if (isa<BitCastInst>(PtrInst) || isa<GetElementPtrInst>(PtrInst))
// cerr << "OP:\t\t\t\t" << *PtrInst->getOperand(0);
PredTranslationFailure:
if (Cache == 0) {
// Refresh the CacheInfo/Cache pointer if it got invalidated.
CacheInfo = &NonLocalPointerDeps[CacheKey];
Cache = &CacheInfo->second;
NumSortedEntries = Cache->size();
} else if (NumSortedEntries != Cache->size()) {
std::sort(Cache->begin(), Cache->end());
NumSortedEntries = Cache->size();
}
// Since we did phi translation, the "Cache" set won't contain all of the
// results for the query. This is ok (we can still use it to accelerate
// specific block queries) but we can't do the fastpath "return all
// results from the set" Clear out the indicator for this.
CacheInfo->first = BBSkipFirstBlockPair();
// If *nothing* works, mark the pointer as being clobbered by the first
// instruction in this block.
//
// If this is the magic first block, return this as a clobber of the whole
// incoming value. Since we can't phi translate to one of the predecessors,
// we have to bail out.
if (SkipFirstBlock)
return true;
for (NonLocalDepInfo::reverse_iterator I = Cache->rbegin(); ; ++I) {
assert(I != Cache->rend() && "Didn't find current block??");
if (I->first != BB)
continue;
assert(I->second.isNonLocal() &&
"Should only be here with transparent block");
I->second = MemDepResult::getClobber(BB->begin());
ReverseNonLocalPtrDeps[BB->begin()].insert(CacheKey.getOpaqueValue());
Result.push_back(*I);
break;
}
}
// Okay, we're done now. If we added new values to the cache, re-sort it.
switch (Cache->size()-NumSortedEntries) {
case 0:
// done, no new entries.
break;
case 2: {
// Two new entries, insert the last one into place.
NonLocalDepEntry Val = Cache->back();
Cache->pop_back();
NonLocalDepInfo::iterator Entry =
std::upper_bound(Cache->begin(), Cache->end()-1, Val);
Cache->insert(Entry, Val);
// FALL THROUGH.
}
case 1:
// One new entry, Just insert the new value at the appropriate position.
if (Cache->size() != 1) {
NonLocalDepEntry Val = Cache->back();
Cache->pop_back();
NonLocalDepInfo::iterator Entry =
std::upper_bound(Cache->begin(), Cache->end(), Val);
Cache->insert(Entry, Val);
}
break;
default:
// Added many values, do a full scale sort.
std::sort(Cache->begin(), Cache->end());
}
DEBUG(AssertSorted(*Cache));
return false;
}
void StackColoring::remapInstructions(DenseMap<int, int> &SlotRemap) {
unsigned FixedInstr = 0;
unsigned FixedMemOp = 0;
unsigned FixedDbg = 0;
MachineModuleInfo *MMI = &MF->getMMI();
// Remap debug information that refers to stack slots.
MachineModuleInfo::VariableDbgInfoMapTy &VMap = MMI->getVariableDbgInfo();
for (MachineModuleInfo::VariableDbgInfoMapTy::iterator VI = VMap.begin(),
VE = VMap.end(); VI != VE; ++VI) {
const MDNode *Var = VI->first;
if (!Var) continue;
std::pair<unsigned, DebugLoc> &VP = VI->second;
if (SlotRemap.count(VP.first)) {
DEBUG(dbgs()<<"Remapping debug info for ["<<Var->getName()<<"].\n");
VP.first = SlotRemap[VP.first];
FixedDbg++;
}
}
// Keep a list of *allocas* which need to be remapped.
DenseMap<const AllocaInst*, const AllocaInst*> Allocas;
for (DenseMap<int, int>::const_iterator it = SlotRemap.begin(),
e = SlotRemap.end(); it != e; ++it) {
const AllocaInst *From = MFI->getObjectAllocation(it->first);
const AllocaInst *To = MFI->getObjectAllocation(it->second);
assert(To && From && "Invalid allocation object");
Allocas[From] = To;
}
// Remap all instructions to the new stack slots.
MachineFunction::iterator BB, BBE;
MachineBasicBlock::iterator I, IE;
for (BB = MF->begin(), BBE = MF->end(); BB != BBE; ++BB)
for (I = BB->begin(), IE = BB->end(); I != IE; ++I) {
// Skip lifetime markers. We'll remove them soon.
if (I->getOpcode() == TargetOpcode::LIFETIME_START ||
I->getOpcode() == TargetOpcode::LIFETIME_END)
continue;
// Update the MachineMemOperand to use the new alloca.
for (MachineInstr::mmo_iterator MM = I->memoperands_begin(),
E = I->memoperands_end(); MM != E; ++MM) {
MachineMemOperand *MMO = *MM;
const Value *V = MMO->getValue();
if (!V)
continue;
const PseudoSourceValue *PSV = dyn_cast<const PseudoSourceValue>(V);
if (PSV && PSV->isConstant(MFI))
continue;
// Climb up and find the original alloca.
V = GetUnderlyingObject(V);
// If we did not find one, or if the one that we found is not in our
// map, then move on.
if (!V || !isa<AllocaInst>(V)) {
// Clear mem operand since we don't know for sure that it doesn't
// alias a merged alloca.
MMO->setValue(0);
continue;
}
const AllocaInst *AI= cast<AllocaInst>(V);
if (!Allocas.count(AI))
continue;
MMO->setValue(Allocas[AI]);
FixedMemOp++;
}
// Update all of the machine instruction operands.
for (unsigned i = 0 ; i < I->getNumOperands(); ++i) {
MachineOperand &MO = I->getOperand(i);
if (!MO.isFI())
continue;
int FromSlot = MO.getIndex();
// Don't touch arguments.
if (FromSlot<0)
continue;
// Only look at mapped slots.
if (!SlotRemap.count(FromSlot))
continue;
// In a debug build, check that the instruction that we are modifying is
// inside the expected live range. If the instruction is not inside
// the calculated range then it means that the alloca usage moved
// outside of the lifetime markers, or that the user has a bug.
// NOTE: Alloca address calculations which happen outside the lifetime
// zone are are okay, despite the fact that we don't have a good way
// for validating all of the usages of the calculation.
#ifndef NDEBUG
bool TouchesMemory = I->mayLoad() || I->mayStore();
// If we *don't* protect the user from escaped allocas, don't bother
// validating the instructions.
if (!I->isDebugValue() && TouchesMemory && ProtectFromEscapedAllocas) {
SlotIndex Index = Indexes->getInstructionIndex(I);
LiveInterval *Interval = Intervals[FromSlot];
assert(Interval->find(Index) != Interval->end() &&
"Found instruction usage outside of live range.");
}
#endif
// Fix the machine instructions.
int ToSlot = SlotRemap[FromSlot];
MO.setIndex(ToSlot);
FixedInstr++;
}
}
DEBUG(dbgs()<<"Fixed "<<FixedMemOp<<" machine memory operands.\n");
DEBUG(dbgs()<<"Fixed "<<FixedDbg<<" debug locations.\n");
DEBUG(dbgs()<<"Fixed "<<FixedInstr<<" machine instructions.\n");
}
bool GuardWideningImpl::eliminateGuardViaWidening(
IntrinsicInst *GuardInst, const df_iterator<DomTreeNode *> &DFSI,
const DenseMap<BasicBlock *, SmallVector<IntrinsicInst *, 8>> &
GuardsInBlock) {
IntrinsicInst *BestSoFar = nullptr;
auto BestScoreSoFar = WS_IllegalOrNegative;
auto *GuardInstLoop = LI.getLoopFor(GuardInst->getParent());
// In the set of dominating guards, find the one we can merge GuardInst with
// for the most profit.
for (unsigned i = 0, e = DFSI.getPathLength(); i != e; ++i) {
auto *CurBB = DFSI.getPath(i)->getBlock();
auto *CurLoop = LI.getLoopFor(CurBB);
assert(GuardsInBlock.count(CurBB) && "Must have been populated by now!");
const auto &GuardsInCurBB = GuardsInBlock.find(CurBB)->second;
auto I = GuardsInCurBB.begin();
auto E = GuardsInCurBB.end();
#ifndef NDEBUG
{
unsigned Index = 0;
for (auto &I : *CurBB) {
if (Index == GuardsInCurBB.size())
break;
if (GuardsInCurBB[Index] == &I)
Index++;
}
assert(Index == GuardsInCurBB.size() &&
"Guards expected to be in order!");
}
#endif
assert((i == (e - 1)) == (GuardInst->getParent() == CurBB) && "Bad DFS?");
if (i == (e - 1)) {
// Corner case: make sure we're only looking at guards strictly dominating
// GuardInst when visiting GuardInst->getParent().
auto NewEnd = std::find(I, E, GuardInst);
assert(NewEnd != E && "GuardInst not in its own block?");
E = NewEnd;
}
for (auto *Candidate : make_range(I, E)) {
auto Score =
computeWideningScore(GuardInst, GuardInstLoop, Candidate, CurLoop);
DEBUG(dbgs() << "Score between " << *GuardInst->getArgOperand(0)
<< " and " << *Candidate->getArgOperand(0) << " is "
<< scoreTypeToString(Score) << "\n");
if (Score > BestScoreSoFar) {
BestScoreSoFar = Score;
BestSoFar = Candidate;
}
}
}
if (BestScoreSoFar == WS_IllegalOrNegative) {
DEBUG(dbgs() << "Did not eliminate guard " << *GuardInst << "\n");
return false;
}
assert(BestSoFar != GuardInst && "Should have never visited same guard!");
assert(DT.dominates(BestSoFar, GuardInst) && "Should be!");
DEBUG(dbgs() << "Widening " << *GuardInst << " into " << *BestSoFar
<< " with score " << scoreTypeToString(BestScoreSoFar) << "\n");
widenGuard(BestSoFar, GuardInst->getArgOperand(0));
GuardInst->setArgOperand(0, ConstantInt::getTrue(GuardInst->getContext()));
EliminatedGuards.push_back(GuardInst);
WidenedGuards.insert(BestSoFar);
return true;
}
std::map<GraphNode*, std::vector<GraphNode*> > llvm::Graph::getEveryDependency(
llvm::Value* sink, std::set<llvm::Value*> sources, bool skipMemoryNodes) {
std::map<llvm::GraphNode*, std::vector<GraphNode*> > result;
DenseMap<GraphNode*, GraphNode*> parent;
std::vector<GraphNode*> path;
// errs() << "--- Get every dep --- \n";
if (GraphNode* startNode = findNode(sink)) {
// errs() << "found sink\n";
// errs() << "Starting search from " << startNode->getLabel() << "\n";
std::set<GraphNode*> sourceNodes = findNodes(sources);
std::map<GraphNode*, int> nodeColor;
std::list<GraphNode*> workList;
// int size = 0;
for (std::set<GraphNode*>::iterator Nit = nodes.begin(), Nend =
nodes.end(); Nit != Nend; Nit++) {
// size++;
if (skipMemoryNodes && isa<MemNode> (*Nit))
nodeColor[*Nit] = 1;
else
nodeColor[*Nit] = 0;
}
workList.push_back(startNode);
nodeColor[startNode] = 1;
/*
* we will do a breadth search on the predecessors of each node,
* until we find one of the sources. If we don't find any, then the
* sink doesn't depend on any source.
*/
// int pb = 1;
while (!workList.empty()) {
GraphNode* workNode = workList.front();
workList.pop_front();
if (sourceNodes.count(workNode)) {
//Retrieve path
path.clear();
GraphNode* n = workNode;
path.push_back(n);
while (parent.count(n)) {
path.push_back(parent[n]);
n = parent[n];
}
// std::reverse(path.begin(), path.end());
// errs() << "Path: ";
// for (std::vector<GraphNode*>::iterator i = path.begin(), e = path.end(); i != e; ++i) {
// errs() << (*i)->getLabel() << " | ";
// }
// errs() << "\n";
result[workNode] = path;
}
std::map<GraphNode*, edgeType> preds = workNode->getPredecessors();
for (std::map<GraphNode*, edgeType>::iterator pred = preds.begin(),
pend = preds.end(); pred != pend; pred++) {
if (nodeColor[pred->first] == 0) { // the node hasn't been processed yet
nodeColor[pred->first] = 1;
workList.push_back(pred->first);
// pb++;
parent[pred->first] = workNode;
}
}
// errs() << pb << "/" << size << "\n";
}
}
return result;
}
void Expr::init(Value* V, int level){
//Level 0: stop recursion
//Level 1: recursion just one time
//Level 2: unlimited recursion
assert(V && "Constructor expected non-null parameter");
if (const ConstantInt *CI = dyn_cast<ConstantInt>(V)) {
Expr_ = GiNaC::ex((long int)CI->getValue().getSExtValue());
return;
}
if (Exprs.count(std::pair<Value*, int>(V,level))) {
Expr_ = Exprs[std::pair<Value*, int>(V,level)];
return;
}
//Level 0: stop recursion
if (level == 0) {
Twine Name;
if (V->hasName()) {
if (isa<Instruction>(V) || isa<Argument>(V))
Name = V->getName();
else
Name = "__SRA_SYM_UNKNOWN_" + V->getName() + "__";
} else {
Name = "__SRA_SYM_UNAMED__";
}
std::string NameStr = Name.str();
Expr_ = GiNaC::ex(GiNaC::symbol(NameStr));
Exprs[std::pair<Value*, int>(V,level)] = Expr_;
RevExprs[NameStr] = V;
return;
}
int old_level = level;
level = (level == 1 ? 0 : 2);
//Handle instructions
if (const BinaryOperator* BI = dyn_cast<llvm::BinaryOperator>(V)){
switch (BI->getOpcode()){
case Instruction::Add:
{
Expr ADD_Op1(BI->getOperand(0), level);
Expr ADD_Op2(BI->getOperand(1), level);
Expr_ = (ADD_Op1.getExpr() + ADD_Op2.getExpr());
Exprs[std::pair<Value*, int>(V,old_level)] = Expr_;
return;
}
case Instruction::Sub:
{
Expr SUB_Op1(BI->getOperand(0), level);
Expr SUB_Op2(BI->getOperand(1), level);
Expr_ = (SUB_Op1.getExpr() + SUB_Op2.getExpr());
Exprs[std::pair<Value*, int>(V,old_level)] = Expr_;
return;
}
case Instruction::Mul:
{
Expr MUL_Op1(BI->getOperand(0), level);
Expr MUL_Op2(BI->getOperand(1), level);
Expr_ = (MUL_Op1.getExpr() * MUL_Op2.getExpr());
Exprs[std::pair<Value*, int>(V,old_level)] = Expr_;
return;
}
case Instruction::SDiv:
case Instruction::UDiv:
{
Expr DIV_Op1(BI->getOperand(0), level);
Expr DIV_Op2(BI->getOperand(1), level);
Expr_ = (DIV_Op1.getExpr() / DIV_Op2.getExpr());
Exprs[std::pair<Value*, int>(V,old_level)] = Expr_;
return;
}
default:
break;
}
}
if (const CastInst* CI = dyn_cast<CastInst>(V)){
Expr CAST_Op(CI->getOperand(0), old_level);
Expr_ = CAST_Op.getExpr();
Exprs[std::pair<Value*, int>(V,old_level)] = Expr_;
return;
}
//Unhandled instructions: treat them like level 0
Expr tmp(V,0);
Expr_ = tmp.getExpr();
}
bool LiveRangeShrink::runOnMachineFunction(MachineFunction &MF) {
if (skipFunction(MF.getFunction()))
return false;
MachineRegisterInfo &MRI = MF.getRegInfo();
LLVM_DEBUG(dbgs() << "**** Analysing " << MF.getName() << '\n');
InstOrderMap IOM;
// Map from register to instruction order (value of IOM) where the
// register is used last. When moving instructions up, we need to
// make sure all its defs (including dead def) will not cross its
// last use when moving up.
DenseMap<unsigned, std::pair<unsigned, MachineInstr *>> UseMap;
for (MachineBasicBlock &MBB : MF) {
if (MBB.empty())
continue;
bool SawStore = false;
BuildInstOrderMap(MBB.begin(), IOM);
UseMap.clear();
for (MachineBasicBlock::iterator Next = MBB.begin(); Next != MBB.end();) {
MachineInstr &MI = *Next;
++Next;
if (MI.isPHI() || MI.isDebugInstr())
continue;
if (MI.mayStore())
SawStore = true;
unsigned CurrentOrder = IOM[&MI];
unsigned Barrier = 0;
MachineInstr *BarrierMI = nullptr;
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isReg() || MO.isDebug())
continue;
if (MO.isUse())
UseMap[MO.getReg()] = std::make_pair(CurrentOrder, &MI);
else if (MO.isDead() && UseMap.count(MO.getReg()))
// Barrier is the last instruction where MO get used. MI should not
// be moved above Barrier.
if (Barrier < UseMap[MO.getReg()].first) {
Barrier = UseMap[MO.getReg()].first;
BarrierMI = UseMap[MO.getReg()].second;
}
}
if (!MI.isSafeToMove(nullptr, SawStore)) {
// If MI has side effects, it should become a barrier for code motion.
// IOM is rebuild from the next instruction to prevent later
// instructions from being moved before this MI.
if (MI.hasUnmodeledSideEffects() && Next != MBB.end()) {
BuildInstOrderMap(Next, IOM);
SawStore = false;
}
continue;
}
const MachineOperand *DefMO = nullptr;
MachineInstr *Insert = nullptr;
// Number of live-ranges that will be shortened. We do not count
// live-ranges that are defined by a COPY as it could be coalesced later.
unsigned NumEligibleUse = 0;
for (const MachineOperand &MO : MI.operands()) {
if (!MO.isReg() || MO.isDead() || MO.isDebug())
continue;
unsigned Reg = MO.getReg();
// Do not move the instruction if it def/uses a physical register,
// unless it is a constant physical register or a noreg.
if (!TargetRegisterInfo::isVirtualRegister(Reg)) {
if (!Reg || MRI.isConstantPhysReg(Reg))
continue;
Insert = nullptr;
break;
}
if (MO.isDef()) {
// Do not move if there is more than one def.
if (DefMO) {
Insert = nullptr;
break;
}
DefMO = &MO;
} else if (MRI.hasOneNonDBGUse(Reg) && MRI.hasOneDef(Reg) && DefMO &&
MRI.getRegClass(DefMO->getReg()) ==
MRI.getRegClass(MO.getReg())) {
// The heuristic does not handle different register classes yet
// (registers of different sizes, looser/tighter constraints). This
// is because it needs more accurate model to handle register
// pressure correctly.
MachineInstr &DefInstr = *MRI.def_instr_begin(Reg);
if (!DefInstr.isCopy())
NumEligibleUse++;
Insert = FindDominatedInstruction(DefInstr, Insert, IOM);
} else {
Insert = nullptr;
break;
}
}
// If Barrier equals IOM[I], traverse forward to find if BarrierMI is
// after Insert, if yes, then we should not hoist.
for (MachineInstr *I = Insert; I && IOM[I] == Barrier;
I = I->getNextNode())
if (I == BarrierMI) {
Insert = nullptr;
break;
}
// Move the instruction when # of shrunk live range > 1.
if (DefMO && Insert && NumEligibleUse > 1 && Barrier <= IOM[Insert]) {
MachineBasicBlock::iterator I = std::next(Insert->getIterator());
// Skip all the PHI and debug instructions.
while (I != MBB.end() && (I->isPHI() || I->isDebugInstr()))
I = std::next(I);
if (I == MI.getIterator())
continue;
// Update the dominator order to be the same as the insertion point.
// We do this to maintain a non-decreasing order without need to update
// all instruction orders after the insertion point.
unsigned NewOrder = IOM[&*I];
IOM[&MI] = NewOrder;
NumInstrsHoistedToShrinkLiveRange++;
// Find MI's debug value following MI.
MachineBasicBlock::iterator EndIter = std::next(MI.getIterator());
if (MI.getOperand(0).isReg())
for (; EndIter != MBB.end() && EndIter->isDebugValue() &&
EndIter->getOperand(0).isReg() &&
EndIter->getOperand(0).getReg() == MI.getOperand(0).getReg();
++EndIter, ++Next)
IOM[&*EndIter] = NewOrder;
MBB.splice(I, &MBB, MI.getIterator(), EndIter);
}
}
}
return false;
}
void WinEHStatePass::addStateStores(Function &F, WinEHFuncInfo &FuncInfo) {
// Mark the registration node. The backend needs to know which alloca it is so
// that it can recover the original frame pointer.
IRBuilder<> Builder(RegNode->getNextNode());
Value *RegNodeI8 = Builder.CreateBitCast(RegNode, Builder.getInt8PtrTy());
Builder.CreateCall(
Intrinsic::getDeclaration(TheModule, Intrinsic::x86_seh_ehregnode),
{RegNodeI8});
if (EHGuardNode) {
IRBuilder<> Builder(EHGuardNode->getNextNode());
Value *EHGuardNodeI8 =
Builder.CreateBitCast(EHGuardNode, Builder.getInt8PtrTy());
Builder.CreateCall(
Intrinsic::getDeclaration(TheModule, Intrinsic::x86_seh_ehguard),
{EHGuardNodeI8});
}
// Calculate state numbers.
if (isAsynchronousEHPersonality(Personality))
calculateSEHStateNumbers(&F, FuncInfo);
else
calculateWinCXXEHStateNumbers(&F, FuncInfo);
// Iterate all the instructions and emit state number stores.
DenseMap<BasicBlock *, ColorVector> BlockColors = colorEHFunclets(F);
ReversePostOrderTraversal<Function *> RPOT(&F);
// InitialStates yields the state of the first call-site for a BasicBlock.
DenseMap<BasicBlock *, int> InitialStates;
// FinalStates yields the state of the last call-site for a BasicBlock.
DenseMap<BasicBlock *, int> FinalStates;
// Worklist used to revisit BasicBlocks with indeterminate
// Initial/Final-States.
std::deque<BasicBlock *> Worklist;
// Fill in InitialStates and FinalStates for BasicBlocks with call-sites.
for (BasicBlock *BB : RPOT) {
int InitialState = OverdefinedState;
int FinalState;
if (&F.getEntryBlock() == BB)
InitialState = FinalState = ParentBaseState;
for (Instruction &I : *BB) {
CallSite CS(&I);
if (!isStateStoreNeeded(Personality, CS))
continue;
int State = getStateForCallSite(BlockColors, FuncInfo, CS);
if (InitialState == OverdefinedState)
InitialState = State;
FinalState = State;
}
// No call-sites in this basic block? That's OK, we will come back to these
// in a later pass.
if (InitialState == OverdefinedState) {
Worklist.push_back(BB);
continue;
}
DEBUG(dbgs() << "X86WinEHState: " << BB->getName()
<< " InitialState=" << InitialState << '\n');
DEBUG(dbgs() << "X86WinEHState: " << BB->getName()
<< " FinalState=" << FinalState << '\n');
InitialStates.insert({BB, InitialState});
FinalStates.insert({BB, FinalState});
}
// Try to fill-in InitialStates and FinalStates which have no call-sites.
while (!Worklist.empty()) {
BasicBlock *BB = Worklist.front();
Worklist.pop_front();
// This BasicBlock has already been figured out, nothing more we can do.
if (InitialStates.count(BB) != 0)
continue;
int PredState = getPredState(FinalStates, F, ParentBaseState, BB);
if (PredState == OverdefinedState)
continue;
// We successfully inferred this BasicBlock's state via it's predecessors;
// enqueue it's successors to see if we can infer their states.
InitialStates.insert({BB, PredState});
FinalStates.insert({BB, PredState});
for (BasicBlock *SuccBB : successors(BB))
Worklist.push_back(SuccBB);
}
// Try to hoist stores from successors.
for (BasicBlock *BB : RPOT) {
int SuccState = getSuccState(InitialStates, F, ParentBaseState, BB);
if (SuccState == OverdefinedState)
continue;
// Update our FinalState to reflect the common InitialState of our
// successors.
FinalStates.insert({BB, SuccState});
}
// Finally, insert state stores before call-sites which transition us to a new
// state.
for (BasicBlock *BB : RPOT) {
auto &BBColors = BlockColors[BB];
BasicBlock *FuncletEntryBB = BBColors.front();
if (isa<CleanupPadInst>(FuncletEntryBB->getFirstNonPHI()))
continue;
int PrevState = getPredState(FinalStates, F, ParentBaseState, BB);
DEBUG(dbgs() << "X86WinEHState: " << BB->getName()
<< " PrevState=" << PrevState << '\n');
for (Instruction &I : *BB) {
CallSite CS(&I);
if (!isStateStoreNeeded(Personality, CS))
continue;
int State = getStateForCallSite(BlockColors, FuncInfo, CS);
if (State != PrevState)
insertStateNumberStore(&I, State);
PrevState = State;
}
// We might have hoisted a state store into this block, emit it now.
auto EndState = FinalStates.find(BB);
if (EndState != FinalStates.end())
if (EndState->second != PrevState)
insertStateNumberStore(BB->getTerminator(), EndState->second);
}
SmallVector<CallSite, 1> SetJmp3CallSites;
for (BasicBlock *BB : RPOT) {
for (Instruction &I : *BB) {
CallSite CS(&I);
if (!CS)
continue;
if (CS.getCalledValue()->stripPointerCasts() !=
SetJmp3->stripPointerCasts())
continue;
SetJmp3CallSites.push_back(CS);
}
}
for (CallSite CS : SetJmp3CallSites) {
auto &BBColors = BlockColors[CS->getParent()];
BasicBlock *FuncletEntryBB = BBColors.front();
bool InCleanup = isa<CleanupPadInst>(FuncletEntryBB->getFirstNonPHI());
IRBuilder<> Builder(CS.getInstruction());
Value *State;
if (InCleanup) {
Value *StateField =
Builder.CreateStructGEP(nullptr, RegNode, StateFieldIndex);
State = Builder.CreateLoad(StateField);
} else {
State = Builder.getInt32(getStateForCallSite(BlockColors, FuncInfo, CS));
}
rewriteSetJmpCallSite(Builder, F, CS, State);
}
}
bool InterleavedAccess::tryReplaceExtracts(
ArrayRef<ExtractElementInst *> Extracts,
ArrayRef<ShuffleVectorInst *> Shuffles) {
// If there aren't any extractelement instructions to modify, there's nothing
// to do.
if (Extracts.empty())
return true;
// Maps extractelement instructions to vector-index pairs. The extractlement
// instructions will be modified to use the new vector and index operands.
DenseMap<ExtractElementInst *, std::pair<Value *, int>> ReplacementMap;
for (auto *Extract : Extracts) {
// The vector index that is extracted.
auto *IndexOperand = cast<ConstantInt>(Extract->getIndexOperand());
auto Index = IndexOperand->getSExtValue();
// Look for a suitable shufflevector instruction. The goal is to modify the
// extractelement instruction (which uses an interleaved load) to use one
// of the shufflevector instructions instead of the load.
for (auto *Shuffle : Shuffles) {
// If the shufflevector instruction doesn't dominate the extract, we
// can't create a use of it.
if (!DT->dominates(Shuffle, Extract))
continue;
// Inspect the indices of the shufflevector instruction. If the shuffle
// selects the same index that is extracted, we can modify the
// extractelement instruction.
SmallVector<int, 4> Indices;
Shuffle->getShuffleMask(Indices);
for (unsigned I = 0; I < Indices.size(); ++I)
if (Indices[I] == Index) {
assert(Extract->getOperand(0) == Shuffle->getOperand(0) &&
"Vector operations do not match");
ReplacementMap[Extract] = std::make_pair(Shuffle, I);
break;
}
// If we found a suitable shufflevector instruction, stop looking.
if (ReplacementMap.count(Extract))
break;
}
// If we did not find a suitable shufflevector instruction, the
// extractelement instruction cannot be modified, so we must give up.
if (!ReplacementMap.count(Extract))
return false;
}
// Finally, perform the replacements.
IRBuilder<> Builder(Extracts[0]->getContext());
for (auto &Replacement : ReplacementMap) {
auto *Extract = Replacement.first;
auto *Vector = Replacement.second.first;
auto Index = Replacement.second.second;
Builder.SetInsertPoint(Extract);
Extract->replaceAllUsesWith(Builder.CreateExtractElement(Vector, Index));
Extract->eraseFromParent();
}
return true;
}
