// Move the phi operands of Header from Latch out of AftBlocks to InsertLoc.
static void moveHeaderPhiOperandsToForeBlocks(BasicBlock *Header,
BasicBlock *Latch,
Instruction *InsertLoc,
BasicBlockSet &AftBlocks) {
// We need to ensure we move the instructions in the correct order,
// starting with the earliest required instruction and moving forward.
std::vector<Instruction *> Worklist;
std::vector<Instruction *> Visited;
for (auto &Phi : Header->phis()) {
Value *V = Phi.getIncomingValueForBlock(Latch);
if (Instruction *I = dyn_cast<Instruction>(V))
Worklist.push_back(I);
}
while (!Worklist.empty()) {
Instruction *I = Worklist.back();
Worklist.pop_back();
if (!AftBlocks.count(I->getParent()))
continue;
Visited.push_back(I);
for (auto &U : I->operands())
if (Instruction *II = dyn_cast<Instruction>(U))
Worklist.push_back(II);
}
// Move all instructions in program order to before the InsertLoc
BasicBlock *InsertLocBB = InsertLoc->getParent();
for (Instruction *I : reverse(Visited)) {
if (I->getParent() != InsertLocBB)
I->moveBefore(InsertLoc);
}
}
// Partition blocks in an outer/inner loop pair into blocks before and after
// the loop
static bool partitionOuterLoopBlocks(Loop *L, Loop *SubLoop,
BasicBlockSet &ForeBlocks,
BasicBlockSet &SubLoopBlocks,
BasicBlockSet &AftBlocks,
DominatorTree *DT) {
BasicBlock *SubLoopLatch = SubLoop->getLoopLatch();
SubLoopBlocks.insert(SubLoop->block_begin(), SubLoop->block_end());
for (BasicBlock *BB : L->blocks()) {
if (!SubLoop->contains(BB)) {
if (DT->dominates(SubLoopLatch, BB))
AftBlocks.insert(BB);
else
ForeBlocks.insert(BB);
}
}
// Check that all blocks in ForeBlocks together dominate the subloop
// TODO: This might ideally be done better with a dominator/postdominators.
BasicBlock *SubLoopPreHeader = SubLoop->getLoopPreheader();
for (BasicBlock *BB : ForeBlocks) {
if (BB == SubLoopPreHeader)
continue;
TerminatorInst *TI = BB->getTerminator();
for (unsigned i = 0, e = TI->getNumSuccessors(); i != e; ++i)
if (!ForeBlocks.count(TI->getSuccessor(i)))
return false;
}
return true;
}
LowerEmAsyncify::BasicBlockSet LowerEmAsyncify::FindReachableBlocksFrom(BasicBlock *src) {
BasicBlockSet ReachableBlockSet;
std::vector<BasicBlock*> pending;
ReachableBlockSet.insert(src);
pending.push_back(src);
while (!pending.empty()) {
BasicBlock *CurBlock = pending.back();
pending.pop_back();
for (succ_iterator SI = succ_begin(CurBlock), SE = succ_end(CurBlock); SI != SE; ++SI) {
if (ReachableBlockSet.count(*SI) == 0) {
ReachableBlockSet.insert(*SI);
pending.push_back(*SI);
}
}
}
return ReachableBlockSet;
}
void LowerEmAsyncify::transformAsyncFunction(Function &F, Instructions const& AsyncCalls) {
assert(!AsyncCalls.empty());
// Pass 0
// collect all the return instructions from the original function
// will use later
std::vector<ReturnInst*> OrigReturns;
for (inst_iterator I = inst_begin(&F), E = inst_end(&F); I != E; ++I) {
if (ReturnInst *RI = dyn_cast<ReturnInst>(&*I)) {
OrigReturns.push_back(RI);
}
}
// Pass 1
// Scan each async call and make the basic structure:
// All these will be cloned into the callback functions
// - allocate the async context before calling an async function
// - check async right after calling an async function, save context & return if async, continue if not
// - retrieve the async return value and free the async context if the called function turns out to be sync
std::vector<AsyncCallEntry> AsyncCallEntries;
AsyncCallEntries.reserve(AsyncCalls.size());
for (Instructions::const_iterator I = AsyncCalls.begin(), E = AsyncCalls.end(); I != E; ++I) {
// prepare blocks
Instruction *CurAsyncCall = *I;
// The block containing the async call
BasicBlock *CurBlock = CurAsyncCall->getParent();
// The block should run after the async call
BasicBlock *AfterCallBlock = SplitBlock(CurBlock, CurAsyncCall->getNextNode());
// The block where we store the context and return
BasicBlock *SaveAsyncCtxBlock = BasicBlock::Create(TheModule->getContext(), "SaveAsyncCtx", &F, AfterCallBlock);
// return a dummy value at the end, to make the block valid
new UnreachableInst(TheModule->getContext(), SaveAsyncCtxBlock);
// allocate the context before making the call
// we don't know the size yet, will fix it later
// we cannot insert the instruction later because,
// we need to make sure that all the instructions and blocks are fixed before we can generate DT and find context variables
// In CallHandler.h `sp` will be put as the second parameter
// such that we can take a note of the original sp
CallInst *AllocAsyncCtxInst = CallInst::Create(AllocAsyncCtxFunction, Constant::getNullValue(I32), "AsyncCtx", CurAsyncCall);
// Right after the call
// check async and return if so
// TODO: we can define truly async functions and partial async functions
{
// remove old terminator, which came from SplitBlock
CurBlock->getTerminator()->eraseFromParent();
// go to SaveAsyncCtxBlock if the previous call is async
// otherwise just continue to AfterCallBlock
CallInst *CheckAsync = CallInst::Create(CheckAsyncFunction, "IsAsync", CurBlock);
BranchInst::Create(SaveAsyncCtxBlock, AfterCallBlock, CheckAsync, CurBlock);
}
// take a note of this async call
AsyncCallEntry CurAsyncCallEntry;
CurAsyncCallEntry.AsyncCallInst = CurAsyncCall;
CurAsyncCallEntry.AfterCallBlock = AfterCallBlock;
CurAsyncCallEntry.AllocAsyncCtxInst = AllocAsyncCtxInst;
CurAsyncCallEntry.SaveAsyncCtxBlock = SaveAsyncCtxBlock;
// create an empty function for the callback, which will be constructed later
CurAsyncCallEntry.CallbackFunc = Function::Create(CallbackFunctionType, F.getLinkage(), F.getName() + "__async_cb", TheModule);
AsyncCallEntries.push_back(CurAsyncCallEntry);
}
// Pass 2
// analyze the context variables and construct SaveAsyncCtxBlock for each async call
// also calculate the size of the context and allocate the async context accordingly
for (std::vector<AsyncCallEntry>::iterator EI = AsyncCallEntries.begin(), EE = AsyncCallEntries.end(); EI != EE; ++EI) {
AsyncCallEntry & CurEntry = *EI;
// Collect everything to be saved
FindContextVariables(CurEntry);
// Pack the variables as a struct
{
// TODO: sort them from large memeber to small ones, in order to make the struct compact even when aligned
SmallVector<Type*, 8> Types;
Types.push_back(CallbackFunctionType->getPointerTo());
for (Values::iterator VI = CurEntry.ContextVariables.begin(), VE = CurEntry.ContextVariables.end(); VI != VE; ++VI) {
Types.push_back((*VI)->getType());
}
CurEntry.ContextStructType = StructType::get(TheModule->getContext(), Types);
}
// fix the size of allocation
CurEntry.AllocAsyncCtxInst->setOperand(0,
ConstantInt::get(I32, DL->getTypeStoreSize(CurEntry.ContextStructType)));
// construct SaveAsyncCtxBlock
{
// fill in SaveAsyncCtxBlock
// temporarily remove the terminator for convenience
CurEntry.SaveAsyncCtxBlock->getTerminator()->eraseFromParent();
assert(CurEntry.SaveAsyncCtxBlock->empty());
Type *AsyncCtxAddrTy = CurEntry.ContextStructType->getPointerTo();
BitCastInst *AsyncCtxAddr = new BitCastInst(CurEntry.AllocAsyncCtxInst, AsyncCtxAddrTy, "AsyncCtxAddr", CurEntry.SaveAsyncCtxBlock);
SmallVector<Value*, 2> Indices;
// store the callback
{
Indices.push_back(ConstantInt::get(I32, 0));
Indices.push_back(ConstantInt::get(I32, 0));
GetElementPtrInst *AsyncVarAddr = GetElementPtrInst::Create(AsyncCtxAddrTy, AsyncCtxAddr, Indices, "", CurEntry.SaveAsyncCtxBlock);
new StoreInst(CurEntry.CallbackFunc, AsyncVarAddr, CurEntry.SaveAsyncCtxBlock);
}
// store the context variables
for (size_t i = 0; i < CurEntry.ContextVariables.size(); ++i) {
Indices.clear();
Indices.push_back(ConstantInt::get(I32, 0));
Indices.push_back(ConstantInt::get(I32, i + 1)); // the 0th element is the callback function
GetElementPtrInst *AsyncVarAddr = GetElementPtrInst::Create(AsyncCtxAddrTy, AsyncCtxAddr, Indices, "", CurEntry.SaveAsyncCtxBlock);
new StoreInst(CurEntry.ContextVariables[i], AsyncVarAddr, CurEntry.SaveAsyncCtxBlock);
}
// to exit the block, we want to return without unwinding the stack frame
CallInst::Create(DoNotUnwindFunction, "", CurEntry.SaveAsyncCtxBlock);
ReturnInst::Create(TheModule->getContext(),
(F.getReturnType()->isVoidTy() ? 0 : Constant::getNullValue(F.getReturnType())),
CurEntry.SaveAsyncCtxBlock);
}
}
// Pass 3
// now all the SaveAsyncCtxBlock's have been constructed
// we can clone F and construct callback functions
// we could not construct the callbacks in Pass 2 because we need _all_ those SaveAsyncCtxBlock's appear in _each_ callback
for (std::vector<AsyncCallEntry>::iterator EI = AsyncCallEntries.begin(), EE = AsyncCallEntries.end(); EI != EE; ++EI) {
AsyncCallEntry & CurEntry = *EI;
Function *CurCallbackFunc = CurEntry.CallbackFunc;
ValueToValueMapTy VMap;
// Add the entry block
// load variables from the context
// also update VMap for CloneFunction
BasicBlock *EntryBlock = BasicBlock::Create(TheModule->getContext(), "AsyncCallbackEntry", CurCallbackFunc);
std::vector<LoadInst *> LoadedAsyncVars;
{
Type *AsyncCtxAddrTy = CurEntry.ContextStructType->getPointerTo();
BitCastInst *AsyncCtxAddr = new BitCastInst(CurCallbackFunc->arg_begin(), AsyncCtxAddrTy, "AsyncCtx", EntryBlock);
SmallVector<Value*, 2> Indices;
for (size_t i = 0; i < CurEntry.ContextVariables.size(); ++i) {
Indices.clear();
Indices.push_back(ConstantInt::get(I32, 0));
Indices.push_back(ConstantInt::get(I32, i + 1)); // the 0th element of AsyncCtx is the callback function
GetElementPtrInst *AsyncVarAddr = GetElementPtrInst::Create(AsyncCtxAddrTy, AsyncCtxAddr, Indices, "", EntryBlock);
LoadedAsyncVars.push_back(new LoadInst(AsyncVarAddr, "", EntryBlock));
// we want the argument to be replaced by the loaded value
if (isa<Argument>(CurEntry.ContextVariables[i]))
VMap[CurEntry.ContextVariables[i]] = LoadedAsyncVars.back();
}
}
// we don't need any argument, just leave dummy entries there to cheat CloneFunctionInto
for (Function::const_arg_iterator AI = F.arg_begin(), AE = F.arg_end(); AI != AE; ++AI) {
if (VMap.count(AI) == 0)
VMap[AI] = Constant::getNullValue(AI->getType());
}
// Clone the function
{
SmallVector<ReturnInst*, 8> Returns;
CloneFunctionInto(CurCallbackFunc, &F, VMap, false, Returns);
// return type of the callback functions is always void
// need to fix the return type
if (!F.getReturnType()->isVoidTy()) {
// for those return instructions that are from the original function
// it means we are 'truly' leaving this function
// need to store the return value right before ruturn
for (size_t i = 0; i < OrigReturns.size(); ++i) {
ReturnInst *RI = cast<ReturnInst>(VMap[OrigReturns[i]]);
// Need to store the return value into the global area
CallInst *RawRetValAddr = CallInst::Create(GetAsyncReturnValueAddrFunction, "", RI);
BitCastInst *RetValAddr = new BitCastInst(RawRetValAddr, F.getReturnType()->getPointerTo(), "AsyncRetValAddr", RI);
new StoreInst(RI->getOperand(0), RetValAddr, RI);
}
// we want to unwind the stack back to where it was before the original function as called
// but we don't actually need to do this here
// at this point it must be true that no callback is pended
// so the scheduler will correct the stack pointer and pop the frame
// here we just fix the return type
for (size_t i = 0; i < Returns.size(); ++i) {
ReplaceInstWithInst(Returns[i], ReturnInst::Create(TheModule->getContext()));
}
}
}
// the callback function does not have any return value
// so clear all the attributes for return
{
AttributeSet Attrs = CurCallbackFunc->getAttributes();
CurCallbackFunc->setAttributes(
Attrs.removeAttributes(TheModule->getContext(), AttributeSet::ReturnIndex, Attrs.getRetAttributes())
);
}
// in the callback function, we never allocate a new async frame
// instead we reuse the existing one
for (std::vector<AsyncCallEntry>::iterator EI = AsyncCallEntries.begin(), EE = AsyncCallEntries.end(); EI != EE; ++EI) {
Instruction *I = cast<Instruction>(VMap[EI->AllocAsyncCtxInst]);
ReplaceInstWithInst(I, CallInst::Create(ReallocAsyncCtxFunction, I->getOperand(0), "ReallocAsyncCtx"));
}
// mapped entry point & async call
BasicBlock *ResumeBlock = cast<BasicBlock>(VMap[CurEntry.AfterCallBlock]);
Instruction *MappedAsyncCall = cast<Instruction>(VMap[CurEntry.AsyncCallInst]);
// To save space, for each async call in the callback function, we just ignore the sync case, and leave it to the scheduler
// TODO need an option for this
{
for (std::vector<AsyncCallEntry>::iterator EI = AsyncCallEntries.begin(), EE = AsyncCallEntries.end(); EI != EE; ++EI) {
AsyncCallEntry & CurEntry = *EI;
Instruction *MappedAsyncCallInst = cast<Instruction>(VMap[CurEntry.AsyncCallInst]);
BasicBlock *MappedAsyncCallBlock = MappedAsyncCallInst->getParent();
BasicBlock *MappedAfterCallBlock = cast<BasicBlock>(VMap[CurEntry.AfterCallBlock]);
// for the sync case of the call, go to NewBlock (instead of MappedAfterCallBlock)
BasicBlock *NewBlock = BasicBlock::Create(TheModule->getContext(), "", CurCallbackFunc, MappedAfterCallBlock);
MappedAsyncCallBlock->getTerminator()->setSuccessor(1, NewBlock);
// store the return value
if (!MappedAsyncCallInst->use_empty()) {
CallInst *RawRetValAddr = CallInst::Create(GetAsyncReturnValueAddrFunction, "", NewBlock);
BitCastInst *RetValAddr = new BitCastInst(RawRetValAddr, MappedAsyncCallInst->getType()->getPointerTo(), "AsyncRetValAddr", NewBlock);
new StoreInst(MappedAsyncCallInst, RetValAddr, NewBlock);
}
// tell the scheduler that we want to keep the current async stack frame
CallInst::Create(DoNotUnwindAsyncFunction, "", NewBlock);
// finally we go to the SaveAsyncCtxBlock, to register the callbac, save the local variables and leave
BasicBlock *MappedSaveAsyncCtxBlock = cast<BasicBlock>(VMap[CurEntry.SaveAsyncCtxBlock]);
BranchInst::Create(MappedSaveAsyncCtxBlock, NewBlock);
}
}
std::vector<AllocaInst*> ToPromote;
// applying loaded variables in the entry block
{
BasicBlockSet ReachableBlocks = FindReachableBlocksFrom(ResumeBlock);
for (size_t i = 0; i < CurEntry.ContextVariables.size(); ++i) {
Value *OrigVar = CurEntry.ContextVariables[i];
if (isa<Argument>(OrigVar)) continue; // already processed
Value *CurVar = VMap[OrigVar];
assert(CurVar != MappedAsyncCall);
if (Instruction *Inst = dyn_cast<Instruction>(CurVar)) {
if (ReachableBlocks.count(Inst->getParent())) {
// Inst could be either defined or loaded from the async context
// Do the dirty works in memory
// TODO: might need to check the safety first
// TODO: can we create phi directly?
AllocaInst *Addr = DemoteRegToStack(*Inst, false);
new StoreInst(LoadedAsyncVars[i], Addr, EntryBlock);
ToPromote.push_back(Addr);
} else {
// The parent block is not reachable, which means there is no confliction
// it's safe to replace Inst with the loaded value
assert(Inst != LoadedAsyncVars[i]); // this should only happen when OrigVar is an Argument
Inst->replaceAllUsesWith(LoadedAsyncVars[i]);
}
}
}
}
// resolve the return value of the previous async function
// it could be the value just loaded from the global area
// or directly returned by the function (in its sync case)
if (!CurEntry.AsyncCallInst->use_empty()) {
// load the async return value
CallInst *RawRetValAddr = CallInst::Create(GetAsyncReturnValueAddrFunction, "", EntryBlock);
BitCastInst *RetValAddr = new BitCastInst(RawRetValAddr, MappedAsyncCall->getType()->getPointerTo(), "AsyncRetValAddr", EntryBlock);
LoadInst *RetVal = new LoadInst(RetValAddr, "AsyncRetVal", EntryBlock);
AllocaInst *Addr = DemoteRegToStack(*MappedAsyncCall, false);
new StoreInst(RetVal, Addr, EntryBlock);
ToPromote.push_back(Addr);
}
// TODO remove unreachable blocks before creating phi
// We go right to ResumeBlock from the EntryBlock
BranchInst::Create(ResumeBlock, EntryBlock);
/*
* Creating phi's
* Normal stack frames and async stack frames are interleaving with each other.
* In a callback function, if we call an async function, we might need to realloc the async ctx.
* at this point we don't want anything stored after the ctx,
* such that we can free and extend the ctx by simply update STACKTOP.
* Therefore we don't want any alloca's in callback functions.
*
*/
if (!ToPromote.empty()) {
DominatorTreeWrapperPass DTW;
DTW.runOnFunction(*CurCallbackFunc);
PromoteMemToReg(ToPromote, DTW.getDomTree());
}
removeUnreachableBlocks(*CurCallbackFunc);
}
// Pass 4
// Here are modifications to the original function, which we won't want to be cloned into the callback functions
for (std::vector<AsyncCallEntry>::iterator EI = AsyncCallEntries.begin(), EE = AsyncCallEntries.end(); EI != EE; ++EI) {
AsyncCallEntry & CurEntry = *EI;
// remove the frame if no async functinon has been called
CallInst::Create(FreeAsyncCtxFunction, CurEntry.AllocAsyncCtxInst, "", CurEntry.AfterCallBlock->getFirstNonPHI());
}
}
bool llvm::isSafeToUnrollAndJam(Loop *L, ScalarEvolution &SE, DominatorTree &DT,
DependenceInfo &DI) {
/* We currently handle outer loops like this:
|
ForeFirst <----\ }
Blocks | } ForeBlocks
ForeLast | }
| |
SubLoopFirst <\ | }
Blocks | | } SubLoopBlocks
SubLoopLast -/ | }
| |
AftFirst | }
Blocks | } AftBlocks
AftLast ------/ }
|
There are (theoretically) any number of blocks in ForeBlocks, SubLoopBlocks
and AftBlocks, providing that there is one edge from Fores to SubLoops,
one edge from SubLoops to Afts and a single outer loop exit (from Afts).
In practice we currently limit Aft blocks to a single block, and limit
things further in the profitablility checks of the unroll and jam pass.
Because of the way we rearrange basic blocks, we also require that
the Fore blocks on all unrolled iterations are safe to move before the
SubLoop blocks of all iterations. So we require that the phi node looping
operands of ForeHeader can be moved to at least the end of ForeEnd, so that
we can arrange cloned Fore Blocks before the subloop and match up Phi's
correctly.
i.e. The old order of blocks used to be F1 S1_1 S1_2 A1 F2 S2_1 S2_2 A2.
It needs to be safe to tranform this to F1 F2 S1_1 S2_1 S1_2 S2_2 A1 A2.
There are then a number of checks along the lines of no calls, no
exceptions, inner loop IV is consistent, etc. Note that for loops requiring
runtime unrolling, UnrollRuntimeLoopRemainder can also fail in
UnrollAndJamLoop if the trip count cannot be easily calculated.
*/
if (!L->isLoopSimplifyForm() || L->getSubLoops().size() != 1)
return false;
Loop *SubLoop = L->getSubLoops()[0];
if (!SubLoop->isLoopSimplifyForm())
return false;
BasicBlock *Header = L->getHeader();
BasicBlock *Latch = L->getLoopLatch();
BasicBlock *Exit = L->getExitingBlock();
BasicBlock *SubLoopHeader = SubLoop->getHeader();
BasicBlock *SubLoopLatch = SubLoop->getLoopLatch();
BasicBlock *SubLoopExit = SubLoop->getExitingBlock();
if (Latch != Exit)
return false;
if (SubLoopLatch != SubLoopExit)
return false;
if (Header->hasAddressTaken() || SubLoopHeader->hasAddressTaken())
return false;
// Split blocks into Fore/SubLoop/Aft based on dominators
BasicBlockSet SubLoopBlocks;
BasicBlockSet ForeBlocks;
BasicBlockSet AftBlocks;
if (!partitionOuterLoopBlocks(L, SubLoop, ForeBlocks, SubLoopBlocks,
AftBlocks, &DT))
return false;
// Aft blocks may need to move instructions to fore blocks, which becomes more
// difficult if there are multiple (potentially conditionally executed)
// blocks. For now we just exclude loops with multiple aft blocks.
if (AftBlocks.size() != 1)
return false;
// Check inner loop IV is consistent between all iterations
const SCEV *SubLoopBECountSC = SE.getExitCount(SubLoop, SubLoopLatch);
if (isa<SCEVCouldNotCompute>(SubLoopBECountSC) ||
!SubLoopBECountSC->getType()->isIntegerTy())
return false;
ScalarEvolution::LoopDisposition LD =
SE.getLoopDisposition(SubLoopBECountSC, L);
if (LD != ScalarEvolution::LoopInvariant)
return false;
// Check the loop safety info for exceptions.
LoopSafetyInfo LSI;
computeLoopSafetyInfo(&LSI, L);
if (LSI.MayThrow)
return false;
// We've ruled out the easy stuff and now need to check that there are no
// interdependencies which may prevent us from moving the:
// ForeBlocks before Subloop and AftBlocks.
// Subloop before AftBlocks.
// ForeBlock phi operands before the subloop
// Make sure we can move all instructions we need to before the subloop
SmallVector<Instruction *, 8> Worklist;
SmallPtrSet<Instruction *, 8> Visited;
for (auto &Phi : Header->phis()) {
Value *V = Phi.getIncomingValueForBlock(Latch);
if (Instruction *I = dyn_cast<Instruction>(V))
Worklist.push_back(I);
}
while (!Worklist.empty()) {
Instruction *I = Worklist.back();
Worklist.pop_back();
if (Visited.insert(I).second) {
if (SubLoop->contains(I->getParent()))
return false;
if (AftBlocks.count(I->getParent())) {
// If we hit a phi node in afts we know we are done (probably LCSSA)
if (isa<PHINode>(I))
return false;
if (I->mayHaveSideEffects() || I->mayReadOrWriteMemory())
return false;
for (auto &U : I->operands())
if (Instruction *II = dyn_cast<Instruction>(U))
Worklist.push_back(II);
}
}
}
// Check for memory dependencies which prohibit the unrolling we are doing.
// Because of the way we are unrolling Fore/Sub/Aft blocks, we need to check
// there are no dependencies between Fore-Sub, Fore-Aft, Sub-Aft and Sub-Sub.
if (!checkDependencies(L, ForeBlocks, SubLoopBlocks, AftBlocks, DI))
return false;
return true;
}
/*
This method performs Unroll and Jam. For a simple loop like:
for (i = ..)
Fore(i)
for (j = ..)
SubLoop(i, j)
Aft(i)
Instead of doing normal inner or outer unrolling, we do:
for (i = .., i+=2)
Fore(i)
Fore(i+1)
for (j = ..)
SubLoop(i, j)
SubLoop(i+1, j)
Aft(i)
Aft(i+1)
So the outer loop is essetially unrolled and then the inner loops are fused
("jammed") together into a single loop. This can increase speed when there
are loads in SubLoop that are invariant to i, as they become shared between
the now jammed inner loops.
We do this by spliting the blocks in the loop into Fore, Subloop and Aft.
Fore blocks are those before the inner loop, Aft are those after. Normal
Unroll code is used to copy each of these sets of blocks and the results are
combined together into the final form above.
isSafeToUnrollAndJam should be used prior to calling this to make sure the
unrolling will be valid. Checking profitablility is also advisable.
*/
LoopUnrollResult
llvm::UnrollAndJamLoop(Loop *L, unsigned Count, unsigned TripCount,
unsigned TripMultiple, bool UnrollRemainder,
LoopInfo *LI, ScalarEvolution *SE, DominatorTree *DT,
AssumptionCache *AC, OptimizationRemarkEmitter *ORE) {
// When we enter here we should have already checked that it is safe
BasicBlock *Header = L->getHeader();
assert(L->getSubLoops().size() == 1);
Loop *SubLoop = *L->begin();
// Don't enter the unroll code if there is nothing to do.
if (TripCount == 0 && Count < 2) {
LLVM_DEBUG(dbgs() << "Won't unroll; almost nothing to do\n");
return LoopUnrollResult::Unmodified;
}
assert(Count > 0);
assert(TripMultiple > 0);
assert(TripCount == 0 || TripCount % TripMultiple == 0);
// Are we eliminating the loop control altogether?
bool CompletelyUnroll = (Count == TripCount);
// We use the runtime remainder in cases where we don't know trip multiple
if (TripMultiple == 1 || TripMultiple % Count != 0) {
if (!UnrollRuntimeLoopRemainder(L, Count, /*AllowExpensiveTripCount*/ false,
/*UseEpilogRemainder*/ true,
UnrollRemainder, LI, SE, DT, AC, true)) {
LLVM_DEBUG(dbgs() << "Won't unroll-and-jam; remainder loop could not be "
"generated when assuming runtime trip count\n");
return LoopUnrollResult::Unmodified;
}
}
// Notify ScalarEvolution that the loop will be substantially changed,
// if not outright eliminated.
if (SE) {
SE->forgetLoop(L);
SE->forgetLoop(SubLoop);
}
using namespace ore;
// Report the unrolling decision.
if (CompletelyUnroll) {
LLVM_DEBUG(dbgs() << "COMPLETELY UNROLL AND JAMMING loop %"
<< Header->getName() << " with trip count " << TripCount
<< "!\n");
ORE->emit(OptimizationRemark(DEBUG_TYPE, "FullyUnrolled", L->getStartLoc(),
L->getHeader())
<< "completely unroll and jammed loop with "
<< NV("UnrollCount", TripCount) << " iterations");
} else {
auto DiagBuilder = [&]() {
OptimizationRemark Diag(DEBUG_TYPE, "PartialUnrolled", L->getStartLoc(),
L->getHeader());
return Diag << "unroll and jammed loop by a factor of "
<< NV("UnrollCount", Count);
};
LLVM_DEBUG(dbgs() << "UNROLL AND JAMMING loop %" << Header->getName()
<< " by " << Count);
if (TripMultiple != 1) {
LLVM_DEBUG(dbgs() << " with " << TripMultiple << " trips per branch");
ORE->emit([&]() {
return DiagBuilder() << " with " << NV("TripMultiple", TripMultiple)
<< " trips per branch";
});
} else {
LLVM_DEBUG(dbgs() << " with run-time trip count");
ORE->emit([&]() { return DiagBuilder() << " with run-time trip count"; });
}
LLVM_DEBUG(dbgs() << "!\n");
}
BasicBlock *Preheader = L->getLoopPreheader();
BasicBlock *LatchBlock = L->getLoopLatch();
BranchInst *BI = dyn_cast<BranchInst>(LatchBlock->getTerminator());
assert(Preheader && LatchBlock && Header);
assert(BI && !BI->isUnconditional());
bool ContinueOnTrue = L->contains(BI->getSuccessor(0));
BasicBlock *LoopExit = BI->getSuccessor(ContinueOnTrue);
bool SubLoopContinueOnTrue = SubLoop->contains(
SubLoop->getLoopLatch()->getTerminator()->getSuccessor(0));
// Partition blocks in an outer/inner loop pair into blocks before and after
// the loop
BasicBlockSet SubLoopBlocks;
BasicBlockSet ForeBlocks;
BasicBlockSet AftBlocks;
partitionOuterLoopBlocks(L, SubLoop, ForeBlocks, SubLoopBlocks, AftBlocks,
DT);
// We keep track of the entering/first and exiting/last block of each of
// Fore/SubLoop/Aft in each iteration. This helps make the stapling up of
// blocks easier.
std::vector<BasicBlock *> ForeBlocksFirst;
std::vector<BasicBlock *> ForeBlocksLast;
std::vector<BasicBlock *> SubLoopBlocksFirst;
std::vector<BasicBlock *> SubLoopBlocksLast;
std::vector<BasicBlock *> AftBlocksFirst;
std::vector<BasicBlock *> AftBlocksLast;
ForeBlocksFirst.push_back(Header);
ForeBlocksLast.push_back(SubLoop->getLoopPreheader());
SubLoopBlocksFirst.push_back(SubLoop->getHeader());
SubLoopBlocksLast.push_back(SubLoop->getExitingBlock());
AftBlocksFirst.push_back(SubLoop->getExitBlock());
AftBlocksLast.push_back(L->getExitingBlock());
// Maps Blocks[0] -> Blocks[It]
ValueToValueMapTy LastValueMap;
// Move any instructions from fore phi operands from AftBlocks into Fore.
moveHeaderPhiOperandsToForeBlocks(
Header, LatchBlock, SubLoop->getLoopPreheader()->getTerminator(),
AftBlocks);
// The current on-the-fly SSA update requires blocks to be processed in
// reverse postorder so that LastValueMap contains the correct value at each
// exit.
LoopBlocksDFS DFS(L);
DFS.perform(LI);
// Stash the DFS iterators before adding blocks to the loop.
LoopBlocksDFS::RPOIterator BlockBegin = DFS.beginRPO();
LoopBlocksDFS::RPOIterator BlockEnd = DFS.endRPO();
if (Header->getParent()->isDebugInfoForProfiling())
for (BasicBlock *BB : L->getBlocks())
for (Instruction &I : *BB)
if (!isa<DbgInfoIntrinsic>(&I))
if (const DILocation *DIL = I.getDebugLoc())
I.setDebugLoc(DIL->cloneWithDuplicationFactor(Count));
// Copy all blocks
for (unsigned It = 1; It != Count; ++It) {
std::vector<BasicBlock *> NewBlocks;
// Maps Blocks[It] -> Blocks[It-1]
DenseMap<Value *, Value *> PrevItValueMap;
for (LoopBlocksDFS::RPOIterator BB = BlockBegin; BB != BlockEnd; ++BB) {
ValueToValueMapTy VMap;
BasicBlock *New = CloneBasicBlock(*BB, VMap, "." + Twine(It));
Header->getParent()->getBasicBlockList().push_back(New);
if (ForeBlocks.count(*BB)) {
L->addBasicBlockToLoop(New, *LI);
if (*BB == ForeBlocksFirst[0])
ForeBlocksFirst.push_back(New);
if (*BB == ForeBlocksLast[0])
ForeBlocksLast.push_back(New);
} else if (SubLoopBlocks.count(*BB)) {
SubLoop->addBasicBlockToLoop(New, *LI);
if (*BB == SubLoopBlocksFirst[0])
SubLoopBlocksFirst.push_back(New);
if (*BB == SubLoopBlocksLast[0])
SubLoopBlocksLast.push_back(New);
} else if (AftBlocks.count(*BB)) {
L->addBasicBlockToLoop(New, *LI);
if (*BB == AftBlocksFirst[0])
AftBlocksFirst.push_back(New);
if (*BB == AftBlocksLast[0])
AftBlocksLast.push_back(New);
} else {
llvm_unreachable("BB being cloned should be in Fore/Sub/Aft");
}
// Update our running maps of newest clones
PrevItValueMap[New] = (It == 1 ? *BB : LastValueMap[*BB]);
LastValueMap[*BB] = New;
for (ValueToValueMapTy::iterator VI = VMap.begin(), VE = VMap.end();
VI != VE; ++VI) {
PrevItValueMap[VI->second] =
const_cast<Value *>(It == 1 ? VI->first : LastValueMap[VI->first]);
LastValueMap[VI->first] = VI->second;
}
NewBlocks.push_back(New);
// Update DomTree:
if (*BB == ForeBlocksFirst[0])
DT->addNewBlock(New, ForeBlocksLast[It - 1]);
else if (*BB == SubLoopBlocksFirst[0])
DT->addNewBlock(New, SubLoopBlocksLast[It - 1]);
else if (*BB == AftBlocksFirst[0])
DT->addNewBlock(New, AftBlocksLast[It - 1]);
else {
// Each set of blocks (Fore/Sub/Aft) will have the same internal domtree
// structure.
auto BBDomNode = DT->getNode(*BB);
auto BBIDom = BBDomNode->getIDom();
BasicBlock *OriginalBBIDom = BBIDom->getBlock();
assert(OriginalBBIDom);
assert(LastValueMap[cast<Value>(OriginalBBIDom)]);
DT->addNewBlock(
New, cast<BasicBlock>(LastValueMap[cast<Value>(OriginalBBIDom)]));
}
}
// Remap all instructions in the most recent iteration
for (BasicBlock *NewBlock : NewBlocks) {
for (Instruction &I : *NewBlock) {
::remapInstruction(&I, LastValueMap);
if (auto *II = dyn_cast<IntrinsicInst>(&I))
if (II->getIntrinsicID() == Intrinsic::assume)
AC->registerAssumption(II);
}
}
// Alter the ForeBlocks phi's, pointing them at the latest version of the
// value from the previous iteration's phis
for (PHINode &Phi : ForeBlocksFirst[It]->phis()) {
Value *OldValue = Phi.getIncomingValueForBlock(AftBlocksLast[It]);
assert(OldValue && "should have incoming edge from Aft[It]");
Value *NewValue = OldValue;
if (Value *PrevValue = PrevItValueMap[OldValue])
NewValue = PrevValue;
assert(Phi.getNumOperands() == 2);
Phi.setIncomingBlock(0, ForeBlocksLast[It - 1]);
Phi.setIncomingValue(0, NewValue);
Phi.removeIncomingValue(1);
}
}
// Now that all the basic blocks for the unrolled iterations are in place,
// finish up connecting the blocks and phi nodes. At this point LastValueMap
// is the last unrolled iterations values.
// Update Phis in BB from OldBB to point to NewBB
auto updatePHIBlocks = [](BasicBlock *BB, BasicBlock *OldBB,
BasicBlock *NewBB) {
for (PHINode &Phi : BB->phis()) {
int I = Phi.getBasicBlockIndex(OldBB);
Phi.setIncomingBlock(I, NewBB);
}
};
// Update Phis in BB from OldBB to point to NewBB and use the latest value
// from LastValueMap
auto updatePHIBlocksAndValues = [](BasicBlock *BB, BasicBlock *OldBB,
BasicBlock *NewBB,
ValueToValueMapTy &LastValueMap) {
for (PHINode &Phi : BB->phis()) {
for (unsigned b = 0; b < Phi.getNumIncomingValues(); ++b) {
if (Phi.getIncomingBlock(b) == OldBB) {
Value *OldValue = Phi.getIncomingValue(b);
if (Value *LastValue = LastValueMap[OldValue])
Phi.setIncomingValue(b, LastValue);
Phi.setIncomingBlock(b, NewBB);
break;
}
}
}
};
// Move all the phis from Src into Dest
auto movePHIs = [](BasicBlock *Src, BasicBlock *Dest) {
Instruction *insertPoint = Dest->getFirstNonPHI();
while (PHINode *Phi = dyn_cast<PHINode>(Src->begin()))
Phi->moveBefore(insertPoint);
};
// Update the PHI values outside the loop to point to the last block
updatePHIBlocksAndValues(LoopExit, AftBlocksLast[0], AftBlocksLast.back(),
LastValueMap);
// Update ForeBlocks successors and phi nodes
BranchInst *ForeTerm =
cast<BranchInst>(ForeBlocksLast.back()->getTerminator());
BasicBlock *Dest = SubLoopBlocksFirst[0];
ForeTerm->setSuccessor(0, Dest);
if (CompletelyUnroll) {
while (PHINode *Phi = dyn_cast<PHINode>(ForeBlocksFirst[0]->begin())) {
Phi->replaceAllUsesWith(Phi->getIncomingValueForBlock(Preheader));
Phi->getParent()->getInstList().erase(Phi);
}
} else {
// Update the PHI values to point to the last aft block
updatePHIBlocksAndValues(ForeBlocksFirst[0], AftBlocksLast[0],
AftBlocksLast.back(), LastValueMap);
}
for (unsigned It = 1; It != Count; It++) {
// Remap ForeBlock successors from previous iteration to this
BranchInst *ForeTerm =
cast<BranchInst>(ForeBlocksLast[It - 1]->getTerminator());
BasicBlock *Dest = ForeBlocksFirst[It];
ForeTerm->setSuccessor(0, Dest);
}
// Subloop successors and phis
BranchInst *SubTerm =
cast<BranchInst>(SubLoopBlocksLast.back()->getTerminator());
SubTerm->setSuccessor(!SubLoopContinueOnTrue, SubLoopBlocksFirst[0]);
SubTerm->setSuccessor(SubLoopContinueOnTrue, AftBlocksFirst[0]);
updatePHIBlocks(SubLoopBlocksFirst[0], ForeBlocksLast[0],
ForeBlocksLast.back());
updatePHIBlocks(SubLoopBlocksFirst[0], SubLoopBlocksLast[0],
SubLoopBlocksLast.back());
for (unsigned It = 1; It != Count; It++) {
// Replace the conditional branch of the previous iteration subloop with an
// unconditional one to this one
BranchInst *SubTerm =
cast<BranchInst>(SubLoopBlocksLast[It - 1]->getTerminator());
BranchInst::Create(SubLoopBlocksFirst[It], SubTerm);
SubTerm->eraseFromParent();
updatePHIBlocks(SubLoopBlocksFirst[It], ForeBlocksLast[It],
ForeBlocksLast.back());
updatePHIBlocks(SubLoopBlocksFirst[It], SubLoopBlocksLast[It],
SubLoopBlocksLast.back());
movePHIs(SubLoopBlocksFirst[It], SubLoopBlocksFirst[0]);
}
// Aft blocks successors and phis
BranchInst *Term = cast<BranchInst>(AftBlocksLast.back()->getTerminator());
if (CompletelyUnroll) {
BranchInst::Create(LoopExit, Term);
Term->eraseFromParent();
} else {
Term->setSuccessor(!ContinueOnTrue, ForeBlocksFirst[0]);
}
updatePHIBlocks(AftBlocksFirst[0], SubLoopBlocksLast[0],
SubLoopBlocksLast.back());
for (unsigned It = 1; It != Count; It++) {
// Replace the conditional branch of the previous iteration subloop with an
// unconditional one to this one
BranchInst *AftTerm =
cast<BranchInst>(AftBlocksLast[It - 1]->getTerminator());
BranchInst::Create(AftBlocksFirst[It], AftTerm);
AftTerm->eraseFromParent();
updatePHIBlocks(AftBlocksFirst[It], SubLoopBlocksLast[It],
SubLoopBlocksLast.back());
movePHIs(AftBlocksFirst[It], AftBlocksFirst[0]);
}
// Dominator Tree. Remove the old links between Fore, Sub and Aft, adding the
// new ones required.
if (Count != 1) {
SmallVector<DominatorTree::UpdateType, 4> DTUpdates;
DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete, ForeBlocksLast[0],
SubLoopBlocksFirst[0]);
DTUpdates.emplace_back(DominatorTree::UpdateKind::Delete,
SubLoopBlocksLast[0], AftBlocksFirst[0]);
DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert,
ForeBlocksLast.back(), SubLoopBlocksFirst[0]);
DTUpdates.emplace_back(DominatorTree::UpdateKind::Insert,
SubLoopBlocksLast.back(), AftBlocksFirst[0]);
DT->applyUpdates(DTUpdates);
}
// Merge adjacent basic blocks, if possible.
SmallPtrSet<BasicBlock *, 16> MergeBlocks;
MergeBlocks.insert(ForeBlocksLast.begin(), ForeBlocksLast.end());
MergeBlocks.insert(SubLoopBlocksLast.begin(), SubLoopBlocksLast.end());
MergeBlocks.insert(AftBlocksLast.begin(), AftBlocksLast.end());
while (!MergeBlocks.empty()) {
BasicBlock *BB = *MergeBlocks.begin();
BranchInst *Term = dyn_cast<BranchInst>(BB->getTerminator());
if (Term && Term->isUnconditional() && L->contains(Term->getSuccessor(0))) {
BasicBlock *Dest = Term->getSuccessor(0);
if (BasicBlock *Fold = foldBlockIntoPredecessor(Dest, LI, SE, DT)) {
// Don't remove BB and add Fold as they are the same BB
assert(Fold == BB);
(void)Fold;
MergeBlocks.erase(Dest);
} else
MergeBlocks.erase(BB);
} else
MergeBlocks.erase(BB);
}
// At this point, the code is well formed. We now do a quick sweep over the
// inserted code, doing constant propagation and dead code elimination as we
// go.
simplifyLoopAfterUnroll(SubLoop, true, LI, SE, DT, AC);
simplifyLoopAfterUnroll(L, !CompletelyUnroll && Count > 1, LI, SE, DT, AC);
NumCompletelyUnrolledAndJammed += CompletelyUnroll;
++NumUnrolledAndJammed;
#ifndef NDEBUG
// We shouldn't have done anything to break loop simplify form or LCSSA.
Loop *OuterL = L->getParentLoop();
Loop *OutestLoop = OuterL ? OuterL : (!CompletelyUnroll ? L : SubLoop);
assert(OutestLoop->isRecursivelyLCSSAForm(*DT, *LI));
if (!CompletelyUnroll)
assert(L->isLoopSimplifyForm());
assert(SubLoop->isLoopSimplifyForm());
assert(DT->verify());
#endif
// Update LoopInfo if the loop is completely removed.
if (CompletelyUnroll)
LI->erase(L);
return CompletelyUnroll ? LoopUnrollResult::FullyUnrolled
: LoopUnrollResult::PartiallyUnrolled;
}