// If one loop has very large self trip count
// we don't want to unroll it.
// self trip count means trip count divide by the parent's trip count. for example
// for (int i = 0; i < 16; i++) {
// for (int j = 0; j < 4; j++) {
// for (int k = 0; k < 2; k++) {
// ...
// }
// ...
// }
// The inner loops j and k could be unrolled, but the loop i will not be unrolled.
// The return value true means the L could be unrolled, otherwise, it could not
// be unrolled.
bool handleParentLoops(Loop *L, LPPassManager &LPM) {
Loop *currL = L;
ScalarEvolution *SE = &getAnalysis<ScalarEvolution>();
BasicBlock *ExitBlock = currL->getLoopLatch();
if (!ExitBlock || !L->isLoopExiting(ExitBlock))
ExitBlock = currL->getExitingBlock();
unsigned currTripCount = 0;
bool shouldUnroll = true;
if (ExitBlock)
currTripCount = SE->getSmallConstantTripCount(L, ExitBlock);
while(currL) {
Loop *parentL = currL->getParentLoop();
unsigned parentTripCount = 0;
if (parentL) {
BasicBlock *parentExitBlock = parentL->getLoopLatch();
if (!parentExitBlock || !parentL->isLoopExiting(parentExitBlock))
parentExitBlock = parentL->getExitingBlock();
if (parentExitBlock)
parentTripCount = SE->getSmallConstantTripCount(parentL, parentExitBlock);
}
if ((parentTripCount != 0 && currTripCount / parentTripCount > 16) ||
(currTripCount > 32)) {
if (currL == L)
shouldUnroll = false;
setUnrollID(currL, false);
if (currL != L)
LPM.deleteLoopFromQueue(currL);
}
currL = parentL;
currTripCount = parentTripCount;
}
return shouldUnroll;
}
bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
LoopInfo *LI = &getAnalysis<LoopInfo>();
ScalarEvolution *SE = &getAnalysis<ScalarEvolution>();
BasicBlock *Header = L->getHeader();
DEBUG(dbgs() << "Loop Unroll: F[" << Header->getParent()->getName()
<< "] Loop %" << Header->getName() << "\n");
(void)Header;
// Determine the current unrolling threshold. While this is normally set
// from UnrollThreshold, it is overridden to a smaller value if the current
// function is marked as optimize-for-size, and the unroll threshold was
// not user specified.
unsigned Threshold = CurrentThreshold;
if (!UserThreshold &&
Header->getParent()->hasFnAttr(Attribute::OptimizeForSize))
Threshold = OptSizeUnrollThreshold;
// Find trip count and trip multiple if count is not available
unsigned TripCount = 0;
unsigned TripMultiple = 1;
if (!NoSCEVUnroll) {
// Find "latch trip count". UnrollLoop assumes that control cannot exit
// via the loop latch on any iteration prior to TripCount. The loop may exit
// early via an earlier branch.
BasicBlock *LatchBlock = L->getLoopLatch();
if (LatchBlock) {
TripCount = SE->getSmallConstantTripCount(L, LatchBlock);
TripMultiple = SE->getSmallConstantTripMultiple(L, LatchBlock);
}
}
else {
TripCount = L->getSmallConstantTripCount();
if (TripCount == 0)
TripMultiple = L->getSmallConstantTripMultiple();
}
// Automatically select an unroll count.
unsigned Count = CurrentCount;
if (Count == 0) {
// Conservative heuristic: if we know the trip count, see if we can
// completely unroll (subject to the threshold, checked below); otherwise
// try to find greatest modulo of the trip count which is still under
// threshold value.
if (TripCount == 0)
return false;
Count = TripCount;
}
// Enforce the threshold.
if (Threshold != NoThreshold) {
const TargetData *TD = getAnalysisIfAvailable<TargetData>();
unsigned NumInlineCandidates;
unsigned LoopSize = ApproximateLoopSize(L, NumInlineCandidates, TD);
DEBUG(dbgs() << " Loop Size = " << LoopSize << "\n");
if (NumInlineCandidates != 0) {
DEBUG(dbgs() << " Not unrolling loop with inlinable calls.\n");
return false;
}
uint64_t Size = (uint64_t)LoopSize*Count;
if (TripCount != 1 && Size > Threshold) {
DEBUG(dbgs() << " Too large to fully unroll with count: " << Count
<< " because size: " << Size << ">" << Threshold << "\n");
if (!CurrentAllowPartial) {
DEBUG(dbgs() << " will not try to unroll partially because "
<< "-unroll-allow-partial not given\n");
return false;
}
// Reduce unroll count to be modulo of TripCount for partial unrolling
Count = Threshold / LoopSize;
while (Count != 0 && TripCount%Count != 0) {
Count--;
}
if (Count < 2) {
DEBUG(dbgs() << " could not unroll partially\n");
return false;
}
DEBUG(dbgs() << " partially unrolling with count: " << Count << "\n");
}
}
// Unroll the loop.
if (!UnrollLoop(L, Count, TripCount, TripMultiple, LI, &LPM))
return false;
return true;
}
bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
if (skipOptnoneFunction(L))
return false;
Function &F = *L->getHeader()->getParent();
LoopInfo *LI = &getAnalysis<LoopInfoWrapperPass>().getLoopInfo();
ScalarEvolution *SE = &getAnalysis<ScalarEvolution>();
const TargetTransformInfo &TTI =
getAnalysis<TargetTransformInfoWrapperPass>().getTTI(F);
auto &AC = getAnalysis<AssumptionCacheTracker>().getAssumptionCache(F);
BasicBlock *Header = L->getHeader();
DEBUG(dbgs() << "Loop Unroll: F[" << Header->getParent()->getName()
<< "] Loop %" << Header->getName() << "\n");
if (HasUnrollDisablePragma(L)) {
return false;
}
bool PragmaFullUnroll = HasUnrollFullPragma(L);
unsigned PragmaCount = UnrollCountPragmaValue(L);
bool HasPragma = PragmaFullUnroll || PragmaCount > 0;
TargetTransformInfo::UnrollingPreferences UP;
getUnrollingPreferences(L, TTI, UP);
// Find trip count and trip multiple if count is not available
unsigned TripCount = 0;
unsigned TripMultiple = 1;
// If there are multiple exiting blocks but one of them is the latch, use the
// latch for the trip count estimation. Otherwise insist on a single exiting
// block for the trip count estimation.
BasicBlock *ExitingBlock = L->getLoopLatch();
if (!ExitingBlock || !L->isLoopExiting(ExitingBlock))
ExitingBlock = L->getExitingBlock();
if (ExitingBlock) {
TripCount = SE->getSmallConstantTripCount(L, ExitingBlock);
TripMultiple = SE->getSmallConstantTripMultiple(L, ExitingBlock);
}
// Select an initial unroll count. This may be reduced later based
// on size thresholds.
bool CountSetExplicitly;
unsigned Count = selectUnrollCount(L, TripCount, PragmaFullUnroll,
PragmaCount, UP, CountSetExplicitly);
unsigned NumInlineCandidates;
bool notDuplicatable;
unsigned LoopSize =
ApproximateLoopSize(L, NumInlineCandidates, notDuplicatable, TTI, &AC);
DEBUG(dbgs() << " Loop Size = " << LoopSize << "\n");
// When computing the unrolled size, note that the conditional branch on the
// backedge and the comparison feeding it are not replicated like the rest of
// the loop body (which is why 2 is subtracted).
uint64_t UnrolledSize = (uint64_t)(LoopSize-2) * Count + 2;
if (notDuplicatable) {
DEBUG(dbgs() << " Not unrolling loop which contains non-duplicatable"
<< " instructions.\n");
return false;
}
if (NumInlineCandidates != 0) {
DEBUG(dbgs() << " Not unrolling loop with inlinable calls.\n");
return false;
}
unsigned Threshold, PartialThreshold;
unsigned PercentDynamicCostSavedThreshold;
unsigned DynamicCostSavingsDiscount;
selectThresholds(L, HasPragma, UP, Threshold, PartialThreshold,
PercentDynamicCostSavedThreshold,
DynamicCostSavingsDiscount);
// Given Count, TripCount and thresholds determine the type of
// unrolling which is to be performed.
enum { Full = 0, Partial = 1, Runtime = 2 };
int Unrolling;
if (TripCount && Count == TripCount) {
Unrolling = Partial;
// If the loop is really small, we don't need to run an expensive analysis.
if (canUnrollCompletely(L, Threshold, 100, DynamicCostSavingsDiscount,
UnrolledSize, UnrolledSize)) {
Unrolling = Full;
} else {
// The loop isn't that small, but we still can fully unroll it if that
// helps to remove a significant number of instructions.
// To check that, run additional analysis on the loop.
if (Optional<EstimatedUnrollCost> Cost = analyzeLoopUnrollCost(
L, TripCount, *SE, TTI, Threshold + DynamicCostSavingsDiscount))
if (canUnrollCompletely(L, Threshold, PercentDynamicCostSavedThreshold,
DynamicCostSavingsDiscount, Cost->UnrolledCost,
Cost->RolledDynamicCost)) {
Unrolling = Full;
}
}
} else if (TripCount && Count < TripCount) {
Unrolling = Partial;
} else {
Unrolling = Runtime;
}
// Reduce count based on the type of unrolling and the threshold values.
unsigned OriginalCount = Count;
bool AllowRuntime =
(PragmaCount > 0) || (UserRuntime ? CurrentRuntime : UP.Runtime);
// Don't unroll a runtime trip count loop with unroll full pragma.
if (HasRuntimeUnrollDisablePragma(L) || PragmaFullUnroll) {
AllowRuntime = false;
}
if (Unrolling == Partial) {
bool AllowPartial = UserAllowPartial ? CurrentAllowPartial : UP.Partial;
if (!AllowPartial && !CountSetExplicitly) {
DEBUG(dbgs() << " will not try to unroll partially because "
<< "-unroll-allow-partial not given\n");
return false;
}
if (PartialThreshold != NoThreshold && UnrolledSize > PartialThreshold) {
// Reduce unroll count to be modulo of TripCount for partial unrolling.
Count = (std::max(PartialThreshold, 3u)-2) / (LoopSize-2);
while (Count != 0 && TripCount % Count != 0)
Count--;
}
} else if (Unrolling == Runtime) {
if (!AllowRuntime && !CountSetExplicitly) {
DEBUG(dbgs() << " will not try to unroll loop with runtime trip count "
<< "-unroll-runtime not given\n");
return false;
}
// Reduce unroll count to be the largest power-of-two factor of
// the original count which satisfies the threshold limit.
while (Count != 0 && UnrolledSize > PartialThreshold) {
Count >>= 1;
UnrolledSize = (LoopSize-2) * Count + 2;
}
if (Count > UP.MaxCount)
Count = UP.MaxCount;
DEBUG(dbgs() << " partially unrolling with count: " << Count << "\n");
}
if (HasPragma) {
if (PragmaCount != 0)
// If loop has an unroll count pragma mark loop as unrolled to prevent
// unrolling beyond that requested by the pragma.
SetLoopAlreadyUnrolled(L);
// Emit optimization remarks if we are unable to unroll the loop
// as directed by a pragma.
DebugLoc LoopLoc = L->getStartLoc();
Function *F = Header->getParent();
LLVMContext &Ctx = F->getContext();
if (PragmaFullUnroll && PragmaCount == 0) {
if (TripCount && Count != TripCount) {
emitOptimizationRemarkMissed(
Ctx, DEBUG_TYPE, *F, LoopLoc,
"Unable to fully unroll loop as directed by unroll(full) pragma "
"because unrolled size is too large.");
} else if (!TripCount) {
emitOptimizationRemarkMissed(
Ctx, DEBUG_TYPE, *F, LoopLoc,
"Unable to fully unroll loop as directed by unroll(full) pragma "
"because loop has a runtime trip count.");
}
} else if (PragmaCount > 0 && Count != OriginalCount) {
emitOptimizationRemarkMissed(
Ctx, DEBUG_TYPE, *F, LoopLoc,
"Unable to unroll loop the number of times directed by "
"unroll_count pragma because unrolled size is too large.");
}
}
if (Unrolling != Full && Count < 2) {
// Partial unrolling by 1 is a nop. For full unrolling, a factor
// of 1 makes sense because loop control can be eliminated.
return false;
}
// Unroll the loop.
if (!UnrollLoop(L, Count, TripCount, AllowRuntime, UP.AllowExpensiveTripCount,
TripMultiple, LI, this, &LPM, &AC))
return false;
return true;
}
bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
if (skipOptnoneFunction(L))
return false;
LoopInfo *LI = &getAnalysis<LoopInfo>();
ScalarEvolution *SE = &getAnalysis<ScalarEvolution>();
const TargetTransformInfo &TTI = getAnalysis<TargetTransformInfo>();
BasicBlock *Header = L->getHeader();
DEBUG(dbgs() << "Loop Unroll: F[" << Header->getParent()->getName()
<< "] Loop %" << Header->getName() << "\n");
if (HasUnrollDisablePragma(L)) {
return false;
}
bool HasEnablePragma = HasUnrollEnablePragma(L);
unsigned PragmaCount = UnrollCountPragmaValue(L);
bool HasPragma = HasEnablePragma || PragmaCount > 0;
TargetTransformInfo::UnrollingPreferences UP;
getUnrollingPreferences(L, TTI, UP);
// Find trip count and trip multiple if count is not available
unsigned TripCount = 0;
unsigned TripMultiple = 1;
// Find "latch trip count". UnrollLoop assumes that control cannot exit
// via the loop latch on any iteration prior to TripCount. The loop may exit
// early via an earlier branch.
BasicBlock *LatchBlock = L->getLoopLatch();
if (LatchBlock) {
TripCount = SE->getSmallConstantTripCount(L, LatchBlock);
TripMultiple = SE->getSmallConstantTripMultiple(L, LatchBlock);
}
// Select an initial unroll count. This may be reduced later based
// on size thresholds.
bool CountSetExplicitly;
unsigned Count = selectUnrollCount(L, TripCount, HasEnablePragma, PragmaCount,
UP, CountSetExplicitly);
unsigned NumInlineCandidates;
bool notDuplicatable;
unsigned LoopSize =
ApproximateLoopSize(L, NumInlineCandidates, notDuplicatable, TTI);
DEBUG(dbgs() << " Loop Size = " << LoopSize << "\n");
uint64_t UnrolledSize = (uint64_t)LoopSize * Count;
if (notDuplicatable) {
DEBUG(dbgs() << " Not unrolling loop which contains non-duplicatable"
<< " instructions.\n");
return false;
}
if (NumInlineCandidates != 0) {
DEBUG(dbgs() << " Not unrolling loop with inlinable calls.\n");
return false;
}
unsigned Threshold, PartialThreshold;
selectThresholds(L, HasPragma, UP, Threshold, PartialThreshold);
// Given Count, TripCount and thresholds determine the type of
// unrolling which is to be performed.
enum { Full = 0, Partial = 1, Runtime = 2 };
int Unrolling;
if (TripCount && Count == TripCount) {
if (Threshold != NoThreshold && UnrolledSize > Threshold) {
DEBUG(dbgs() << " Too large to fully unroll with count: " << Count
<< " because size: " << UnrolledSize << ">" << Threshold
<< "\n");
Unrolling = Partial;
} else {
Unrolling = Full;
}
} else if (TripCount && Count < TripCount) {
Unrolling = Partial;
} else {
Unrolling = Runtime;
}
// Reduce count based on the type of unrolling and the threshold values.
unsigned OriginalCount = Count;
bool AllowRuntime = UserRuntime ? CurrentRuntime : UP.Runtime;
if (Unrolling == Partial) {
bool AllowPartial = UserAllowPartial ? CurrentAllowPartial : UP.Partial;
if (!AllowPartial && !CountSetExplicitly) {
DEBUG(dbgs() << " will not try to unroll partially because "
<< "-unroll-allow-partial not given\n");
return false;
}
if (PartialThreshold != NoThreshold && UnrolledSize > PartialThreshold) {
// Reduce unroll count to be modulo of TripCount for partial unrolling.
Count = PartialThreshold / LoopSize;
while (Count != 0 && TripCount % Count != 0)
Count--;
}
} else if (Unrolling == Runtime) {
if (!AllowRuntime && !CountSetExplicitly) {
DEBUG(dbgs() << " will not try to unroll loop with runtime trip count "
<< "-unroll-runtime not given\n");
return false;
}
// Reduce unroll count to be the largest power-of-two factor of
// the original count which satisfies the threshold limit.
while (Count != 0 && UnrolledSize > PartialThreshold) {
Count >>= 1;
UnrolledSize = LoopSize * Count;
}
if (Count > UP.MaxCount)
Count = UP.MaxCount;
DEBUG(dbgs() << " partially unrolling with count: " << Count << "\n");
}
if (HasPragma) {
// Emit optimization remarks if we are unable to unroll the loop
// as directed by a pragma.
DebugLoc LoopLoc = L->getStartLoc();
Function *F = Header->getParent();
LLVMContext &Ctx = F->getContext();
if (HasEnablePragma && PragmaCount == 0) {
if (TripCount && Count != TripCount) {
emitOptimizationRemarkMissed(
Ctx, DEBUG_TYPE, *F, LoopLoc,
"Unable to fully unroll loop as directed by unroll(enable) pragma "
"because unrolled size is too large.");
} else if (!TripCount) {
emitOptimizationRemarkMissed(
Ctx, DEBUG_TYPE, *F, LoopLoc,
"Unable to fully unroll loop as directed by unroll(enable) pragma "
"because loop has a runtime trip count.");
}
} else if (PragmaCount > 0 && Count != OriginalCount) {
emitOptimizationRemarkMissed(
Ctx, DEBUG_TYPE, *F, LoopLoc,
"Unable to unroll loop the number of times directed by "
"unroll_count pragma because unrolled size is too large.");
}
}
if (Unrolling != Full && Count < 2) {
// Partial unrolling by 1 is a nop. For full unrolling, a factor
// of 1 makes sense because loop control can be eliminated.
return false;
}
// Unroll the loop.
if (!UnrollLoop(L, Count, TripCount, AllowRuntime, TripMultiple, LI, this, &LPM))
return false;
return true;
}
bool CustomUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
errs() << "Entering loop unroll\n";
LoopInfo *LI = &getAnalysis<LoopInfo>();
// LP = &getAnalysis<LAMPLoadProfile>();
// LL = &getAnalysis<label_loop>();
ScalarEvolution *SE = &getAnalysis<ScalarEvolution>();
const TargetTransformInfo &TTI = getAnalysis<TargetTransformInfo>();
BasicBlock *Header = L->getHeader();
DEBUG(dbgs() << "Loop Unroll: F[" << Header->getParent()->getName()
<< "] Loop %" << Header->getName() << "\n");
(void)Header;
// errs() << "ID of this loop is " << LP->LoopToIdMap[Header] << "\n";
// errs() << "ID of this loop is " << LL->LoopToIdMap[Header] << "\n";
// Determine the current unrolling threshold. While this is normally set
// from UnrollThreshold, it is overridden to a smaller value if the current
// function is marked as optimize-for-size, and the unroll threshold was
// not user specified.
// unsigned Threshold = CurrentThreshold;
// if (!UserThreshold &&
// Header->getParent()->getAttributes().
// hasAttribute(AttributeSet::FunctionIndex,
// Attribute::OptimizeForSize))
// Threshold = OptSizeUnrollThreshold;
// Find trip count and trip multiple if count is not available
unsigned TripCount = 0;
unsigned TripMultiple = 1;
// Find "latch trip count". UnrollLoop assumes that control cannot exit
// via the loop latch on any iteration prior to TripCount. The loop may exit
// early via an earlier branch.
BasicBlock *LatchBlock = L->getLoopLatch();
if (LatchBlock) {
errs() << "Assigning tripcount and tripmultiple\n";
TripCount = SE->getSmallConstantTripCount(L, LatchBlock);
TripMultiple = SE->getSmallConstantTripMultiple(L, LatchBlock);
}
// Use a default unroll-count if the user doesn't specify a value
// and the trip count is a run-time value. The default is different
// for run-time or compile-time trip count loops.
unsigned Count = CurrentCount;
// if (UnrollRuntime && CurrentCount == 0 && TripCount == 0)
// Count = UnrollRuntimeCount;
errs() << "Chaecking for tripcount, count is " << Count << " tripcount is " << TripCount << "\n";
if (Count == 0) {
// Conservative heuristic: if we know the trip count, see if we can
// completely unroll (subject to the threshold, checked below); otherwise
// try to find greatest modulo of the trip count which is still under
// threshold value.
if (TripCount == 0)
return false;
Count = TripCount;
}
errs() << "Chaecking for threshold\n";
// Enforce the threshold.
// if (Threshold != NoThreshold) {
unsigned NumInlineCandidates;
bool notDuplicatable;
unsigned LoopSize = ApproximateLoopSize(L, NumInlineCandidates,
notDuplicatable, TTI);
errs() << "Approximate loop size is " << LoopSize << "\n";
DEBUG(dbgs() << " Loop Size = " << LoopSize << "\n");
if (notDuplicatable) {
DEBUG(dbgs() << " Not unrolling loop which contains non duplicatable"
<< " instructions.\n");
return false;
}
// if (NumInlineCandidates != 0) {
// DEBUG(dbgs() << " Not unrolling loop with inlinable calls.\n");
// return false;
// }
// uint64_t Size = (uint64_t)LoopSize*Count;
// if (TripCount != 1 && Size > Threshold) {
// DEBUG(dbgs() << " Too large to fully unroll with count: " << Count
// << " because size: " << Size << ">" << Threshold << "\n");
// if (!CurrentAllowPartial && !(UnrollRuntime && TripCount == 0)) {
// DEBUG(dbgs() << " will not try to unroll partially because "
// << "-custom-allow-partial not given\n");
// return false;
// }
// if (TripCount) {
// // Reduce unroll count to be modulo of TripCount for partial unrolling
// Count = Threshold / LoopSize;
// while (Count != 0 && TripCount%Count != 0)
// Count--;
// }
// else if (UnrollRuntime) {
// // Reduce unroll count to be a lower power-of-two value
// while (Count != 0 && Size > Threshold) {
// Count >>= 1;
// Size = LoopSize*Count;
// }
// }
// if (Count < 2) {
// DEBUG(dbgs() << " could not unroll partially\n");
// return false;
// }
// DEBUG(dbgs() << " partially unrolling with count: " << Count << "\n");
// }
// }
errs() << "Unrolling the loop\n";
errs()<< "Count is " << Count << ", tripcount is " << TripCount << ", allow runtime is " << UnrollRuntime << ", trip multiple is " << TripMultiple << "\n";
// Unroll the loop.
if (!UnrollLoop(L, Count, TripCount, UnrollRuntime, TripMultiple, LI, &LPM)) {
errs() << "Unrolling the loop failed\n";
return false;
}
return true;
}
bool LoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
if (skipOptnoneFunction(L))
return false;
LoopInfo *LI = &getAnalysis<LoopInfo>();
ScalarEvolution *SE = &getAnalysis<ScalarEvolution>();
const TargetTransformInfo &TTI = getAnalysis<TargetTransformInfo>();
BasicBlock *Header = L->getHeader();
DEBUG(dbgs() << "Loop Unroll: F[" << Header->getParent()->getName()
<< "] Loop %" << Header->getName() << "\n");
(void)Header;
TargetTransformInfo::UnrollingPreferences UP;
UP.Threshold = CurrentThreshold;
UP.OptSizeThreshold = OptSizeUnrollThreshold;
UP.Count = CurrentCount;
UP.Partial = CurrentAllowPartial;
UP.Runtime = CurrentRuntime;
TTI.getUnrollingPreferences(L, UP);
// Determine the current unrolling threshold. While this is normally set
// from UnrollThreshold, it is overridden to a smaller value if the current
// function is marked as optimize-for-size, and the unroll threshold was
// not user specified.
unsigned Threshold = UserThreshold ? CurrentThreshold : UP.Threshold;
if (!UserThreshold &&
Header->getParent()->getAttributes().
hasAttribute(AttributeSet::FunctionIndex,
Attribute::OptimizeForSize))
Threshold = UP.OptSizeThreshold;
// Find trip count and trip multiple if count is not available
unsigned TripCount = 0;
unsigned TripMultiple = 1;
// Find "latch trip count". UnrollLoop assumes that control cannot exit
// via the loop latch on any iteration prior to TripCount. The loop may exit
// early via an earlier branch.
BasicBlock *LatchBlock = L->getLoopLatch();
if (LatchBlock) {
TripCount = SE->getSmallConstantTripCount(L, LatchBlock);
TripMultiple = SE->getSmallConstantTripMultiple(L, LatchBlock);
}
bool Runtime = UserRuntime ? CurrentRuntime : UP.Runtime;
// Use a default unroll-count if the user doesn't specify a value
// and the trip count is a run-time value. The default is different
// for run-time or compile-time trip count loops.
unsigned Count = UserCount ? CurrentCount : UP.Count;
if (Runtime && Count == 0 && TripCount == 0)
Count = UnrollRuntimeCount;
if (Count == 0) {
// Conservative heuristic: if we know the trip count, see if we can
// completely unroll (subject to the threshold, checked below); otherwise
// try to find greatest modulo of the trip count which is still under
// threshold value.
if (TripCount == 0)
return false;
Count = TripCount;
}
// Enforce the threshold.
if (Threshold != NoThreshold) {
unsigned NumInlineCandidates;
bool notDuplicatable;
unsigned LoopSize = ApproximateLoopSize(L, NumInlineCandidates,
notDuplicatable, TTI);
DEBUG(dbgs() << " Loop Size = " << LoopSize << "\n");
if (notDuplicatable) {
DEBUG(dbgs() << " Not unrolling loop which contains non-duplicatable"
<< " instructions.\n");
return false;
}
if (NumInlineCandidates != 0) {
DEBUG(dbgs() << " Not unrolling loop with inlinable calls.\n");
return false;
}
uint64_t Size = (uint64_t)LoopSize*Count;
if (TripCount != 1 && Size > Threshold) {
DEBUG(dbgs() << " Too large to fully unroll with count: " << Count
<< " because size: " << Size << ">" << Threshold << "\n");
bool AllowPartial = UserAllowPartial ? CurrentAllowPartial : UP.Partial;
if (!AllowPartial && !(Runtime && TripCount == 0)) {
DEBUG(dbgs() << " will not try to unroll partially because "
<< "-unroll-allow-partial not given\n");
return false;
}
if (TripCount) {
// Reduce unroll count to be modulo of TripCount for partial unrolling
Count = Threshold / LoopSize;
while (Count != 0 && TripCount%Count != 0)
Count--;
}
else if (Runtime) {
// Reduce unroll count to be a lower power-of-two value
while (Count != 0 && Size > Threshold) {
Count >>= 1;
Size = LoopSize*Count;
}
}
if (Count < 2) {
DEBUG(dbgs() << " could not unroll partially\n");
return false;
}
DEBUG(dbgs() << " partially unrolling with count: " << Count << "\n");
}
bool TrivialLoopUnroll::runOnLoop(Loop *L, LPPassManager &LPM) {
// Only unroll the deepest loops in the loop nest.
if (!L->empty()) return false;
LoopInfo *LI = &getAnalysis<LoopInfo>();
ScalarEvolution *SE = &getAnalysis<ScalarEvolution>();
BasicBlock *Header = L->getHeader();
DEBUG(dbgs() << "Loop Unroll: F[" << Header->getParent()->getName()
<< "] Loop %" << Header->getName() << "\n");
(void)Header;
// Find trip count and trip multiple if count is not available
unsigned TripCount = 0;
unsigned TripMultiple = 1;
// Find "latch trip count". UnrollLoop assumes that control cannot exit
// via the loop latch on any iteration prior to TripCount. The loop may exit
// early via an earlier branch.
BasicBlock *LatchBlock = L->getLoopLatch();
if (LatchBlock) {
TripCount = SE->getSmallConstantTripCount(L, LatchBlock);
TripMultiple = SE->getSmallConstantTripMultiple(L, LatchBlock);
}
// Use a default unroll-count if the user doesn't specify a value
// and the trip count is a run-time value. The default is different
// for run-time or compile-time trip count loops.
// Conservative heuristic: if we know the trip count, see if we can
// completely unroll (subject to the threshold, checked below); otherwise
// try to find greatest modulo of the trip count which is still under
// threshold value.
if (TripCount == 0)
return false;
unsigned Count = TripCount;
LoopMetrics Metrics(L, &getAnalysis<TargetData>(), *SE);
if (!Metrics.initialize(LI, &getAnalysis<AliasAnalysis>())) {
DEBUG(dbgs() << " Not unrolling loop with strange instructions.\n");
return false;
}
// FIXME: Read the threshold from the constraints script.
uint64_t Threshold = 256000;
if (TripCount != 1 && !Metrics.isUnrollAccaptable(Count, Threshold)) {
DEBUG(dbgs() << " Too large to fully unroll with count: " << Count
<< " because size >" << Threshold << "\n");
if (TripCount) {
// Search a feasible count by binary search.
unsigned MaxCount = Count, MinCount = 1;
while (MinCount <= MaxCount) {
unsigned MidCount = MinCount + (MaxCount - MinCount) / 2;
if (Metrics.isUnrollAccaptable(MidCount, Threshold)) {
// MidCount is ok, try a bigger one.
Count = MidCount;
MinCount = MidCount + 1;
} else
// Else we had to try a smaller count.
MaxCount = MidCount - 1;
}
// Reduce unroll count to be modulo of TripCount for partial unrolling
while (Count != 0 && TripCount % Count != 0)
--Count;
}
if (Count < 2) {
DEBUG(dbgs() << " could not unroll partially\n");
return false;
}
DEBUG(dbgs() << " partially unrolling with count: " << Count << "\n");
}
//assert(TripCount % Count == 0 && "Bad unroll count!");
//assert(Metrics.isUnrollAccaptable(Count, Threshold) && "Bad unroll count!");
// Unroll the loop.
if (!UnrollLoop(L, Count, TripCount, false, TripMultiple, LI, &LPM))
return false;
return true;
}
