void optimize(IRUnit& unit, IRBuilder& irBuilder, TransKind kind) {
auto finishPass = [&](const char* msg) {
dumpTrace(6, unit, folly::format("after {}", msg).str().c_str());
assert(checkCfg(unit));
assert(checkTmpsSpanningCalls(unit));
if (debug) {
forEachInst(rpoSortCfg(unit), assertOperandTypes);
}
};
auto doPass = [&](void (*fn)(IRUnit&), const char* msg) {
fn(unit);
finishPass(msg);
};
auto dce = [&](const char* which) {
if (!RuntimeOption::EvalHHIRDeadCodeElim) return;
eliminateDeadCode(unit);
finishPass(folly::format("{} DCE", which).str().c_str());
};
if (RuntimeOption::EvalHHIRRelaxGuards) {
auto const simpleRelax = kind == TransProfile;
auto changed = relaxGuards(unit, *irBuilder.guards(), simpleRelax);
if (changed) finishPass("guard relaxation");
}
if (RuntimeOption::EvalHHIRRefcountOpts) {
optimizeRefcounts(unit);
finishPass("refcount opts");
}
dce("initial");
if (RuntimeOption::EvalHHIRPredictionOpts) {
doPass(optimizePredictions, "prediction opts");
}
if (RuntimeOption::EvalHHIRExtraOptPass
&& (RuntimeOption::EvalHHIRCse
|| RuntimeOption::EvalHHIRSimplification)) {
irBuilder.reoptimize();
finishPass("reoptimize");
// Cleanup any dead code left around by CSE/Simplification
// Ideally, this would be controlled by a flag returned
// by optimzeTrace indicating whether DCE is necessary
dce("reoptimize");
}
if (RuntimeOption::EvalHHIRJumpOpts) {
doPass(optimizeJumps, "jumpopts");
dce("jump opts");
}
if (RuntimeOption::EvalHHIRGenerateAsserts) {
doPass(insertAsserts, "RefCnt asserts");
}
}
/*
* Records any type/reffiness predictions we depend on in the region. Guards
* for locals and stack cells that are not used will be eliminated by the call
* to relaxGuards.
*/
void RegionFormer::recordDependencies() {
// Record the incrementally constructed reffiness predictions.
assertx(!m_region->empty());
auto& frontBlock = *m_region->blocks().front();
for (auto const& dep : m_refDeps.m_arMap) {
frontBlock.addReffinessPred(m_startSk, {dep.second.m_mask,
dep.second.m_vals,
dep.first});
}
// Relax guards and record the ones that survived.
auto& firstBlock = *m_region->blocks().front();
auto blockStart = firstBlock.start();
auto& unit = m_irgs.unit;
auto const doRelax = RuntimeOption::EvalHHIRRelaxGuards;
bool changed = false;
if (doRelax) {
Timer _t(Timer::selectTracelet_relaxGuards);
// The IR is going to be discarded immediately, so skip reflowing
// the types in relaxGuards to save JIT time.
RelaxGuardsFlags flags = m_profiling ? RelaxSimple : RelaxNormal;
changed = relaxGuards(unit, *m_irgs.irb->guards(), flags);
}
auto guardMap = std::map<RegionDesc::Location,Type>{};
ITRACE(2, "Visiting guards\n");
visitGuards(unit, [&](const RegionDesc::Location& loc, Type type) {
Trace::Indent indent;
ITRACE(3, "{}: {}\n", show(loc), type);
if (type <= TCls) return;
auto inret = guardMap.insert(std::make_pair(loc, type));
if (inret.second) return;
auto& oldTy = inret.first->second;
if (oldTy == TGen) {
// This is the case that we see an inner type prediction for a GuardLoc
// that got relaxed to Gen.
return;
}
oldTy &= type;
});
for (auto& kv : guardMap) {
if (kv.second == TGen) {
// Guard was relaxed to Gen---don't record it.
continue;
}
auto const preCond = RegionDesc::TypedLocation { kv.first, kv.second };
ITRACE(1, "selectTracelet adding guard {}\n", show(preCond));
firstBlock.addPreCondition(blockStart, preCond);
}
if (changed) {
printUnit(3, unit, " after guard relaxation ", nullptr,
m_irgs.irb->guards());
}
}
void optimizeTrace(IRTrace* trace, TraceBuilder& traceBuilder) {
auto& irFactory = traceBuilder.factory();
auto finishPass = [&](const char* msg) {
dumpTrace(6, trace, folly::format("after {}", msg).str().c_str());
assert(checkCfg(trace, irFactory));
assert(checkTmpsSpanningCalls(trace, irFactory));
if (debug) forEachTraceInst(trace, assertOperandTypes);
};
auto doPass = [&](void (*fn)(IRTrace*, IRFactory&),
const char* msg) {
fn(trace, irFactory);
finishPass(msg);
};
auto dce = [&](const char* which) {
if (!RuntimeOption::EvalHHIRDeadCodeElim) return;
eliminateDeadCode(trace, irFactory);
finishPass(folly::format("{} DCE", which).str().c_str());
};
if (RuntimeOption::EvalHHIRRelaxGuards) {
auto changed = relaxGuards(trace, irFactory, *traceBuilder.guards());
if (changed) finishPass("guard relaxation");
}
dce("initial");
if (RuntimeOption::EvalHHIRPredictionOpts) {
doPass(optimizePredictions, "prediction opts");
}
if (RuntimeOption::EvalHHIRExtraOptPass
&& (RuntimeOption::EvalHHIRCse
|| RuntimeOption::EvalHHIRSimplification)) {
traceBuilder.reoptimize();
finishPass("reoptimize");
// Cleanup any dead code left around by CSE/Simplification
// Ideally, this would be controlled by a flag returned
// by optimzeTrace indicating whether DCE is necessary
dce("reoptimize");
}
if (RuntimeOption::EvalHHIRJumpOpts) {
doPass(optimizeJumps, "jumpopts");
dce("jump opts");
}
if (RuntimeOption::EvalHHIRGenerateAsserts) {
doPass(insertAsserts, "RefCnt asserts");
}
}
void optimize(IRUnit& unit, IRBuilder& irBuilder, TransKind kind) {
Timer _t(Timer::optimize);
auto const finishPass = [&] (const char* msg) {
if (msg) {
printUnit(6, unit, folly::format("after {}", msg).str().c_str());
}
assertx(checkCfg(unit));
assertx(checkTmpsSpanningCalls(unit));
if (debug) {
forEachInst(rpoSortCfg(unit), [&](IRInstruction* inst) {
assertx(checkOperandTypes(inst, &unit));
});
}
};
auto const doPass = [&] (void (*fn)(IRUnit&), const char* msg = nullptr) {
fn(unit);
finishPass(msg);
};
auto const dce = [&] (const char* which) {
if (!RuntimeOption::EvalHHIRDeadCodeElim) return;
eliminateDeadCode(unit);
finishPass(folly::format("{} DCE", which).str().c_str());
};
auto const simplifyPass = [] (IRUnit& unit) {
boost::dynamic_bitset<> reachable(unit.numBlocks());
reachable.set(unit.entry()->id());
auto const blocks = rpoSortCfg(unit);
for (auto block : blocks) {
// Skip unreachable blocks, or simplify() cries.
if (!reachable.test(block->id())) continue;
for (auto& inst : *block) simplify(unit, &inst);
if (auto const b = block->back().next()) reachable.set(b->id());
if (auto const b = block->back().taken()) reachable.set(b->id());
}
};
auto const doSimplify = RuntimeOption::EvalHHIRExtraOptPass &&
RuntimeOption::EvalHHIRSimplification;
auto const hasLoop = RuntimeOption::EvalJitLoops && cfgHasLoop(unit);
auto const traceMode = kind != TransKind::Optimize ||
RuntimeOption::EvalJitPGORegionSelector == "hottrace";
// TODO (#5792564): Guard relaxation doesn't work with loops.
// TODO (#6599498): Guard relaxation is broken in wholecfg mode.
if (shouldHHIRRelaxGuards() && !hasLoop && traceMode) {
Timer _t(Timer::optimize_relaxGuards);
const bool simple = kind == TransKind::Profile &&
(RuntimeOption::EvalJitRegionSelector == "tracelet" ||
RuntimeOption::EvalJitRegionSelector == "method");
RelaxGuardsFlags flags = (RelaxGuardsFlags)
(RelaxReflow | (simple ? RelaxSimple : RelaxNormal));
auto changed = relaxGuards(unit, *irBuilder.guards(), flags);
if (changed) finishPass("guard relaxation");
if (doSimplify) {
doPass(simplifyPass, "guard relaxation simplify");
}
}
// This is vestigial (it removes some instructions needed by the old refcount
// opts pass), and will be removed soon.
eliminateTakes(unit);
dce("initial");
if (RuntimeOption::EvalHHIRPredictionOpts) {
doPass(optimizePredictions, "prediction opts");
}
if (doSimplify) {
doPass(simplifyPass, "simplify");
dce("simplify");
}
if (RuntimeOption::EvalHHIRGlobalValueNumbering) {
doPass(gvn);
dce("gvn");
}
if (kind != TransKind::Profile && RuntimeOption::EvalHHIRMemoryOpts) {
doPass(optimizeLoads);
dce("loadelim");
}
/*
* Note: doing this pass this late might not be ideal, in particular because
* we've already turned some StLoc instructions into StLocNT.
*
* But right now there are assumptions preventing us from doing it before
* refcount opts. (Refcount opts needs to see all the StLocs explicitly
* because it makes assumptions about whether references are consumed based
* on that.)
*/
if (kind != TransKind::Profile && RuntimeOption::EvalHHIRMemoryOpts) {
doPass(optimizeStores);
dce("storeelim");
}
if (kind != TransKind::Profile && RuntimeOption::EvalHHIRRefcountOpts) {
doPass(optimizeRefcounts2);
dce("refcount");
}
if (RuntimeOption::EvalHHIRGenerateAsserts) {
doPass(insertAsserts);
}
}
void optimize(IRUnit& unit, IRBuilder& irBuilder, TransKind kind) {
Timer timer(Timer::optimize);
assertx(checkEverything(unit));
auto const hasLoop = RuntimeOption::EvalJitLoops && cfgHasLoop(unit);
auto const func = unit.entry()->front().marker().func();
auto const regionMode = pgoRegionMode(*func);
auto const traceMode = kind != TransKind::Optimize ||
regionMode == PGORegionMode::Hottrace;
// TODO (#5792564): Guard relaxation doesn't work with loops.
// TODO (#6599498): Guard relaxation is broken in wholecfg mode.
if (shouldHHIRRelaxGuards() && !hasLoop && traceMode) {
Timer _t(Timer::optimize_relaxGuards);
const bool simple = kind == TransKind::Profile &&
(RuntimeOption::EvalJitRegionSelector == "tracelet" ||
RuntimeOption::EvalJitRegionSelector == "method");
RelaxGuardsFlags flags = (RelaxGuardsFlags)
(RelaxReflow | (simple ? RelaxSimple : RelaxNormal));
auto changed = relaxGuards(unit, *irBuilder.guards(), flags);
if (changed) {
printUnit(6, unit, "after guard relaxation");
mandatoryDCE(unit); // relaxGuards can leave unreachable preds.
}
if (RuntimeOption::EvalHHIRSimplification) {
doPass(unit, simplifyPass, DCE::Minimal);
doPass(unit, cleanCfg, DCE::None);
}
}
fullDCE(unit);
printUnit(6, unit, " after initial DCE ");
assertx(checkEverything(unit));
if (RuntimeOption::EvalHHIRTypeCheckHoisting) {
doPass(unit, hoistTypeChecks, DCE::None);
}
if (RuntimeOption::EvalHHIRPredictionOpts) {
doPass(unit, optimizePredictions, DCE::None);
}
if (RuntimeOption::EvalHHIRSimplification) {
doPass(unit, simplifyPass, DCE::Full);
doPass(unit, cleanCfg, DCE::None);
}
if (RuntimeOption::EvalHHIRGlobalValueNumbering) {
doPass(unit, gvn, DCE::Full);
}
if (kind != TransKind::Profile && RuntimeOption::EvalHHIRMemoryOpts) {
doPass(unit, optimizeLoads, DCE::Full);
}
if (kind != TransKind::Profile && RuntimeOption::EvalHHIRMemoryOpts) {
doPass(unit, optimizeStores, DCE::Full);
}
if (kind != TransKind::Profile && RuntimeOption::EvalHHIRRefcountOpts) {
doPass(unit, optimizeRefcounts2, DCE::Full);
}
if (RuntimeOption::EvalHHIRLICM) {
if (kind != TransKind::Profile && hasLoop) {
// The clean pass is just to stress lack of pre_headers for now, since
// LICM is a disabled prototype pass.
doPass(unit, cleanCfg, DCE::None);
doPass(unit, optimizeLoopInvariantCode, DCE::Minimal);
}
}
doPass(unit, removeExitPlaceholders, DCE::Full);
if (RuntimeOption::EvalHHIRGenerateAsserts) {
doPass(unit, insertAsserts, DCE::None);
}
}
void optimize(IRUnit& unit, IRBuilder& irBuilder, TransKind kind) {
Timer _t(Timer::optimize);
auto finishPass = [&](const char* msg) {
if (msg) {
printUnit(6, unit, folly::format("after {}", msg).str().c_str());
}
assert(checkCfg(unit));
assert(checkTmpsSpanningCalls(unit));
if (debug) {
forEachInst(rpoSortCfg(unit), [&](IRInstruction* inst) {
assert(checkOperandTypes(inst, &unit));
});
}
};
auto doPass = [&](void (*fn)(IRUnit&), const char* msg = nullptr) {
fn(unit);
finishPass(msg);
};
auto dce = [&](const char* which) {
if (!RuntimeOption::EvalHHIRDeadCodeElim) return;
eliminateDeadCode(unit);
finishPass(folly::format("{} DCE", which).str().c_str());
};
auto const doReoptimize = RuntimeOption::EvalHHIRExtraOptPass &&
(RuntimeOption::EvalHHIRCse || RuntimeOption::EvalHHIRSimplification);
auto const hasLoop = RuntimeOption::EvalJitLoops && cfgHasLoop(unit);
// TODO(#5792564): Guard relaxation doesn't work with loops.
if (shouldHHIRRelaxGuards() && !hasLoop) {
Timer _t(Timer::optimize_relaxGuards);
const bool simple = kind == TransKind::Profile &&
(RuntimeOption::EvalJitRegionSelector == "tracelet" ||
RuntimeOption::EvalJitRegionSelector == "method");
RelaxGuardsFlags flags = (RelaxGuardsFlags)
(RelaxReflow | (simple ? RelaxSimple : RelaxNormal));
auto changed = relaxGuards(unit, *irBuilder.guards(), flags);
if (changed) finishPass("guard relaxation");
if (doReoptimize) {
irBuilder.reoptimize();
finishPass("guard relaxation reoptimize");
}
}
if (RuntimeOption::EvalHHIRRefcountOpts) {
optimizeRefcounts(unit, FrameStateMgr{unit.entry()->front().marker()});
finishPass("refcount opts");
}
dce("initial");
if (RuntimeOption::EvalHHIRPredictionOpts) {
doPass(optimizePredictions, "prediction opts");
}
if (doReoptimize) {
irBuilder.reoptimize();
finishPass("reoptimize");
dce("reoptimize");
}
if (RuntimeOption::EvalHHIRGlobalValueNumbering) {
doPass(gvn);
dce("gvn");
}
if (kind != TransKind::Profile && RuntimeOption::EvalHHIRMemoryOpts) {
doPass(optimizeLoads);
dce("loadelim");
}
/*
* Note: doing this pass this late might not be ideal, in particular because
* we've already turned some StLoc instructions into StLocNT.
*
* But right now there are assumptions preventing us from doing it before
* refcount opts. (Refcount opts needs to see all the StLocs explicitly
* because it makes assumptions about whether references are consumed based
* on that.)
*/
if (kind != TransKind::Profile && RuntimeOption::EvalHHIRMemoryOpts) {
doPass(optimizeStores);
dce("storeelim");
}
if (RuntimeOption::EvalHHIRGenerateAsserts) {
doPass(insertAsserts, "RefCnt asserts");
}
}
void optimize(IRUnit& unit, IRBuilder& irBuilder, TransKind kind) {
Timer _t(Timer::optimize);
auto finishPass = [&](const char* msg) {
dumpTrace(6, unit, folly::format("after {}", msg).str().c_str());
assert(checkCfg(unit));
assert(checkTmpsSpanningCalls(unit));
if (debug) {
forEachInst(rpoSortCfg(unit), assertOperandTypes);
}
};
auto doPass = [&](void (*fn)(IRUnit&), const char* msg) {
fn(unit);
finishPass(msg);
};
auto dce = [&](const char* which) {
if (!RuntimeOption::EvalHHIRDeadCodeElim) return;
eliminateDeadCode(unit);
finishPass(folly::format("{} DCE", which).str().c_str());
};
if (RuntimeOption::EvalHHIRRelaxGuards) {
/*
* In TransProfile mode, we can only relax the guards in tracelet
* region mode. If the region came from analyze() and we relax the
* guards here, then the RegionDesc's TypePreds in ProfData won't
* accurately reflect the generated guards. This can result in a
* TransOptimze region to be formed with types that are incompatible,
* e.g.:
* B1: TypePred: Loc0: Bool // but this gets relaxed to Uncounted
* PostCond: Loc0: Uncounted // post-conds are accurate
* B2: TypePred: Loc0: Int // this will always fail
*/
const bool relax = kind != TransProfile ||
RuntimeOption::EvalJitRegionSelector == "tracelet";
if (relax) {
Timer _t(Timer::optimize_relaxGuards);
const bool simple = kind == TransProfile &&
RuntimeOption::EvalJitRegionSelector == "tracelet";
auto changed = relaxGuards(unit, *irBuilder.guards(), simple);
if (changed) finishPass("guard relaxation");
}
}
if (RuntimeOption::EvalHHIRRefcountOpts) {
optimizeRefcounts(unit, FrameState{unit, unit.entry()->front().marker()});
finishPass("refcount opts");
}
dce("initial");
if (RuntimeOption::EvalHHIRPredictionOpts) {
doPass(optimizePredictions, "prediction opts");
}
if (RuntimeOption::EvalHHIRExtraOptPass
&& (RuntimeOption::EvalHHIRCse
|| RuntimeOption::EvalHHIRSimplification)) {
irBuilder.reoptimize();
finishPass("reoptimize");
// Cleanup any dead code left around by CSE/Simplification
// Ideally, this would be controlled by a flag returned
// by optimizeTrace indicating whether DCE is necessary
dce("reoptimize");
}
if (RuntimeOption::EvalHHIRJumpOpts) {
doPass(optimizeJumps, "jumpopts");
dce("jump opts");
}
if (RuntimeOption::EvalHHIRGenerateAsserts) {
doPass(insertAsserts, "RefCnt asserts");
}
}
