/**
* Regionize a func, so that each node and each arc in its TransCFG is
* "covered". A node is covered if any region contains it. An arc T1->T2
* is covered if either:
*
* a) T1 and T2 are in the same region R and T2 immediately follows
* T1 in R.
* b) T2 is the head (first translation) of a region.
*
* Basic algorithm:
*
* 1) sort nodes in decreasing weight order
* 2) for each node N:
* 2.1) if N and all its incoming arcs are covered, then continue
* 2.2) select a region starting at this node and mark nodes/arcs as
* covered appropriately
*/
void regionizeFunc(const Func* func,
MCGenerator* mcg,
RegionVec& regions) {
const Timer rf_timer(Timer::regionizeFunc);
assert(RuntimeOption::EvalJitPGO);
auto const funcId = func->getFuncId();
auto const profData = mcg->tx().profData();
TransCFG cfg(funcId, profData, mcg->tx().getSrcDB(),
mcg->getJmpToTransIDMap());
if (Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
auto dotFileName = folly::to<std::string>(
"/tmp/func-cfg-", funcId, ".dot");
cfg.print(dotFileName, funcId, profData, nullptr);
FTRACE(5, "regionizeFunc: initial CFG for func {} saved to file {}\n",
funcId, dotFileName);
}
TransCFG::ArcPtrVec arcs = cfg.arcs();
std::vector<TransID> nodes = cfg.nodes();
std::sort(nodes.begin(), nodes.end(),
[&](TransID tid1, TransID tid2) -> bool {
if (RuntimeOption::EvalJitPGORegionSelector == "wholecfg") {
auto bcOff1 = profData->transStartBcOff(tid1);
auto bcOff2 = profData->transStartBcOff(tid2);
if (bcOff1 != bcOff2) return bcOff1 < bcOff2;
}
if (cfg.weight(tid1) != cfg.weight(tid2)) {
return cfg.weight(tid1) > cfg.weight(tid2);
}
// In case of ties, pick older translations first, in an
// attempt to start loops at their headers.
return tid1 < tid2;
});
TransCFG::ArcPtrSet coveredArcs;
TransIDSet coveredNodes;
TransIDSet heads;
TransIDToRegionMap headToRegion;
RegionToTransIDsMap regionToTransIds;
regions.clear();
for (auto node : nodes) {
if (!coveredNodes.count(node) ||
!allArcsCovered(cfg.inArcs(node), coveredArcs)) {
TransID newHead = node;
FTRACE(6, "regionizeFunc: selecting trace to cover node {}\n", newHead);
TransIDSet selectedSet;
TransIDVec selectedVec;
RegionDescPtr region;
if (RuntimeOption::EvalJitPGORegionSelector == "hottrace") {
region = selectHotTrace(newHead, profData, cfg,
selectedSet, &selectedVec);
} else if (RuntimeOption::EvalJitPGORegionSelector == "wholecfg") {
region = selectWholeCFG(newHead, profData, cfg, selectedSet,
&selectedVec);
} else {
always_assert(0 && "Invalid value for EvalJitPGORegionSelector");
}
FTRACE(6, "regionizeFunc: selected region to cover node {}\n{}\n",
newHead, show(*region));
profData->setOptimized(profData->transSrcKey(newHead));
assert(selectedVec.size() > 0 && selectedVec[0] == newHead);
regions.push_back(region);
heads.insert(newHead);
markCovered(cfg, region, selectedVec, heads, coveredNodes, coveredArcs);
regionToTransIds[region] = selectedVec;
headToRegion[newHead] = region;
FTRACE(6, "regionizeFunc: selected trace: {}\n",
folly::join(", ", selectedVec));
}
}
assert(coveredNodes.size() == cfg.nodes().size());
assert(coveredArcs.size() == arcs.size());
sortRegions(regions, func, cfg, profData, headToRegion, regionToTransIds);
if (debug && Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
FTRACE(5, "\n--------------------------------------------\n"
"regionizeFunc({}): computed regions:\n", funcId);
for (auto region : regions) {
FTRACE(5, "{}\n\n", show(*region));
}
}
}
RegionDescPtr selectHotTrace(TransID triggerId,
const ProfData* profData,
TransCFG& cfg,
TransIDSet& selectedSet,
TransIDVec* selectedVec) {
auto region = std::make_shared<RegionDesc>();
TransID tid = triggerId;
TransID prevId = kInvalidTransID;
selectedSet.clear();
if (selectedVec) selectedVec->clear();
PostConditions accumPostConds;
// Maps BlockIds to the set of BC offsets for its successor blocks.
// Used to prevent multiple successors with the same SrcKey for now.
// This can go away once task #4157613 is done.
hphp_hash_map<RegionDesc::BlockId, SrcKeySet> succSKSet;
// Maps from BlockIds to accumulated post conditions for that block.
// Used to determine if we can add branch-over edges by checking the
// pre-conditions of the successor block.
hphp_hash_map<RegionDesc::BlockId, PostConditions> blockPostConds;
while (!selectedSet.count(tid)) {
RegionDescPtr blockRegion = profData->transRegion(tid);
if (blockRegion == nullptr) break;
// If the debugger is attached, only allow single-block regions.
if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} "
"because of debugger is attached\n", tid);
break;
}
// Break if block is not the first and requires reffiness checks.
// Task #2589970: fix translateRegion to support mid-region reffiness checks
if (prevId != kInvalidTransID) {
auto nRefDeps = blockRegion->blocks[0]->reffinessPreds().size();
if (nRefDeps > 0) {
FTRACE(2, "selectHotTrace: breaking region because of refDeps ({}) at "
"Translation {}\n", nRefDeps, tid);
break;
}
}
// Break if block is not the first and it corresponds to the main
// function body entry. This is to prevent creating multiple
// large regions containing the function body (starting at various
// DV funclets).
if (prevId != kInvalidTransID) {
const Func* func = profData->transFunc(tid);
Offset bcOffset = profData->transStartBcOff(tid);
if (func->base() == bcOffset) {
FTRACE(2, "selectHotTrace: breaking region because reached the main "
"function body entry at Translation {} (BC offset {})\n",
tid, bcOffset);
break;
}
}
if (prevId != kInvalidTransID) {
auto sk = profData->transSrcKey(tid);
if (profData->optimized(sk)) {
FTRACE(2, "selectHotTrace: breaking region because next sk already "
"optimized, for Translation {}\n", tid);
break;
}
}
// Break trace if translation tid cannot follow the execution of
// the entire translation prevId. This can only happen if the
// execution of prevId takes a side exit that leads to the
// execution of tid.
if (prevId != kInvalidTransID) {
Op* lastInstr = profData->transLastInstr(prevId);
const Unit* unit = profData->transFunc(prevId)->unit();
OffsetSet succOffs = findSuccOffsets(lastInstr, unit);
if (!succOffs.count(profData->transSrcKey(tid).offset())) {
if (HPHP::Trace::moduleEnabled(HPHP::Trace::pgo, 2)) {
FTRACE(2, "selectHotTrace: WARNING: Breaking region @: {}\n",
show(*region));
FTRACE(2, "selectHotTrace: next translation selected: tid = {}\n{}\n",
tid, show(*blockRegion));
FTRACE(2, "\nsuccOffs = {}\n", folly::join(", ", succOffs));
}
break;
}
}
if (region->blocks.size() > 0) {
auto& newBlock = blockRegion->blocks.front();
auto newBlockId = newBlock->id();
auto predBlockId = region->blocks.back().get()->id();
if (!RuntimeOption::EvalHHIRBytecodeControlFlow) {
region->addArc(predBlockId, newBlockId);
} else {
// With bytecode control-flow, we add all forward arcs in the TransCFG
// that are induced by the blocks in the region, as a simple way
// to expose control-flow for now.
// This can go away once Task #4075822 is done.
auto newBlockSrcKey = blockRegion->blocks.front().get()->start();
if (succSKSet[predBlockId].count(newBlockSrcKey)) break;
region->addArc(predBlockId, newBlockId);
succSKSet[predBlockId].insert(newBlockSrcKey);
assert(hasTransId(newBlockId));
auto newTransId = getTransId(newBlockId);
for (auto iOther = 0; iOther < region->blocks.size(); iOther++) {
auto other = region->blocks[iOther];
auto otherBlockId = other.get()->id();
if (!hasTransId(otherBlockId)) continue;
auto otherTransId = getTransId(otherBlockId);
auto otherBlockSrcKey = other.get()->start();
if (cfg.hasArc(otherTransId, newTransId) &&
!other.get()->inlinedCallee() &&
// Task #4157613 will allow the following check to go away
!succSKSet[otherBlockId].count(newBlockSrcKey) &&
preCondsAreSatisfied(newBlock, blockPostConds[otherBlockId])) {
region->addArc(otherBlockId, newBlockId);
succSKSet[otherBlockId].insert(newBlockSrcKey);
}
// When Eval.JitLoops is set, insert back-edges in the
// region if they exist in the TransCFG.
if (RuntimeOption::EvalJitLoops &&
cfg.hasArc(newTransId, otherTransId) &&
// Task #4157613 will allow the following check to go away
!succSKSet[newBlockId].count(otherBlockSrcKey)) {
region->addArc(newBlockId, otherBlockId);
succSKSet[newBlockId].insert(otherBlockSrcKey);
}
}
}
}
region->blocks.insert(region->blocks.end(), blockRegion->blocks.begin(),
blockRegion->blocks.end());
region->arcs.insert(region->arcs.end(), blockRegion->arcs.begin(),
blockRegion->arcs.end());
if (cfg.outArcs(tid).size() > 1) {
region->setSideExitingBlock(blockRegion->blocks.front()->id());
}
selectedSet.insert(tid);
if (selectedVec) selectedVec->push_back(tid);
Op lastOp = *(profData->transLastInstr(tid));
if (breaksRegion(lastOp)) {
FTRACE(2, "selectHotTrace: breaking region because of last instruction "
"in Translation {}: {}\n", tid, opcodeToName(lastOp));
break;
}
auto outArcs = cfg.outArcs(tid);
if (outArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because there's no successor "
"for Translation {}\n", tid);
break;
}
auto lastNewBlock = blockRegion->blocks.back();
discardPoppedTypes(accumPostConds,
blockRegion->blocks[0]->initialSpOffset());
mergePostConds(accumPostConds, lastNewBlock->postConds());
blockPostConds[lastNewBlock->id()] = accumPostConds;
TransCFG::ArcPtrVec possibleOutArcs;
for (auto arc : outArcs) {
RegionDesc::BlockPtr possibleNext =
profData->transRegion(arc->dst())->blocks[0];
if (preCondsAreSatisfied(possibleNext, accumPostConds)) {
possibleOutArcs.emplace_back(arc);
}
}
if (possibleOutArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because postcondition check "
"pruned all successors of Translation {}\n", tid);
break;
}
auto maxWeight = std::numeric_limits<int64_t>::min();
TransCFG::Arc* maxArc = nullptr;
for (auto arc : possibleOutArcs) {
if (arc->weight() >= maxWeight) {
maxWeight = arc->weight();
maxArc = arc;
}
}
assert(maxArc != nullptr);
prevId = tid;
tid = maxArc->dst();
}
return region;
}
RegionDescPtr selectHotTrace(HotTransContext& ctx,
TransIDSet& selectedSet,
TransIDVec* selectedVec /* = nullptr */) {
auto region = std::make_shared<RegionDesc>();
TransID tid = ctx.tid;
TransID prevId = kInvalidTransID;
selectedSet.clear();
if (selectedVec) selectedVec->clear();
TypedLocations accumPostConds;
// Maps from BlockIds to accumulated post conditions for that block.
// Used to determine if we can add branch-over edges by checking the
// pre-conditions of the successor block.
hphp_hash_map<RegionDesc::BlockId, TypedLocations> blockPostConds;
auto numBCInstrs = ctx.maxBCInstrs;
FTRACE(1, "selectHotTrace: starting with maxBCInstrs = {}\n", numBCInstrs);
while (!selectedSet.count(tid)) {
auto rec = ctx.profData->transRec(tid);
auto blockRegion = rec->region();
if (blockRegion == nullptr) break;
// Break if region would be larger than the specified limit.
if (blockRegion->instrSize() > numBCInstrs) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} because "
"size would exceed of maximum translation limit\n", tid);
break;
}
// If the debugger is attached, only allow single-block regions.
if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} "
"because of debugger is attached\n", tid);
break;
}
// Break if block is not the first and it corresponds to the main
// function body entry. This is to prevent creating multiple
// large regions containing the function body (starting at various
// DV funclets).
if (prevId != kInvalidTransID) {
auto const func = rec->func();
auto const bcOffset = rec->startBcOff();
if (func->base() == bcOffset) {
FTRACE(2, "selectHotTrace: breaking region because reached the main "
"function body entry at Translation {} (BC offset {})\n",
tid, bcOffset);
break;
}
}
if (prevId != kInvalidTransID) {
auto sk = rec->srcKey();
if (ctx.profData->optimized(sk)) {
FTRACE(2, "selectHotTrace: breaking region because next sk already "
"optimized, for Translation {}\n", tid);
break;
}
}
bool hasPredBlock = !region->empty();
RegionDesc::BlockId predBlockId = (hasPredBlock ?
region->blocks().back().get()->id() : 0);
auto const& newFirstBlock = blockRegion->entry();
auto newFirstBlockId = newFirstBlock->id();
// Add blockRegion's blocks and arcs to region.
region->append(*blockRegion);
numBCInstrs -= blockRegion->instrSize();
assertx(numBCInstrs >= 0);
if (hasPredBlock) {
region->addArc(predBlockId, newFirstBlockId);
}
selectedSet.insert(tid);
if (selectedVec) selectedVec->push_back(tid);
const auto lastSk = rec->lastSrcKey();
if (breaksRegion(lastSk)) {
FTRACE(2, "selectHotTrace: breaking region because of last instruction "
"in Translation {}: {}\n", tid, opcodeToName(lastSk.op()));
break;
}
auto outArcs = ctx.cfg->outArcs(tid);
if (outArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because there's no successor "
"for Translation {}\n", tid);
break;
}
auto newLastBlock = blockRegion->blocks().back();
discardPoppedTypes(accumPostConds,
blockRegion->entry()->initialSpOffset());
mergePostConds(accumPostConds, newLastBlock->postConds());
blockPostConds[newLastBlock->id()] = accumPostConds;
TransCFG::ArcPtrVec possibleOutArcs;
for (auto arc : outArcs) {
auto dstRec = ctx.profData->transRec(arc->dst());
auto possibleNext = dstRec->region()->entry();
if (preCondsAreSatisfied(possibleNext, accumPostConds)) {
possibleOutArcs.emplace_back(arc);
}
}
if (possibleOutArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because postcondition check "
"pruned all successors of Translation {}\n", tid);
break;
}
auto maxWeight = std::numeric_limits<int64_t>::min();
TransCFG::Arc* maxArc = nullptr;
for (auto arc : possibleOutArcs) {
if (arc->weight() >= maxWeight) {
maxWeight = arc->weight();
maxArc = arc;
}
}
assertx(maxArc != nullptr);
prevId = tid;
tid = maxArc->dst();
}
FTRACE(3, "selectHotTrace: before chainRetransBlocks:\n{}\n", show(*region));
region->chainRetransBlocks();
FTRACE(3, "selectHotTrace: after chainRetransBlocks:\n{}\n", show(*region));
return region;
}
RegionDescPtr selectHotTrace(TransID triggerId,
const ProfData* profData,
TransCFG& cfg,
TransIDSet& selectedSet,
TransIDVec* selectedVec) {
auto region = std::make_shared<RegionDesc>();
TransID tid = triggerId;
TransID prevId = kInvalidTransID;
selectedSet.clear();
if (selectedVec) selectedVec->clear();
PostConditions accumPostConds;
// Maps from BlockIds to accumulated post conditions for that block.
// Used to determine if we can add branch-over edges by checking the
// pre-conditions of the successor block.
hphp_hash_map<RegionDesc::BlockId, PostConditions> blockPostConds;
uint32_t numBCInstrs = 0;
while (!selectedSet.count(tid)) {
RegionDescPtr blockRegion = profData->transRegion(tid);
if (blockRegion == nullptr) break;
// Break if region would be larger than the specified limit.
auto newInstrSize = numBCInstrs + blockRegion->instrSize();
if (newInstrSize > RuntimeOption::EvalJitMaxRegionInstrs) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} because "
"size ({}) would exceed of maximum translation limit\n",
tid, newInstrSize);
break;
}
// If the debugger is attached, only allow single-block regions.
if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} "
"because of debugger is attached\n", tid);
break;
}
// Break if block is not the first and it corresponds to the main
// function body entry. This is to prevent creating multiple
// large regions containing the function body (starting at various
// DV funclets).
if (prevId != kInvalidTransID) {
const Func* func = profData->transFunc(tid);
Offset bcOffset = profData->transStartBcOff(tid);
if (func->base() == bcOffset) {
FTRACE(2, "selectHotTrace: breaking region because reached the main "
"function body entry at Translation {} (BC offset {})\n",
tid, bcOffset);
break;
}
}
if (prevId != kInvalidTransID) {
auto sk = profData->transSrcKey(tid);
if (profData->optimized(sk)) {
FTRACE(2, "selectHotTrace: breaking region because next sk already "
"optimized, for Translation {}\n", tid);
break;
}
}
bool hasPredBlock = !region->empty();
RegionDesc::BlockId predBlockId = (hasPredBlock ?
region->blocks().back().get()->id() : 0);
auto const& newFirstBlock = blockRegion->entry();
auto newFirstBlockId = newFirstBlock->id();
auto newLastBlockId = blockRegion->blocks().back()->id();
// Add blockRegion's blocks and arcs to region.
region->append(*blockRegion);
numBCInstrs += blockRegion->instrSize();
if (hasPredBlock) {
region->addArc(predBlockId, newFirstBlockId);
}
// When Eval.JitLoops is set, insert back-edges in the region if
// they exist in the TransCFG.
if (RuntimeOption::EvalJitLoops) {
assertx(hasTransId(newFirstBlockId));
auto newTransId = getTransId(newFirstBlockId);
// Don't add the arc if the last opcode in the source block ends
// the region.
if (!breaksRegion(*profData->transLastInstr(newTransId))) {
auto& blocks = region->blocks();
for (auto iOther = 0; iOther < blocks.size(); iOther++) {
auto other = blocks[iOther];
auto otherFirstBlockId = other.get()->id();
if (!hasTransId(otherFirstBlockId)) continue;
auto otherTransId = getTransId(otherFirstBlockId);
if (cfg.hasArc(newTransId, otherTransId)) {
region->addArc(newLastBlockId, otherFirstBlockId);
}
}
}
}
if (cfg.outArcs(tid).size() > 1) {
region->setSideExitingBlock(blockRegion->entry()->id());
}
selectedSet.insert(tid);
if (selectedVec) selectedVec->push_back(tid);
Op lastOp = *(profData->transLastInstr(tid));
if (breaksRegion(lastOp)) {
FTRACE(2, "selectHotTrace: breaking region because of last instruction "
"in Translation {}: {}\n", tid, opcodeToName(lastOp));
break;
}
auto outArcs = cfg.outArcs(tid);
if (outArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because there's no successor "
"for Translation {}\n", tid);
break;
}
auto newLastBlock = blockRegion->blocks().back();
discardPoppedTypes(accumPostConds,
blockRegion->entry()->initialSpOffset());
mergePostConds(accumPostConds, newLastBlock->postConds());
blockPostConds[newLastBlock->id()] = accumPostConds;
TransCFG::ArcPtrVec possibleOutArcs;
for (auto arc : outArcs) {
RegionDesc::BlockPtr possibleNext =
profData->transRegion(arc->dst())->entry();
if (preCondsAreSatisfied(possibleNext, accumPostConds)) {
possibleOutArcs.emplace_back(arc);
}
}
if (possibleOutArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because postcondition check "
"pruned all successors of Translation {}\n", tid);
break;
}
auto maxWeight = std::numeric_limits<int64_t>::min();
TransCFG::Arc* maxArc = nullptr;
for (auto arc : possibleOutArcs) {
if (arc->weight() >= maxWeight) {
maxWeight = arc->weight();
maxArc = arc;
}
}
assertx(maxArc != nullptr);
prevId = tid;
tid = maxArc->dst();
}
FTRACE(3, "selectHotTrace: before chainRetransBlocks:\n{}\n", show(*region));
region->chainRetransBlocks();
FTRACE(3, "selectHotTrace: after chainRetransBlocks:\n{}\n", show(*region));
return region;
}
/**
* Regionize a func, so that each node and each arc in its TransCFG is
* "covered". A node is covered if any region contains it. An arc T1->T2
* is covered if either:
*
* a) T1 and T2 are in the same region R and T2 immediately follows
* T1 in R.
* b) T2 is the head (first translation) of a region.
*
* Basic algorithm:
*
* 1) sort nodes in decreasing weight order
* 2) for each node N:
* 2.1) if N and all its incoming arcs are covered, then continue
* 2.2) select a region starting at this node and mark nodes/arcs as
* covered appropriately
*/
void regionizeFunc(const Func* func,
Transl::TranslatorX64* tx64,
RegionVec& regions) {
assert(RuntimeOption::EvalJitPGO);
FuncId funcId = func->getFuncId();
ProfData* profData = tx64->profData();
TransCFG cfg(funcId, profData, tx64->getSrcDB(), tx64->getJmpToTransIDMap());
if (Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
string dotFileName = folly::to<string>("/tmp/func-cfg-", funcId, ".dot");
cfg.print(dotFileName, funcId, profData, nullptr);
FTRACE(5, "regionizeFunc: initial CFG for func {} saved to file {}\n",
funcId, dotFileName);
}
TransCFG::ArcPtrVec arcs = cfg.arcs();
vector<TransID> nodes = cfg.nodes();
std::sort(nodes.begin(), nodes.end(),
[&](TransID tid1, TransID tid2) -> bool {
if (cfg.weight(tid1) != cfg.weight(tid2)) {
return cfg.weight(tid1) > cfg.weight(tid2);
}
// In case of ties, pick older translations first, in an
// attempt to start loops at their headers.
return tid1 < tid2;
});
TransCFG::ArcPtrSet coveredArcs;
TransIDSet coveredNodes;
TransIDSet heads;
TransIDToRegionMap headToRegion;
RegionToTransIDsMap regionToTransIds;
regions.clear();
for (auto node : nodes) {
if (!setContains(coveredNodes, node) ||
!allArcsCovered(cfg.inArcs(node), coveredArcs)) {
TransID newHead = node;
FTRACE(6, "regionizeFunc: selecting trace to cover node {}\n", newHead);
TransIDSet selectedSet;
TransIDVec selectedVec;
RegionDescPtr region = selectHotTrace(newHead, profData, cfg,
selectedSet, &selectedVec);
profData->setOptimized(profData->transSrcKey(newHead));
assert(selectedVec.size() > 0 && selectedVec[0] == newHead);
regions.push_back(region);
heads.insert(newHead);
markCovered(cfg, selectedVec, heads, coveredNodes, coveredArcs);
regionToTransIds[region] = selectedVec;
headToRegion[newHead] = region;
FTRACE(6, "regionizeFunc: selected trace: {}\n",
folly::join(", ", selectedVec));
}
}
assert(coveredNodes.size() == cfg.nodes().size());
assert(coveredArcs.size() == arcs.size());
sortRegion(regions, func, cfg, profData, headToRegion, regionToTransIds);
if (debug && Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
FTRACE(5, "\n--------------------------------------------\n"
"regionizeFunc({}): computed regions:\n", funcId);
for (auto region : regions) {
FTRACE(5, "{}\n\n", show(*region));
}
}
}
RegionDescPtr selectHotTrace(TransID triggerId,
const ProfData* profData,
TransCFG& cfg,
TransIDSet& selectedSet,
TransIDVec* selectedVec) {
auto region = std::make_shared<RegionDesc>();
TransID tid = triggerId;
TransID prevId = InvalidID;
selectedSet.clear();
if (selectedVec) selectedVec->clear();
PostConditions accumPostConds;
while (!selectedSet.count(tid)) {
RegionDescPtr blockRegion = profData->transRegion(tid);
if (blockRegion == nullptr) break;
// If the debugger is attached, only allow single-block regions.
if (prevId != InvalidID && isDebuggerAttachedProcess()) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} "
"because of debugger is attached\n", tid);
break;
}
// Break if block is not the first and requires reffiness checks.
// Task #2589970: fix translateRegion to support mid-region reffiness checks
if (prevId != InvalidID) {
auto nRefDeps = blockRegion->blocks[0]->reffinessPreds().size();
if (nRefDeps > 0) {
FTRACE(2, "selectHotTrace: breaking region because of refDeps ({}) at "
"Translation {}\n", nRefDeps, tid);
break;
}
}
// Break if block is not the first and it corresponds to the main
// function body entry. This is to prevent creating multiple
// large regions containing the function body (starting at various
// DV funclets).
if (prevId != InvalidID) {
const Func* func = profData->transFunc(tid);
Offset bcOffset = profData->transStartBcOff(tid);
if (func->base() == bcOffset) {
FTRACE(2, "selectHotTrace: breaking region because reached the main "
"function body entry at Translation {} (BC offset {})\n",
tid, bcOffset);
break;
}
}
if (prevId != InvalidID) {
auto sk = profData->transSrcKey(tid);
if (profData->optimized(sk)) {
FTRACE(2, "selectHotTrace: breaking region because next sk already "
"optimized, for Translation {}\n", tid);
break;
}
}
// Break trace if translation tid cannot follow the execution of
// the entire translation prevId. This can only happen if the
// execution of prevId takes a side exit that leads to the
// execution of tid.
if (prevId != InvalidID) {
Op* lastInstr = profData->transLastInstr(prevId);
const Unit* unit = profData->transFunc(prevId)->unit();
OffsetSet succOffs = findSuccOffsets(lastInstr, unit);
if (!succOffs.count(profData->transSrcKey(tid).offset())) {
if (HPHP::Trace::moduleEnabled(HPHP::Trace::pgo, 2)) {
FTRACE(2, "selectHotTrace: WARNING: Breaking region @: {}\n",
JIT::show(*region));
FTRACE(2, "selectHotTrace: next translation selected: tid = {}\n{}\n",
tid, JIT::show(*blockRegion));
FTRACE(2, "\nsuccOffs = {}\n", folly::join(", ", succOffs));
}
break;
}
}
region->blocks.insert(region->blocks.end(), blockRegion->blocks.begin(),
blockRegion->blocks.end());
selectedSet.insert(tid);
if (selectedVec) selectedVec->push_back(tid);
Op lastOp = *(profData->transLastInstr(tid));
if (breaksRegion(lastOp)) {
FTRACE(2, "selectHotTrace: breaking region because of last instruction "
"in Translation {}: {}\n", tid, opcodeToName(lastOp));
break;
}
auto outArcs = cfg.outArcs(tid);
if (outArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because there's no successor "
"for Translation {}\n", tid);
break;
}
auto lastNewBlock = blockRegion->blocks.back();
discardPoppedTypes(accumPostConds,
blockRegion->blocks[0]->initialSpOffset());
mergePostConds(accumPostConds, lastNewBlock->postConds());
TransCFG::ArcPtrVec possibleOutArcs;
for (auto arc : outArcs) {
RegionDesc::BlockPtr possibleNext =
profData->transRegion(arc->dst())->blocks[0];
if (preCondsAreSatisfied(possibleNext, accumPostConds)) {
possibleOutArcs.emplace_back(arc);
}
}
if (possibleOutArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because postcondition check "
"pruned all successors of Translation {}\n", tid);
break;
}
auto maxWeight = std::numeric_limits<int64_t>::min();
TransCFG::Arc* maxArc = nullptr;
for (auto arc : possibleOutArcs) {
if (arc->weight() >= maxWeight) {
maxWeight = arc->weight();
maxArc = arc;
}
}
assert(maxArc != nullptr);
prevId = tid;
tid = maxArc->dst();
}
return region;
}
RegionDescPtr selectHotTrace(TransID triggerId,
const ProfData* profData,
TransCFG& cfg,
TransIDSet& selectedSet,
TransIDVec* selectedVec) {
auto region = std::make_shared<RegionDesc>();
TransID tid = triggerId;
TransID prevId = kInvalidTransID;
selectedSet.clear();
if (selectedVec) selectedVec->clear();
PostConditions accumPostConds;
// Maps BlockIds to the set of BC offsets for its successor blocks.
// Used to prevent multiple successors with the same SrcKey for now.
// This can go away once task #4157613 is done.
hphp_hash_map<RegionDesc::BlockId, SrcKeySet> succSKSet;
// Maps from BlockIds to accumulated post conditions for that block.
// Used to determine if we can add branch-over edges by checking the
// pre-conditions of the successor block.
hphp_hash_map<RegionDesc::BlockId, PostConditions> blockPostConds;
while (!selectedSet.count(tid)) {
RegionDescPtr blockRegion = profData->transRegion(tid);
if (blockRegion == nullptr) break;
// If the debugger is attached, only allow single-block regions.
if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} "
"because of debugger is attached\n", tid);
break;
}
// Break if block is not the first and it corresponds to the main
// function body entry. This is to prevent creating multiple
// large regions containing the function body (starting at various
// DV funclets).
if (prevId != kInvalidTransID) {
const Func* func = profData->transFunc(tid);
Offset bcOffset = profData->transStartBcOff(tid);
if (func->base() == bcOffset) {
FTRACE(2, "selectHotTrace: breaking region because reached the main "
"function body entry at Translation {} (BC offset {})\n",
tid, bcOffset);
break;
}
}
if (prevId != kInvalidTransID) {
auto sk = profData->transSrcKey(tid);
if (profData->optimized(sk)) {
FTRACE(2, "selectHotTrace: breaking region because next sk already "
"optimized, for Translation {}\n", tid);
break;
}
}
bool hasPredBlock = !region->empty();
RegionDesc::BlockId predBlockId = (hasPredBlock ?
region->blocks().back().get()->id() : 0);
auto const& newFirstBlock = blockRegion->entry();
auto newFirstBlockId = newFirstBlock->id();
auto newFirstBlockSk = newFirstBlock->start();
auto newLastBlockId = blockRegion->blocks().back()->id();
// Make sure we don't end up with multiple successors for the same
// SrcKey. Task #4157613 will allow the following check to go away.
// This needs to be done before we insert blockRegion into region,
// to avoid creating unreachable blocks.
if (RuntimeOption::EvalHHIRBytecodeControlFlow && hasPredBlock &&
succSKSet[predBlockId].count(newFirstBlockSk)) {
break;
}
// Add blockRegion's blocks and arcs to region.
region->append(*blockRegion);
if (hasPredBlock) {
if (RuntimeOption::EvalHHIRBytecodeControlFlow) {
// This is checked above.
assert(succSKSet[predBlockId].count(newFirstBlockSk) == 0);
succSKSet[predBlockId].insert(newFirstBlockSk);
}
region->addArc(predBlockId, newFirstBlockId);
}
// With bytecode control-flow, we add all forward arcs in the TransCFG
// that are induced by the blocks in the region, as a simple way
// to expose control-flow for now.
// This can go away once Task #4075822 is done.
if (RuntimeOption::EvalHHIRBytecodeControlFlow) {
assert(hasTransId(newFirstBlockId));
auto newTransId = getTransId(newFirstBlockId);
auto& blocks = region->blocks();
for (auto iOther = 0; iOther < blocks.size(); iOther++) {
auto other = blocks[iOther];
auto otherFirstBlockId = other.get()->id();
if (!hasTransId(otherFirstBlockId)) continue;
auto otherTransId = getTransId(otherFirstBlockId);
auto otherFirstBlockSk = other.get()->start();
auto otherRegion = profData->transRegion(otherTransId);
auto otherLastBlockId = otherRegion->blocks().back()->id();
// When loops are off, stop once we hit the newTransId we just inserted.
if (!RuntimeOption::EvalJitLoops && otherTransId == newTransId) break;
if (cfg.hasArc(otherTransId, newTransId) &&
// Task #4157613 will allow the following check to go away
!succSKSet[otherLastBlockId].count(newFirstBlockSk) &&
preCondsAreSatisfied(newFirstBlock,
blockPostConds[otherLastBlockId])) {
region->addArc(otherLastBlockId, newFirstBlockId);
succSKSet[otherLastBlockId].insert(newFirstBlockSk);
}
// When Eval.JitLoops is set, insert back-edges in the
// region if they exist in the TransCFG.
if (RuntimeOption::EvalJitLoops &&
cfg.hasArc(newTransId, otherTransId) &&
// Task #4157613 will allow the following check to go away
!succSKSet[newLastBlockId].count(otherFirstBlockSk)) {
region->addArc(newLastBlockId, otherFirstBlockId);
succSKSet[newLastBlockId].insert(otherFirstBlockSk);
}
}
}
if (cfg.outArcs(tid).size() > 1) {
region->setSideExitingBlock(blockRegion->entry()->id());
}
selectedSet.insert(tid);
if (selectedVec) selectedVec->push_back(tid);
Op lastOp = *(profData->transLastInstr(tid));
if (breaksRegion(lastOp)) {
FTRACE(2, "selectHotTrace: breaking region because of last instruction "
"in Translation {}: {}\n", tid, opcodeToName(lastOp));
break;
}
auto outArcs = cfg.outArcs(tid);
if (outArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because there's no successor "
"for Translation {}\n", tid);
break;
}
auto newLastBlock = blockRegion->blocks().back();
discardPoppedTypes(accumPostConds,
blockRegion->entry()->initialSpOffset());
mergePostConds(accumPostConds, newLastBlock->postConds());
blockPostConds[newLastBlock->id()] = accumPostConds;
TransCFG::ArcPtrVec possibleOutArcs;
for (auto arc : outArcs) {
RegionDesc::BlockPtr possibleNext =
profData->transRegion(arc->dst())->entry();
if (preCondsAreSatisfied(possibleNext, accumPostConds)) {
possibleOutArcs.emplace_back(arc);
}
}
if (possibleOutArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because postcondition check "
"pruned all successors of Translation {}\n", tid);
break;
}
auto maxWeight = std::numeric_limits<int64_t>::min();
TransCFG::Arc* maxArc = nullptr;
for (auto arc : possibleOutArcs) {
if (arc->weight() >= maxWeight) {
maxWeight = arc->weight();
maxArc = arc;
}
}
assert(maxArc != nullptr);
prevId = tid;
tid = maxArc->dst();
}
return region;
}
RegionDescPtr selectHotTrace(TransID triggerId,
const ProfData* profData,
TransCFG& cfg,
TransIDSet& selectedSet) {
JIT::RegionDescPtr region = std::make_shared<JIT::RegionDesc>();
TransID tid = triggerId;
TransID prevId = InvalidID;
selectedSet.clear();
PostConditions accumPostConds;
while (!setContains(selectedSet, tid)) {
RegionDescPtr blockRegion = profData->transRegion(tid);
if (blockRegion == nullptr) break;
// If the debugger is attached, only allow single-block regions.
if (prevId != InvalidID && isDebuggerAttachedProcess()) {
FTRACE(5, "selectHotRegion: breaking region at Translation {} "
"because of debugger is attached\n", tid);
break;
}
// Break if block is not the first and requires reffiness checks.
// Task #2589970: fix translateRegion to support mid-region reffiness checks
if (prevId != InvalidID) {
auto nRefDeps = blockRegion->blocks[0]->reffinessPreds().size();
if (nRefDeps > 0) {
FTRACE(5, "selectHotRegion: breaking region because of refDeps ({}) at "
"Translation {}\n", nRefDeps, tid);
break;
}
}
// Break trace if translation tid cannot follow the execution of
// the entire translation prevTd. This can only happen if the
// execution of prevId takes a side exit that leads to the
// execution of tid.
if (prevId != InvalidID) {
Op* lastInstr = profData->transLastInstr(prevId);
const Unit* unit = profData->transFunc(prevId)->unit();
OffsetSet succOffs = findSuccOffsets(lastInstr, unit);
if (!setContains(succOffs, profData->transSrcKey(tid).offset())) {
if (HPHP::Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
FTRACE(5, "selectHotTrace: WARNING: Breaking region @: {}\n",
JIT::show(*region));
FTRACE(5, "selectHotTrace: next translation selected: tid = {}\n{}\n",
tid, JIT::show(*blockRegion));
std::string succStr("succOffs = ");
for (auto succ : succOffs) {
succStr += lexical_cast<std::string>(succ);
}
FTRACE(5, "\n{}\n", succStr);
}
break;
}
}
region->blocks.insert(region->blocks.end(), blockRegion->blocks.begin(),
blockRegion->blocks.end());
selectedSet.insert(tid);
Op lastOp = *(profData->transLastInstr(tid));
if (breaksRegion(lastOp)) {
FTRACE(5, "selectHotTrace: breaking region because of last instruction "
"in Translation {}: {}\n", tid, opcodeToName(lastOp));
break;
}
auto outArcs = cfg.outArcs(tid);
if (outArcs.size() == 0) {
FTRACE(5, "selectHotTrace: breaking region because there's no successor "
"for Translation {}\n", tid);
break;
}
auto lastNewBlock = blockRegion->blocks.back();
mergePostConds(accumPostConds, lastNewBlock->postConds());
TransCFG::ArcPtrVec possibleOutArcs;
for (auto arc : outArcs) {
RegionDesc::BlockPtr possibleNext =
profData->transRegion(arc->dst())->blocks[0];
if (preCondsAreSatisfied(possibleNext, accumPostConds)) {
possibleOutArcs.emplace_back(arc);
}
}
if (possibleOutArcs.size() == 0) {
FTRACE(5, "selectHotTrace: breaking region because postcondition check "
"pruned all successors of Translation {}\n", tid);
break;
}
auto maxWeight = std::numeric_limits<int64_t>::min();
TransCFG::Arc* maxArc = nullptr;
for (auto arc : possibleOutArcs) {
if (arc->weight() >= maxWeight) {
maxWeight = arc->weight();
maxArc = arc;
}
}
assert(maxArc != nullptr);
prevId = tid;
tid = maxArc->dst();
}
return region;
}