/**
* Regionize a func, so that each node and each arc in its TransCFG is
* "covered". A node is covered if any region contains it. An arc T1->T2
* is covered if either:
*
* a) T1 and T2 are in the same region R and T2 immediately follows
* T1 in R.
* b) T2 is the head (first translation) of a region.
*
* Basic algorithm:
*
* 1) sort nodes in decreasing weight order
* 2) for each node N:
* 2.1) if N and all its incoming arcs are covered, then continue
* 2.2) select a region starting at this node and mark nodes/arcs as
* covered appropriately
*/
void regionizeFunc(const Func* func,
MCGenerator* mcg,
RegionVec& regions) {
const Timer rf_timer(Timer::regionizeFunc);
assert(RuntimeOption::EvalJitPGO);
auto const funcId = func->getFuncId();
auto const profData = mcg->tx().profData();
TransCFG cfg(funcId, profData, mcg->tx().getSrcDB(),
mcg->getJmpToTransIDMap());
if (Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
auto dotFileName = folly::to<std::string>(
"/tmp/func-cfg-", funcId, ".dot");
cfg.print(dotFileName, funcId, profData, nullptr);
FTRACE(5, "regionizeFunc: initial CFG for func {} saved to file {}\n",
funcId, dotFileName);
}
TransCFG::ArcPtrVec arcs = cfg.arcs();
std::vector<TransID> nodes = cfg.nodes();
std::sort(nodes.begin(), nodes.end(),
[&](TransID tid1, TransID tid2) -> bool {
if (RuntimeOption::EvalJitPGORegionSelector == "wholecfg") {
auto bcOff1 = profData->transStartBcOff(tid1);
auto bcOff2 = profData->transStartBcOff(tid2);
if (bcOff1 != bcOff2) return bcOff1 < bcOff2;
}
if (cfg.weight(tid1) != cfg.weight(tid2)) {
return cfg.weight(tid1) > cfg.weight(tid2);
}
// In case of ties, pick older translations first, in an
// attempt to start loops at their headers.
return tid1 < tid2;
});
TransCFG::ArcPtrSet coveredArcs;
TransIDSet coveredNodes;
TransIDSet heads;
TransIDToRegionMap headToRegion;
RegionToTransIDsMap regionToTransIds;
regions.clear();
for (auto node : nodes) {
if (!coveredNodes.count(node) ||
!allArcsCovered(cfg.inArcs(node), coveredArcs)) {
TransID newHead = node;
FTRACE(6, "regionizeFunc: selecting trace to cover node {}\n", newHead);
TransIDSet selectedSet;
TransIDVec selectedVec;
RegionDescPtr region;
if (RuntimeOption::EvalJitPGORegionSelector == "hottrace") {
region = selectHotTrace(newHead, profData, cfg,
selectedSet, &selectedVec);
} else if (RuntimeOption::EvalJitPGORegionSelector == "wholecfg") {
region = selectWholeCFG(newHead, profData, cfg, selectedSet,
&selectedVec);
} else {
always_assert(0 && "Invalid value for EvalJitPGORegionSelector");
}
FTRACE(6, "regionizeFunc: selected region to cover node {}\n{}\n",
newHead, show(*region));
profData->setOptimized(profData->transSrcKey(newHead));
assert(selectedVec.size() > 0 && selectedVec[0] == newHead);
regions.push_back(region);
heads.insert(newHead);
markCovered(cfg, region, selectedVec, heads, coveredNodes, coveredArcs);
regionToTransIds[region] = selectedVec;
headToRegion[newHead] = region;
FTRACE(6, "regionizeFunc: selected trace: {}\n",
folly::join(", ", selectedVec));
}
}
assert(coveredNodes.size() == cfg.nodes().size());
assert(coveredArcs.size() == arcs.size());
sortRegions(regions, func, cfg, profData, headToRegion, regionToTransIds);
if (debug && Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
FTRACE(5, "\n--------------------------------------------\n"
"regionizeFunc({}): computed regions:\n", funcId);
for (auto region : regions) {
FTRACE(5, "{}\n\n", show(*region));
}
}
}
/**
* Regionize a func, so that each node and each arc in its TransCFG is
* "covered". A node is covered if any region contains it. An arc T1->T2
* is covered if either:
*
* a) T1 and T2 are in the same region R and T2 immediately follows
* T1 in R.
* b) T2 is the head (first translation) of a region.
*
* Basic algorithm:
*
* 1) sort nodes in decreasing weight order
* 2) for each node N:
* 2.1) if N and all its incoming arcs are covered, then continue
* 2.2) select a region starting at this node and mark nodes/arcs as
* covered appropriately
*/
void regionizeFunc(const Func* func,
Transl::TranslatorX64* tx64,
RegionVec& regions) {
assert(RuntimeOption::EvalJitPGO);
FuncId funcId = func->getFuncId();
ProfData* profData = tx64->profData();
TransCFG cfg(funcId, profData, tx64->getSrcDB(), tx64->getJmpToTransIDMap());
if (Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
string dotFileName = folly::to<string>("/tmp/func-cfg-", funcId, ".dot");
cfg.print(dotFileName, funcId, profData, nullptr);
FTRACE(5, "regionizeFunc: initial CFG for func {} saved to file {}\n",
funcId, dotFileName);
}
TransCFG::ArcPtrVec arcs = cfg.arcs();
vector<TransID> nodes = cfg.nodes();
std::sort(nodes.begin(), nodes.end(),
[&](TransID tid1, TransID tid2) -> bool {
if (cfg.weight(tid1) != cfg.weight(tid2)) {
return cfg.weight(tid1) > cfg.weight(tid2);
}
// In case of ties, pick older translations first, in an
// attempt to start loops at their headers.
return tid1 < tid2;
});
TransCFG::ArcPtrSet coveredArcs;
TransIDSet coveredNodes;
TransIDSet heads;
TransIDToRegionMap headToRegion;
RegionToTransIDsMap regionToTransIds;
regions.clear();
for (auto node : nodes) {
if (!setContains(coveredNodes, node) ||
!allArcsCovered(cfg.inArcs(node), coveredArcs)) {
TransID newHead = node;
FTRACE(6, "regionizeFunc: selecting trace to cover node {}\n", newHead);
TransIDSet selectedSet;
TransIDVec selectedVec;
RegionDescPtr region = selectHotTrace(newHead, profData, cfg,
selectedSet, &selectedVec);
profData->setOptimized(profData->transSrcKey(newHead));
assert(selectedVec.size() > 0 && selectedVec[0] == newHead);
regions.push_back(region);
heads.insert(newHead);
markCovered(cfg, selectedVec, heads, coveredNodes, coveredArcs);
regionToTransIds[region] = selectedVec;
headToRegion[newHead] = region;
FTRACE(6, "regionizeFunc: selected trace: {}\n",
folly::join(", ", selectedVec));
}
}
assert(coveredNodes.size() == cfg.nodes().size());
assert(coveredArcs.size() == arcs.size());
sortRegion(regions, func, cfg, profData, headToRegion, regionToTransIds);
if (debug && Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
FTRACE(5, "\n--------------------------------------------\n"
"regionizeFunc({}): computed regions:\n", funcId);
for (auto region : regions) {
FTRACE(5, "{}\n\n", show(*region));
}
}
}
/**
* Sorts the regions vector in a linear order to be used for
* translation. The goal is to obtain an order that improves locality
* when the function is executed. Each region is translated separately.
*/
static void sortRegions(RegionVec& regions,
const Func* func,
const TransCFG& cfg,
const ProfData* profData,
const TransIDToRegionMap& headToRegion,
const RegionToTransIDsMap& regionToTransIds) {
RegionVec sorted;
RegionSet selected;
if (regions.empty()) return;
// First, pick the region starting at the lowest bytecode offset.
// This will normally correspond to the main function entry (for
// normal, regular bytecode), but it may not be for irregular
// functions written in hhas (like array_map and array_filter). If
// there multiple regions starting at the lowest bytecode offset,
// pick the one with the largest profile weight.
RegionDescPtr entryRegion = nullptr;
int64_t maxEntryWeight = -1;
Offset lowestOffset = kInvalidOffset;
for (const auto& pair : regionToTransIds) {
auto r = pair.first;
auto& tids = pair.second;
TransID firstTid = tids[0];
Offset firstOffset = profData->transSrcKey(firstTid).offset();
int64_t weight = cfg.weight(firstTid);
if (lowestOffset == kInvalidOffset || firstOffset < lowestOffset ||
(firstOffset == lowestOffset && weight > maxEntryWeight)) {
entryRegion = r;
maxEntryWeight = weight;
lowestOffset = firstOffset;
}
}
assert(entryRegion);
sorted.push_back(entryRegion);
selected.insert(entryRegion);
RegionDescPtr region = entryRegion;
// Select the remaining regions, iteratively picking the most likely
// region to execute next.
for (auto i = 1; i < regions.size(); i++) {
int64_t maxWeight = -1;
int64_t maxHeadWeight = -1;
RegionDescPtr bestNext = nullptr;
auto regionTransIds = getRegionTransIDVec(regionToTransIds, region);
for (auto next : regions) {
if (selected.count(next)) continue;
auto nextTransIds = getRegionTransIDVec(regionToTransIds, next);
int64_t weight = interRegionWeight(regionTransIds, nextTransIds[0], cfg);
int64_t headWeight = cfg.weight(nextTransIds[0]);
if ((weight > maxWeight) ||
(weight == maxWeight && headWeight > maxHeadWeight)) {
maxWeight = weight;
maxHeadWeight = headWeight;
bestNext = next;
}
}
assert(bestNext);
sorted.push_back(bestNext);
selected.insert(bestNext);
region = bestNext;
}
assert(sorted.size() == regions.size());
regions = sorted;
if (debug && Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
for (size_t i = 0; i < regions.size(); i++) {
auto r = regions[i];
auto tids = getRegionTransIDVec(regionToTransIds, r);
std::string transIds = folly::join(", ", tids);
FTRACE(6, "sortRegions: region[{}]: {}\n", i, transIds);
}
}
}
/**
* Sorts the regions vector in a linear order to be used for
* translation. The goal is to obtain an order that improves locality
* when the function is executed.
*/
static void sortRegion(RegionVec& regions,
const Func* func,
const TransCFG& cfg,
const ProfData* profData,
const TransIDToRegionMap& headToRegion,
const RegionToTransIDsMap& regionToTransIds) {
RegionVec sorted;
RegionSet selected;
// First, pick the region for the function body entry. There may be
// multiple translations of the function body, so pick the one with
// largest profile weight.
RegionDescPtr entryRegion = nullptr;
int64_t maxEntryWeight = -1;
for (const auto& pair : regionToTransIds) {
auto r = pair.first;
auto& tids = pair.second;
for (auto tid : tids) {
if (profData->transSrcKey(tid).offset() == func->base()) {
int64_t weight = cfg.weight(tid);
if (weight > maxEntryWeight) {
entryRegion = r;
maxEntryWeight = weight;
}
}
}
}
assert(entryRegion);
sorted.push_back(entryRegion);
selected.insert(entryRegion);
RegionDescPtr region = entryRegion;
// Select the remaining regions, iteratively picking the most likely
// region to execute next.
for (auto i = 1; i < regions.size(); i++) {
int64_t maxWeight = -1;
int64_t maxHeadWeight = -1;
RegionDescPtr bestNext = nullptr;
auto regionTransIds = getRegionTransIDVec(regionToTransIds, region);
for (auto next : regions) {
if (setContains(selected, next)) continue;
auto nextTransIds = getRegionTransIDVec(regionToTransIds, next);
int64_t weight = interRegionWeight(regionTransIds, nextTransIds[0], cfg);
int64_t headWeight = cfg.weight(nextTransIds[0]);
if ((weight > maxWeight) ||
(weight == maxWeight && headWeight > maxHeadWeight)) {
maxWeight = weight;
maxHeadWeight = headWeight;
bestNext = next;
}
}
assert(bestNext);
sorted.push_back(bestNext);
selected.insert(bestNext);
region = bestNext;
}
assert(sorted.size() == regions.size());
regions = sorted;
if (debug && Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
for (size_t i = 0; i < regions.size(); i++) {
auto r = regions[i];
auto tids = getRegionTransIDVec(regionToTransIds, r);
std::string transIds = folly::join(", ", tids);
FTRACE(6, "sortRegion: region[{}]: {}\n", i, transIds);
}
}
}