ProfTransRec::ProfTransRec(Offset lastBcOff, SrcKey sk, RegionDescPtr region)
: m_kind(TransKind::Profile)
, m_lastBcOff(lastBcOff)
, m_sk(sk)
, m_region(region) {
assertx(region != nullptr && !region->empty() && region->start() == sk);
}
ProfTransRec::ProfTransRec(TransID id,
TransKind kind,
Offset lastBcOff,
SrcKey sk,
RegionDescPtr region)
: m_id(id)
, m_kind(kind)
, m_lastBcOff(lastBcOff)
, m_region(region)
, m_sk(sk) {
assertx(region == nullptr || (!region->empty() && region->start() == sk));
}
/*
* Returns the last BC offset in the region that corresponds to the
* function where the region starts. This will normally be the offset
* of the last instruction in the last block, except if the function
* ends with an inlined call. In this case, the offset of the
* corresponding FCall* in the function that starts the region is
* returned.
*/
static Offset findLastBcOffset(const RegionDescPtr region) {
assert(!region->empty());
auto& blocks = region->blocks();
FuncId startFuncId = blocks[0]->start().getFuncId();
for (int i = blocks.size() - 1; i >= 0; i--) {
SrcKey sk = blocks[i]->last();
if (sk.getFuncId() == startFuncId) {
return sk.offset();
}
}
not_reached();
}
RegionDescPtr selectHotRegion(TransID transId,
MCGenerator* mcg) {
auto const profData = jit::profData();
assertx(profData);
auto const& func = *profData->transRec(transId)->func();
FuncId funcId = func.getFuncId();
TransCFG cfg(funcId, profData, mcg->srcDB());
assertx(regionMode() != RegionMode::Method);
RegionDescPtr region;
HotTransContext ctx;
ctx.cfg = &cfg;
ctx.profData = profData;
ctx.tid = transId;
ctx.maxBCInstrs = RuntimeOption::EvalJitMaxRegionInstrs;
switch (pgoRegionMode(func)) {
case PGORegionMode::Hottrace:
region = selectHotTrace(ctx);
break;
case PGORegionMode::Hotblock:
region = selectHotBlock(transId, profData, cfg);
break;
case PGORegionMode::WholeCFG:
case PGORegionMode::HotCFG:
region = selectHotCFG(ctx);
break;
}
assertx(region);
if (Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
std::string dotFileName = std::string("/tmp/trans-cfg-") +
folly::to<std::string>(transId) + ".dot";
std::ofstream outFile(dotFileName);
if (outFile.is_open()) {
cfg.print(outFile, funcId, profData);
outFile.close();
}
FTRACE(5, "selectHotRegion: New Translation (file: {}) {}\n",
dotFileName, region ? show(*region) : std::string("empty region"));
}
always_assert(region->instrSize() <= RuntimeOption::EvalJitMaxRegionInstrs);
if (region->empty()) return nullptr;
return region;
}
RegionDescPtr selectHotRegion(TransID transId,
MCGenerator* mcg) {
assertx(RuntimeOption::EvalJitPGO);
const ProfData* profData = mcg->tx().profData();
auto const& func = *(profData->transFunc(transId));
FuncId funcId = func.getFuncId();
TransCFG cfg(funcId, profData, mcg->tx().getSrcDB(),
mcg->getJmpToTransIDMap());
TransIDSet selectedTIDs;
assertx(regionMode() != RegionMode::Method);
RegionDescPtr region;
HotTransContext ctx;
ctx.cfg = &cfg;
ctx.profData = profData;
ctx.tid = transId;
ctx.maxBCInstrs = RuntimeOption::EvalJitMaxRegionInstrs;
switch (pgoRegionMode(func)) {
case PGORegionMode::Hottrace:
region = selectHotTrace(ctx, selectedTIDs);
break;
case PGORegionMode::Hotblock:
region = selectHotBlock(transId, profData, cfg);
break;
case PGORegionMode::WholeCFG:
case PGORegionMode::HotCFG:
region = selectHotCFG(ctx, selectedTIDs);
break;
}
assertx(region);
if (Trace::moduleEnabled(HPHP::Trace::pgo, 5)) {
std::string dotFileName = std::string("/tmp/trans-cfg-") +
folly::to<std::string>(transId) + ".dot";
cfg.print(dotFileName, funcId, profData, &selectedTIDs);
FTRACE(5, "selectHotRegion: New Translation {} (file: {}) {}\n",
mcg->tx().profData()->curTransID(), dotFileName,
region ? show(*region) : std::string("empty region"));
}
always_assert(region->instrSize() <= RuntimeOption::EvalJitMaxRegionInstrs);
return region;
}
/**
* Add to sets coveredNodes and coveredArcs the cfg arcs that are now
* covered given the new region containing the translations in
* selectedVec.
*/
static void markCovered(const TransCFG& cfg, const RegionDescPtr region,
const TransIDVec& selectedVec, const TransIDSet heads,
TransIDSet& coveredNodes,
TransCFG::ArcPtrSet& coveredArcs) {
assert(selectedVec.size() > 0);
TransID newHead = selectedVec[0];
assert(!region->empty());
assert(newHead == getTransId(region->entry()->id()));
// Mark all region's nodes as covered.
coveredNodes.insert(selectedVec.begin(), selectedVec.end());
// Mark all incoming arcs into newHead from covered nodes as covered.
for (auto arc : cfg.inArcs(newHead)) {
TransID src = arc->src();
if (coveredNodes.count(src)) {
coveredArcs.insert(arc);
}
}
// Mark all CFG arcs within the region as covered.
region->forEachArc([&](RegionDesc::BlockId src, RegionDesc::BlockId dst) {
if (!hasTransId(src) || !hasTransId(dst)) return;
TransID srcTid = getTransId(src);
TransID dstTid = getTransId(dst);
assert(cfg.hasArc(srcTid, dstTid));
bool foundArc = false;
for (auto arc : cfg.outArcs(srcTid)) {
if (arc->dst() == dstTid) {
coveredArcs.insert(arc);
foundArc = true;
}
}
always_assert(foundArc);
});
// Mark all outgoing arcs from the region to a head node as covered.
for (auto node : selectedVec) {
for (auto arc : cfg.outArcs(node)) {
if (heads.count(arc->dst())) {
coveredArcs.insert(arc);
}
}
}
}
TransID ProfData::addTransProfile(const RegionDescPtr& region,
const PostConditions& pconds) {
TransID transId = m_numTrans++;
auto const lastBcOff = region->lastSrcKey().offset();
assertx(region);
DEBUG_ONLY auto const nBlocks = region->blocks().size();
assertx(nBlocks == 1);
region->renumberBlock(region->entry()->id(), transId);
for (auto& b : region->blocks()) b->setProfTransID(transId);
region->blocks().back()->setPostConds(pconds);
auto const startSk = region->start();
m_transRecs.emplace_back(new ProfTransRec(lastBcOff, startSk, region));
// If the translation corresponds to a DV Funclet, then add an entry
// into dvFuncletDB.
auto const func = startSk.func();
auto const funcId = func->getFuncId();
auto const bcOffset = startSk.offset();
if (func->isDVEntry(bcOffset)) {
auto const nParams = func->getDVEntryNumParams(bcOffset);
// Normal DV funclets don't have type guards, and thus have a
// single translation. However, some special functions written
// in hhas (e.g. array_map) have complex DV funclets that get
// retranslated for different types. For those functions,
// m_dvFuncletDB keeps the TransID for their first translation.
if (!m_dvFuncletDB.count(std::make_tuple(funcId, nParams))) {
m_dvFuncletDB.emplace(std::make_tuple(funcId, nParams), transId);
}
}
m_funcProfTrans[funcId].push_back(transId);
return transId;
}
TransRec::TransRec(SrcKey _src,
TransID transID,
TransKind _kind,
TCA _aStart,
uint32_t _aLen,
TCA _acoldStart,
uint32_t _acoldLen,
TCA _afrozenStart,
uint32_t _afrozenLen,
RegionDescPtr region,
std::vector<TransBCMapping> _bcMapping,
Annotations&& _annotations,
bool _hasLoop)
: bcMapping(_bcMapping)
, annotations(std::move(_annotations))
, funcName(_src.func()->fullName()->data())
, src(_src)
, md5(_src.func()->unit()->md5())
, aStart(_aStart)
, acoldStart(_acoldStart)
, afrozenStart(_afrozenStart)
, aLen(_aLen)
, acoldLen(_acoldLen)
, afrozenLen(_afrozenLen)
, bcStart(_src.offset())
, id(transID)
, kind(_kind)
, hasLoop(_hasLoop)
{
if (funcName.empty()) funcName = "Pseudo-main";
if (!region) return;
assertx(!region->empty());
for (auto& block : region->blocks()) {
auto sk = block->start();
blocks.emplace_back(Block{sk.unit()->md5(), sk.offset(),
block->last().advanced().offset()});
}
auto& firstBlock = *region->blocks().front();
for (auto const& pred : firstBlock.typePreConditions()) {
guards.emplace_back(show(pred));
}
}
TransRec::TransRec(SrcKey _src,
TransKind _kind,
TCA _aStart,
uint32_t _aLen,
TCA _acoldStart,
uint32_t _acoldLen,
TCA _afrozenStart,
uint32_t _afrozenLen,
RegionDescPtr region,
std::vector<TransBCMapping> _bcMapping,
bool _isLLVM)
: bcMapping(_bcMapping)
, funcName(_src.func()->fullName()->data())
, src(_src)
, md5(_src.func()->unit()->md5())
, aStart(_aStart)
, acoldStart(_acoldStart)
, afrozenStart(_afrozenStart)
, aLen(_aLen)
, acoldLen(_acoldLen)
, afrozenLen(_afrozenLen)
, bcStart(_src.offset())
, id(0)
, kind(_kind)
, isLLVM(_isLLVM)
{
if (funcName.empty()) funcName = "Pseudo-main";
if (!region) return;
assertx(!region->empty());
for (auto& block : region->blocks()) {
auto sk = block->start();
blocks.emplace_back(Block{sk.unit()->md5(), sk.offset(),
block->last().advanced().offset()});
}
auto& firstBlock = *region->blocks().front();
auto guardRange = firstBlock.typePreds().equal_range(firstBlock.start());
for (; guardRange.first != guardRange.second; ++guardRange.first) {
guards.emplace_back(show(guardRange.first->second));
}
}
TransID ProfData::addTransProfile(const RegionDescPtr& region,
const PostConditions& pconds) {
TransID transId = m_numTrans++;
Offset lastBcOff = findLastBcOffset(region);
assert(region);
DEBUG_ONLY size_t nBlocks = region->blocks().size();
assert(nBlocks == 1 || (nBlocks > 1 && region->entry()->inlinedCallee()));
region->renumberBlock(region->entry()->id(), transId);
region->blocks().back()->setPostConditions(pconds);
auto const startSk = region->start();
m_transRecs.emplace_back(new ProfTransRec(transId,
TransKind::Profile,
lastBcOff,
startSk,
region));
// If the translation corresponds to a DV Funclet, then add an entry
// into dvFuncletDB.
const Func* func = startSk.func();
FuncId funcId = func->getFuncId();
Offset bcOffset = startSk.offset();
if (func->isDVEntry(bcOffset)) {
int nParams = func->getDVEntryNumParams(bcOffset);
// Normal DV funclets don't have type guards, and thus have a
// single translation. However, some special functions written
// in hhas (e.g. array_map) have complex DV funclets that get
// retranslated for different types. For those functions,
// m_dvFuncletDB keeps the TransID for their first translation.
if (m_dvFuncletDB.get(funcId, nParams) == kInvalidTransID) {
m_dvFuncletDB.add(funcId, nParams, transId);
}
}
m_funcProfTrans[funcId].push_back(transId);
return transId;
}
RegionDescPtr selectHotTrace(TransID triggerId,
const ProfData* profData,
TransCFG& cfg,
TransIDSet& selectedSet,
TransIDVec* selectedVec) {
auto region = std::make_shared<RegionDesc>();
TransID tid = triggerId;
TransID prevId = kInvalidTransID;
selectedSet.clear();
if (selectedVec) selectedVec->clear();
PostConditions accumPostConds;
// Maps from BlockIds to accumulated post conditions for that block.
// Used to determine if we can add branch-over edges by checking the
// pre-conditions of the successor block.
hphp_hash_map<RegionDesc::BlockId, PostConditions> blockPostConds;
uint32_t numBCInstrs = 0;
while (!selectedSet.count(tid)) {
RegionDescPtr blockRegion = profData->transRegion(tid);
if (blockRegion == nullptr) break;
// Break if region would be larger than the specified limit.
auto newInstrSize = numBCInstrs + blockRegion->instrSize();
if (newInstrSize > RuntimeOption::EvalJitMaxRegionInstrs) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} because "
"size ({}) would exceed of maximum translation limit\n",
tid, newInstrSize);
break;
}
// If the debugger is attached, only allow single-block regions.
if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} "
"because of debugger is attached\n", tid);
break;
}
// Break if block is not the first and it corresponds to the main
// function body entry. This is to prevent creating multiple
// large regions containing the function body (starting at various
// DV funclets).
if (prevId != kInvalidTransID) {
const Func* func = profData->transFunc(tid);
Offset bcOffset = profData->transStartBcOff(tid);
if (func->base() == bcOffset) {
FTRACE(2, "selectHotTrace: breaking region because reached the main "
"function body entry at Translation {} (BC offset {})\n",
tid, bcOffset);
break;
}
}
if (prevId != kInvalidTransID) {
auto sk = profData->transSrcKey(tid);
if (profData->optimized(sk)) {
FTRACE(2, "selectHotTrace: breaking region because next sk already "
"optimized, for Translation {}\n", tid);
break;
}
}
bool hasPredBlock = !region->empty();
RegionDesc::BlockId predBlockId = (hasPredBlock ?
region->blocks().back().get()->id() : 0);
auto const& newFirstBlock = blockRegion->entry();
auto newFirstBlockId = newFirstBlock->id();
auto newLastBlockId = blockRegion->blocks().back()->id();
// Add blockRegion's blocks and arcs to region.
region->append(*blockRegion);
numBCInstrs += blockRegion->instrSize();
if (hasPredBlock) {
region->addArc(predBlockId, newFirstBlockId);
}
// When Eval.JitLoops is set, insert back-edges in the region if
// they exist in the TransCFG.
if (RuntimeOption::EvalJitLoops) {
assertx(hasTransId(newFirstBlockId));
auto newTransId = getTransId(newFirstBlockId);
// Don't add the arc if the last opcode in the source block ends
// the region.
if (!breaksRegion(*profData->transLastInstr(newTransId))) {
auto& blocks = region->blocks();
for (auto iOther = 0; iOther < blocks.size(); iOther++) {
auto other = blocks[iOther];
auto otherFirstBlockId = other.get()->id();
if (!hasTransId(otherFirstBlockId)) continue;
auto otherTransId = getTransId(otherFirstBlockId);
if (cfg.hasArc(newTransId, otherTransId)) {
region->addArc(newLastBlockId, otherFirstBlockId);
}
}
}
}
if (cfg.outArcs(tid).size() > 1) {
region->setSideExitingBlock(blockRegion->entry()->id());
}
selectedSet.insert(tid);
if (selectedVec) selectedVec->push_back(tid);
Op lastOp = *(profData->transLastInstr(tid));
if (breaksRegion(lastOp)) {
FTRACE(2, "selectHotTrace: breaking region because of last instruction "
"in Translation {}: {}\n", tid, opcodeToName(lastOp));
break;
}
auto outArcs = cfg.outArcs(tid);
if (outArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because there's no successor "
"for Translation {}\n", tid);
break;
}
auto newLastBlock = blockRegion->blocks().back();
discardPoppedTypes(accumPostConds,
blockRegion->entry()->initialSpOffset());
mergePostConds(accumPostConds, newLastBlock->postConds());
blockPostConds[newLastBlock->id()] = accumPostConds;
TransCFG::ArcPtrVec possibleOutArcs;
for (auto arc : outArcs) {
RegionDesc::BlockPtr possibleNext =
profData->transRegion(arc->dst())->entry();
if (preCondsAreSatisfied(possibleNext, accumPostConds)) {
possibleOutArcs.emplace_back(arc);
}
}
if (possibleOutArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because postcondition check "
"pruned all successors of Translation {}\n", tid);
break;
}
auto maxWeight = std::numeric_limits<int64_t>::min();
TransCFG::Arc* maxArc = nullptr;
for (auto arc : possibleOutArcs) {
if (arc->weight() >= maxWeight) {
maxWeight = arc->weight();
maxArc = arc;
}
}
assertx(maxArc != nullptr);
prevId = tid;
tid = maxArc->dst();
}
FTRACE(3, "selectHotTrace: before chainRetransBlocks:\n{}\n", show(*region));
region->chainRetransBlocks();
FTRACE(3, "selectHotTrace: after chainRetransBlocks:\n{}\n", show(*region));
return region;
}
RegionDescPtr selectHotTrace(TransID triggerId,
const ProfData* profData,
TransCFG& cfg,
TransIDSet& selectedSet,
TransIDVec* selectedVec) {
auto region = std::make_shared<RegionDesc>();
TransID tid = triggerId;
TransID prevId = kInvalidTransID;
selectedSet.clear();
if (selectedVec) selectedVec->clear();
PostConditions accumPostConds;
// Maps BlockIds to the set of BC offsets for its successor blocks.
// Used to prevent multiple successors with the same SrcKey for now.
// This can go away once task #4157613 is done.
hphp_hash_map<RegionDesc::BlockId, SrcKeySet> succSKSet;
// Maps from BlockIds to accumulated post conditions for that block.
// Used to determine if we can add branch-over edges by checking the
// pre-conditions of the successor block.
hphp_hash_map<RegionDesc::BlockId, PostConditions> blockPostConds;
while (!selectedSet.count(tid)) {
RegionDescPtr blockRegion = profData->transRegion(tid);
if (blockRegion == nullptr) break;
// If the debugger is attached, only allow single-block regions.
if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) {
FTRACE(2, "selectHotTrace: breaking region at Translation {} "
"because of debugger is attached\n", tid);
break;
}
// Break if block is not the first and it corresponds to the main
// function body entry. This is to prevent creating multiple
// large regions containing the function body (starting at various
// DV funclets).
if (prevId != kInvalidTransID) {
const Func* func = profData->transFunc(tid);
Offset bcOffset = profData->transStartBcOff(tid);
if (func->base() == bcOffset) {
FTRACE(2, "selectHotTrace: breaking region because reached the main "
"function body entry at Translation {} (BC offset {})\n",
tid, bcOffset);
break;
}
}
if (prevId != kInvalidTransID) {
auto sk = profData->transSrcKey(tid);
if (profData->optimized(sk)) {
FTRACE(2, "selectHotTrace: breaking region because next sk already "
"optimized, for Translation {}\n", tid);
break;
}
}
bool hasPredBlock = !region->empty();
RegionDesc::BlockId predBlockId = (hasPredBlock ?
region->blocks().back().get()->id() : 0);
auto const& newFirstBlock = blockRegion->entry();
auto newFirstBlockId = newFirstBlock->id();
auto newFirstBlockSk = newFirstBlock->start();
auto newLastBlockId = blockRegion->blocks().back()->id();
// Make sure we don't end up with multiple successors for the same
// SrcKey. Task #4157613 will allow the following check to go away.
// This needs to be done before we insert blockRegion into region,
// to avoid creating unreachable blocks.
if (RuntimeOption::EvalHHIRBytecodeControlFlow && hasPredBlock &&
succSKSet[predBlockId].count(newFirstBlockSk)) {
break;
}
// Add blockRegion's blocks and arcs to region.
region->append(*blockRegion);
if (hasPredBlock) {
if (RuntimeOption::EvalHHIRBytecodeControlFlow) {
// This is checked above.
assert(succSKSet[predBlockId].count(newFirstBlockSk) == 0);
succSKSet[predBlockId].insert(newFirstBlockSk);
}
region->addArc(predBlockId, newFirstBlockId);
}
// With bytecode control-flow, we add all forward arcs in the TransCFG
// that are induced by the blocks in the region, as a simple way
// to expose control-flow for now.
// This can go away once Task #4075822 is done.
if (RuntimeOption::EvalHHIRBytecodeControlFlow) {
assert(hasTransId(newFirstBlockId));
auto newTransId = getTransId(newFirstBlockId);
auto& blocks = region->blocks();
for (auto iOther = 0; iOther < blocks.size(); iOther++) {
auto other = blocks[iOther];
auto otherFirstBlockId = other.get()->id();
if (!hasTransId(otherFirstBlockId)) continue;
auto otherTransId = getTransId(otherFirstBlockId);
auto otherFirstBlockSk = other.get()->start();
auto otherRegion = profData->transRegion(otherTransId);
auto otherLastBlockId = otherRegion->blocks().back()->id();
// When loops are off, stop once we hit the newTransId we just inserted.
if (!RuntimeOption::EvalJitLoops && otherTransId == newTransId) break;
if (cfg.hasArc(otherTransId, newTransId) &&
// Task #4157613 will allow the following check to go away
!succSKSet[otherLastBlockId].count(newFirstBlockSk) &&
preCondsAreSatisfied(newFirstBlock,
blockPostConds[otherLastBlockId])) {
region->addArc(otherLastBlockId, newFirstBlockId);
succSKSet[otherLastBlockId].insert(newFirstBlockSk);
}
// When Eval.JitLoops is set, insert back-edges in the
// region if they exist in the TransCFG.
if (RuntimeOption::EvalJitLoops &&
cfg.hasArc(newTransId, otherTransId) &&
// Task #4157613 will allow the following check to go away
!succSKSet[newLastBlockId].count(otherFirstBlockSk)) {
region->addArc(newLastBlockId, otherFirstBlockId);
succSKSet[newLastBlockId].insert(otherFirstBlockSk);
}
}
}
if (cfg.outArcs(tid).size() > 1) {
region->setSideExitingBlock(blockRegion->entry()->id());
}
selectedSet.insert(tid);
if (selectedVec) selectedVec->push_back(tid);
Op lastOp = *(profData->transLastInstr(tid));
if (breaksRegion(lastOp)) {
FTRACE(2, "selectHotTrace: breaking region because of last instruction "
"in Translation {}: {}\n", tid, opcodeToName(lastOp));
break;
}
auto outArcs = cfg.outArcs(tid);
if (outArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because there's no successor "
"for Translation {}\n", tid);
break;
}
auto newLastBlock = blockRegion->blocks().back();
discardPoppedTypes(accumPostConds,
blockRegion->entry()->initialSpOffset());
mergePostConds(accumPostConds, newLastBlock->postConds());
blockPostConds[newLastBlock->id()] = accumPostConds;
TransCFG::ArcPtrVec possibleOutArcs;
for (auto arc : outArcs) {
RegionDesc::BlockPtr possibleNext =
profData->transRegion(arc->dst())->entry();
if (preCondsAreSatisfied(possibleNext, accumPostConds)) {
possibleOutArcs.emplace_back(arc);
}
}
if (possibleOutArcs.size() == 0) {
FTRACE(2, "selectHotTrace: breaking region because postcondition check "
"pruned all successors of Translation {}\n", tid);
break;
}
auto maxWeight = std::numeric_limits<int64_t>::min();
TransCFG::Arc* maxArc = nullptr;
for (auto arc : possibleOutArcs) {
if (arc->weight() >= maxWeight) {
maxWeight = arc->weight();
maxArc = arc;
}
}
assert(maxArc != nullptr);
prevId = tid;
tid = maxArc->dst();
}
return region;
}