/** * Regionize a func, so that each node and each arc in its TransCFG is * "covered". A node is covered if any region contains it. An arc T1->T2 * is covered if either: * * a) T1 and T2 are in the same region R and T2 immediately follows * T1 in R. * b) T2 is the head (first translation) of a region. * * Basic algorithm: * * 1) sort nodes in decreasing weight order * 2) for each node N: * 2.1) if N and all its incoming arcs are covered, then continue * 2.2) select a region starting at this node and mark nodes/arcs as * covered appropriately */ void regionizeFunc(const Func* func, MCGenerator* mcg, RegionVec& regions) { const Timer rf_timer(Timer::regionizeFunc); assert(RuntimeOption::EvalJitPGO); auto const funcId = func->getFuncId(); auto const profData = mcg->tx().profData(); TransCFG cfg(funcId, profData, mcg->tx().getSrcDB(), mcg->getJmpToTransIDMap()); if (Trace::moduleEnabled(HPHP::Trace::pgo, 5)) { auto dotFileName = folly::to<std::string>( "/tmp/func-cfg-", funcId, ".dot"); cfg.print(dotFileName, funcId, profData, nullptr); FTRACE(5, "regionizeFunc: initial CFG for func {} saved to file {}\n", funcId, dotFileName); } TransCFG::ArcPtrVec arcs = cfg.arcs(); std::vector<TransID> nodes = cfg.nodes(); std::sort(nodes.begin(), nodes.end(), [&](TransID tid1, TransID tid2) -> bool { if (RuntimeOption::EvalJitPGORegionSelector == "wholecfg") { auto bcOff1 = profData->transStartBcOff(tid1); auto bcOff2 = profData->transStartBcOff(tid2); if (bcOff1 != bcOff2) return bcOff1 < bcOff2; } if (cfg.weight(tid1) != cfg.weight(tid2)) { return cfg.weight(tid1) > cfg.weight(tid2); } // In case of ties, pick older translations first, in an // attempt to start loops at their headers. return tid1 < tid2; }); TransCFG::ArcPtrSet coveredArcs; TransIDSet coveredNodes; TransIDSet heads; TransIDToRegionMap headToRegion; RegionToTransIDsMap regionToTransIds; regions.clear(); for (auto node : nodes) { if (!coveredNodes.count(node) || !allArcsCovered(cfg.inArcs(node), coveredArcs)) { TransID newHead = node; FTRACE(6, "regionizeFunc: selecting trace to cover node {}\n", newHead); TransIDSet selectedSet; TransIDVec selectedVec; RegionDescPtr region; if (RuntimeOption::EvalJitPGORegionSelector == "hottrace") { region = selectHotTrace(newHead, profData, cfg, selectedSet, &selectedVec); } else if (RuntimeOption::EvalJitPGORegionSelector == "wholecfg") { region = selectWholeCFG(newHead, profData, cfg, selectedSet, &selectedVec); } else { always_assert(0 && "Invalid value for EvalJitPGORegionSelector"); } FTRACE(6, "regionizeFunc: selected region to cover node {}\n{}\n", newHead, show(*region)); profData->setOptimized(profData->transSrcKey(newHead)); assert(selectedVec.size() > 0 && selectedVec[0] == newHead); regions.push_back(region); heads.insert(newHead); markCovered(cfg, region, selectedVec, heads, coveredNodes, coveredArcs); regionToTransIds[region] = selectedVec; headToRegion[newHead] = region; FTRACE(6, "regionizeFunc: selected trace: {}\n", folly::join(", ", selectedVec)); } } assert(coveredNodes.size() == cfg.nodes().size()); assert(coveredArcs.size() == arcs.size()); sortRegions(regions, func, cfg, profData, headToRegion, regionToTransIds); if (debug && Trace::moduleEnabled(HPHP::Trace::pgo, 5)) { FTRACE(5, "\n--------------------------------------------\n" "regionizeFunc({}): computed regions:\n", funcId); for (auto region : regions) { FTRACE(5, "{}\n\n", show(*region)); } } }
RegionDescPtr selectHotTrace(TransID triggerId, const ProfData* profData, TransCFG& cfg, TransIDSet& selectedSet, TransIDVec* selectedVec) { auto region = std::make_shared<RegionDesc>(); TransID tid = triggerId; TransID prevId = kInvalidTransID; selectedSet.clear(); if (selectedVec) selectedVec->clear(); PostConditions accumPostConds; // Maps BlockIds to the set of BC offsets for its successor blocks. // Used to prevent multiple successors with the same SrcKey for now. // This can go away once task #4157613 is done. hphp_hash_map<RegionDesc::BlockId, SrcKeySet> succSKSet; // Maps from BlockIds to accumulated post conditions for that block. // Used to determine if we can add branch-over edges by checking the // pre-conditions of the successor block. hphp_hash_map<RegionDesc::BlockId, PostConditions> blockPostConds; while (!selectedSet.count(tid)) { RegionDescPtr blockRegion = profData->transRegion(tid); if (blockRegion == nullptr) break; // If the debugger is attached, only allow single-block regions. if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) { FTRACE(2, "selectHotTrace: breaking region at Translation {} " "because of debugger is attached\n", tid); break; } // Break if block is not the first and requires reffiness checks. // Task #2589970: fix translateRegion to support mid-region reffiness checks if (prevId != kInvalidTransID) { auto nRefDeps = blockRegion->blocks[0]->reffinessPreds().size(); if (nRefDeps > 0) { FTRACE(2, "selectHotTrace: breaking region because of refDeps ({}) at " "Translation {}\n", nRefDeps, tid); break; } } // Break if block is not the first and it corresponds to the main // function body entry. This is to prevent creating multiple // large regions containing the function body (starting at various // DV funclets). if (prevId != kInvalidTransID) { const Func* func = profData->transFunc(tid); Offset bcOffset = profData->transStartBcOff(tid); if (func->base() == bcOffset) { FTRACE(2, "selectHotTrace: breaking region because reached the main " "function body entry at Translation {} (BC offset {})\n", tid, bcOffset); break; } } if (prevId != kInvalidTransID) { auto sk = profData->transSrcKey(tid); if (profData->optimized(sk)) { FTRACE(2, "selectHotTrace: breaking region because next sk already " "optimized, for Translation {}\n", tid); break; } } // Break trace if translation tid cannot follow the execution of // the entire translation prevId. This can only happen if the // execution of prevId takes a side exit that leads to the // execution of tid. if (prevId != kInvalidTransID) { Op* lastInstr = profData->transLastInstr(prevId); const Unit* unit = profData->transFunc(prevId)->unit(); OffsetSet succOffs = findSuccOffsets(lastInstr, unit); if (!succOffs.count(profData->transSrcKey(tid).offset())) { if (HPHP::Trace::moduleEnabled(HPHP::Trace::pgo, 2)) { FTRACE(2, "selectHotTrace: WARNING: Breaking region @: {}\n", show(*region)); FTRACE(2, "selectHotTrace: next translation selected: tid = {}\n{}\n", tid, show(*blockRegion)); FTRACE(2, "\nsuccOffs = {}\n", folly::join(", ", succOffs)); } break; } } if (region->blocks.size() > 0) { auto& newBlock = blockRegion->blocks.front(); auto newBlockId = newBlock->id(); auto predBlockId = region->blocks.back().get()->id(); if (!RuntimeOption::EvalHHIRBytecodeControlFlow) { region->addArc(predBlockId, newBlockId); } else { // With bytecode control-flow, we add all forward arcs in the TransCFG // that are induced by the blocks in the region, as a simple way // to expose control-flow for now. // This can go away once Task #4075822 is done. auto newBlockSrcKey = blockRegion->blocks.front().get()->start(); if (succSKSet[predBlockId].count(newBlockSrcKey)) break; region->addArc(predBlockId, newBlockId); succSKSet[predBlockId].insert(newBlockSrcKey); assert(hasTransId(newBlockId)); auto newTransId = getTransId(newBlockId); for (auto iOther = 0; iOther < region->blocks.size(); iOther++) { auto other = region->blocks[iOther]; auto otherBlockId = other.get()->id(); if (!hasTransId(otherBlockId)) continue; auto otherTransId = getTransId(otherBlockId); auto otherBlockSrcKey = other.get()->start(); if (cfg.hasArc(otherTransId, newTransId) && !other.get()->inlinedCallee() && // Task #4157613 will allow the following check to go away !succSKSet[otherBlockId].count(newBlockSrcKey) && preCondsAreSatisfied(newBlock, blockPostConds[otherBlockId])) { region->addArc(otherBlockId, newBlockId); succSKSet[otherBlockId].insert(newBlockSrcKey); } // When Eval.JitLoops is set, insert back-edges in the // region if they exist in the TransCFG. if (RuntimeOption::EvalJitLoops && cfg.hasArc(newTransId, otherTransId) && // Task #4157613 will allow the following check to go away !succSKSet[newBlockId].count(otherBlockSrcKey)) { region->addArc(newBlockId, otherBlockId); succSKSet[newBlockId].insert(otherBlockSrcKey); } } } } region->blocks.insert(region->blocks.end(), blockRegion->blocks.begin(), blockRegion->blocks.end()); region->arcs.insert(region->arcs.end(), blockRegion->arcs.begin(), blockRegion->arcs.end()); if (cfg.outArcs(tid).size() > 1) { region->setSideExitingBlock(blockRegion->blocks.front()->id()); } selectedSet.insert(tid); if (selectedVec) selectedVec->push_back(tid); Op lastOp = *(profData->transLastInstr(tid)); if (breaksRegion(lastOp)) { FTRACE(2, "selectHotTrace: breaking region because of last instruction " "in Translation {}: {}\n", tid, opcodeToName(lastOp)); break; } auto outArcs = cfg.outArcs(tid); if (outArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because there's no successor " "for Translation {}\n", tid); break; } auto lastNewBlock = blockRegion->blocks.back(); discardPoppedTypes(accumPostConds, blockRegion->blocks[0]->initialSpOffset()); mergePostConds(accumPostConds, lastNewBlock->postConds()); blockPostConds[lastNewBlock->id()] = accumPostConds; TransCFG::ArcPtrVec possibleOutArcs; for (auto arc : outArcs) { RegionDesc::BlockPtr possibleNext = profData->transRegion(arc->dst())->blocks[0]; if (preCondsAreSatisfied(possibleNext, accumPostConds)) { possibleOutArcs.emplace_back(arc); } } if (possibleOutArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because postcondition check " "pruned all successors of Translation {}\n", tid); break; } auto maxWeight = std::numeric_limits<int64_t>::min(); TransCFG::Arc* maxArc = nullptr; for (auto arc : possibleOutArcs) { if (arc->weight() >= maxWeight) { maxWeight = arc->weight(); maxArc = arc; } } assert(maxArc != nullptr); prevId = tid; tid = maxArc->dst(); } return region; }
RegionDescPtr selectHotTrace(HotTransContext& ctx, TransIDSet& selectedSet, TransIDVec* selectedVec /* = nullptr */) { auto region = std::make_shared<RegionDesc>(); TransID tid = ctx.tid; TransID prevId = kInvalidTransID; selectedSet.clear(); if (selectedVec) selectedVec->clear(); TypedLocations accumPostConds; // Maps from BlockIds to accumulated post conditions for that block. // Used to determine if we can add branch-over edges by checking the // pre-conditions of the successor block. hphp_hash_map<RegionDesc::BlockId, TypedLocations> blockPostConds; auto numBCInstrs = ctx.maxBCInstrs; FTRACE(1, "selectHotTrace: starting with maxBCInstrs = {}\n", numBCInstrs); while (!selectedSet.count(tid)) { auto rec = ctx.profData->transRec(tid); auto blockRegion = rec->region(); if (blockRegion == nullptr) break; // Break if region would be larger than the specified limit. if (blockRegion->instrSize() > numBCInstrs) { FTRACE(2, "selectHotTrace: breaking region at Translation {} because " "size would exceed of maximum translation limit\n", tid); break; } // If the debugger is attached, only allow single-block regions. if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) { FTRACE(2, "selectHotTrace: breaking region at Translation {} " "because of debugger is attached\n", tid); break; } // Break if block is not the first and it corresponds to the main // function body entry. This is to prevent creating multiple // large regions containing the function body (starting at various // DV funclets). if (prevId != kInvalidTransID) { auto const func = rec->func(); auto const bcOffset = rec->startBcOff(); if (func->base() == bcOffset) { FTRACE(2, "selectHotTrace: breaking region because reached the main " "function body entry at Translation {} (BC offset {})\n", tid, bcOffset); break; } } if (prevId != kInvalidTransID) { auto sk = rec->srcKey(); if (ctx.profData->optimized(sk)) { FTRACE(2, "selectHotTrace: breaking region because next sk already " "optimized, for Translation {}\n", tid); break; } } bool hasPredBlock = !region->empty(); RegionDesc::BlockId predBlockId = (hasPredBlock ? region->blocks().back().get()->id() : 0); auto const& newFirstBlock = blockRegion->entry(); auto newFirstBlockId = newFirstBlock->id(); // Add blockRegion's blocks and arcs to region. region->append(*blockRegion); numBCInstrs -= blockRegion->instrSize(); assertx(numBCInstrs >= 0); if (hasPredBlock) { region->addArc(predBlockId, newFirstBlockId); } selectedSet.insert(tid); if (selectedVec) selectedVec->push_back(tid); const auto lastSk = rec->lastSrcKey(); if (breaksRegion(lastSk)) { FTRACE(2, "selectHotTrace: breaking region because of last instruction " "in Translation {}: {}\n", tid, opcodeToName(lastSk.op())); break; } auto outArcs = ctx.cfg->outArcs(tid); if (outArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because there's no successor " "for Translation {}\n", tid); break; } auto newLastBlock = blockRegion->blocks().back(); discardPoppedTypes(accumPostConds, blockRegion->entry()->initialSpOffset()); mergePostConds(accumPostConds, newLastBlock->postConds()); blockPostConds[newLastBlock->id()] = accumPostConds; TransCFG::ArcPtrVec possibleOutArcs; for (auto arc : outArcs) { auto dstRec = ctx.profData->transRec(arc->dst()); auto possibleNext = dstRec->region()->entry(); if (preCondsAreSatisfied(possibleNext, accumPostConds)) { possibleOutArcs.emplace_back(arc); } } if (possibleOutArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because postcondition check " "pruned all successors of Translation {}\n", tid); break; } auto maxWeight = std::numeric_limits<int64_t>::min(); TransCFG::Arc* maxArc = nullptr; for (auto arc : possibleOutArcs) { if (arc->weight() >= maxWeight) { maxWeight = arc->weight(); maxArc = arc; } } assertx(maxArc != nullptr); prevId = tid; tid = maxArc->dst(); } FTRACE(3, "selectHotTrace: before chainRetransBlocks:\n{}\n", show(*region)); region->chainRetransBlocks(); FTRACE(3, "selectHotTrace: after chainRetransBlocks:\n{}\n", show(*region)); return region; }
RegionDescPtr selectHotTrace(TransID triggerId, const ProfData* profData, TransCFG& cfg, TransIDSet& selectedSet, TransIDVec* selectedVec) { auto region = std::make_shared<RegionDesc>(); TransID tid = triggerId; TransID prevId = kInvalidTransID; selectedSet.clear(); if (selectedVec) selectedVec->clear(); PostConditions accumPostConds; // Maps from BlockIds to accumulated post conditions for that block. // Used to determine if we can add branch-over edges by checking the // pre-conditions of the successor block. hphp_hash_map<RegionDesc::BlockId, PostConditions> blockPostConds; uint32_t numBCInstrs = 0; while (!selectedSet.count(tid)) { RegionDescPtr blockRegion = profData->transRegion(tid); if (blockRegion == nullptr) break; // Break if region would be larger than the specified limit. auto newInstrSize = numBCInstrs + blockRegion->instrSize(); if (newInstrSize > RuntimeOption::EvalJitMaxRegionInstrs) { FTRACE(2, "selectHotTrace: breaking region at Translation {} because " "size ({}) would exceed of maximum translation limit\n", tid, newInstrSize); break; } // If the debugger is attached, only allow single-block regions. if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) { FTRACE(2, "selectHotTrace: breaking region at Translation {} " "because of debugger is attached\n", tid); break; } // Break if block is not the first and it corresponds to the main // function body entry. This is to prevent creating multiple // large regions containing the function body (starting at various // DV funclets). if (prevId != kInvalidTransID) { const Func* func = profData->transFunc(tid); Offset bcOffset = profData->transStartBcOff(tid); if (func->base() == bcOffset) { FTRACE(2, "selectHotTrace: breaking region because reached the main " "function body entry at Translation {} (BC offset {})\n", tid, bcOffset); break; } } if (prevId != kInvalidTransID) { auto sk = profData->transSrcKey(tid); if (profData->optimized(sk)) { FTRACE(2, "selectHotTrace: breaking region because next sk already " "optimized, for Translation {}\n", tid); break; } } bool hasPredBlock = !region->empty(); RegionDesc::BlockId predBlockId = (hasPredBlock ? region->blocks().back().get()->id() : 0); auto const& newFirstBlock = blockRegion->entry(); auto newFirstBlockId = newFirstBlock->id(); auto newLastBlockId = blockRegion->blocks().back()->id(); // Add blockRegion's blocks and arcs to region. region->append(*blockRegion); numBCInstrs += blockRegion->instrSize(); if (hasPredBlock) { region->addArc(predBlockId, newFirstBlockId); } // When Eval.JitLoops is set, insert back-edges in the region if // they exist in the TransCFG. if (RuntimeOption::EvalJitLoops) { assertx(hasTransId(newFirstBlockId)); auto newTransId = getTransId(newFirstBlockId); // Don't add the arc if the last opcode in the source block ends // the region. if (!breaksRegion(*profData->transLastInstr(newTransId))) { auto& blocks = region->blocks(); for (auto iOther = 0; iOther < blocks.size(); iOther++) { auto other = blocks[iOther]; auto otherFirstBlockId = other.get()->id(); if (!hasTransId(otherFirstBlockId)) continue; auto otherTransId = getTransId(otherFirstBlockId); if (cfg.hasArc(newTransId, otherTransId)) { region->addArc(newLastBlockId, otherFirstBlockId); } } } } if (cfg.outArcs(tid).size() > 1) { region->setSideExitingBlock(blockRegion->entry()->id()); } selectedSet.insert(tid); if (selectedVec) selectedVec->push_back(tid); Op lastOp = *(profData->transLastInstr(tid)); if (breaksRegion(lastOp)) { FTRACE(2, "selectHotTrace: breaking region because of last instruction " "in Translation {}: {}\n", tid, opcodeToName(lastOp)); break; } auto outArcs = cfg.outArcs(tid); if (outArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because there's no successor " "for Translation {}\n", tid); break; } auto newLastBlock = blockRegion->blocks().back(); discardPoppedTypes(accumPostConds, blockRegion->entry()->initialSpOffset()); mergePostConds(accumPostConds, newLastBlock->postConds()); blockPostConds[newLastBlock->id()] = accumPostConds; TransCFG::ArcPtrVec possibleOutArcs; for (auto arc : outArcs) { RegionDesc::BlockPtr possibleNext = profData->transRegion(arc->dst())->entry(); if (preCondsAreSatisfied(possibleNext, accumPostConds)) { possibleOutArcs.emplace_back(arc); } } if (possibleOutArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because postcondition check " "pruned all successors of Translation {}\n", tid); break; } auto maxWeight = std::numeric_limits<int64_t>::min(); TransCFG::Arc* maxArc = nullptr; for (auto arc : possibleOutArcs) { if (arc->weight() >= maxWeight) { maxWeight = arc->weight(); maxArc = arc; } } assertx(maxArc != nullptr); prevId = tid; tid = maxArc->dst(); } FTRACE(3, "selectHotTrace: before chainRetransBlocks:\n{}\n", show(*region)); region->chainRetransBlocks(); FTRACE(3, "selectHotTrace: after chainRetransBlocks:\n{}\n", show(*region)); return region; }
/** * Regionize a func, so that each node and each arc in its TransCFG is * "covered". A node is covered if any region contains it. An arc T1->T2 * is covered if either: * * a) T1 and T2 are in the same region R and T2 immediately follows * T1 in R. * b) T2 is the head (first translation) of a region. * * Basic algorithm: * * 1) sort nodes in decreasing weight order * 2) for each node N: * 2.1) if N and all its incoming arcs are covered, then continue * 2.2) select a region starting at this node and mark nodes/arcs as * covered appropriately */ void regionizeFunc(const Func* func, Transl::TranslatorX64* tx64, RegionVec& regions) { assert(RuntimeOption::EvalJitPGO); FuncId funcId = func->getFuncId(); ProfData* profData = tx64->profData(); TransCFG cfg(funcId, profData, tx64->getSrcDB(), tx64->getJmpToTransIDMap()); if (Trace::moduleEnabled(HPHP::Trace::pgo, 5)) { string dotFileName = folly::to<string>("/tmp/func-cfg-", funcId, ".dot"); cfg.print(dotFileName, funcId, profData, nullptr); FTRACE(5, "regionizeFunc: initial CFG for func {} saved to file {}\n", funcId, dotFileName); } TransCFG::ArcPtrVec arcs = cfg.arcs(); vector<TransID> nodes = cfg.nodes(); std::sort(nodes.begin(), nodes.end(), [&](TransID tid1, TransID tid2) -> bool { if (cfg.weight(tid1) != cfg.weight(tid2)) { return cfg.weight(tid1) > cfg.weight(tid2); } // In case of ties, pick older translations first, in an // attempt to start loops at their headers. return tid1 < tid2; }); TransCFG::ArcPtrSet coveredArcs; TransIDSet coveredNodes; TransIDSet heads; TransIDToRegionMap headToRegion; RegionToTransIDsMap regionToTransIds; regions.clear(); for (auto node : nodes) { if (!setContains(coveredNodes, node) || !allArcsCovered(cfg.inArcs(node), coveredArcs)) { TransID newHead = node; FTRACE(6, "regionizeFunc: selecting trace to cover node {}\n", newHead); TransIDSet selectedSet; TransIDVec selectedVec; RegionDescPtr region = selectHotTrace(newHead, profData, cfg, selectedSet, &selectedVec); profData->setOptimized(profData->transSrcKey(newHead)); assert(selectedVec.size() > 0 && selectedVec[0] == newHead); regions.push_back(region); heads.insert(newHead); markCovered(cfg, selectedVec, heads, coveredNodes, coveredArcs); regionToTransIds[region] = selectedVec; headToRegion[newHead] = region; FTRACE(6, "regionizeFunc: selected trace: {}\n", folly::join(", ", selectedVec)); } } assert(coveredNodes.size() == cfg.nodes().size()); assert(coveredArcs.size() == arcs.size()); sortRegion(regions, func, cfg, profData, headToRegion, regionToTransIds); if (debug && Trace::moduleEnabled(HPHP::Trace::pgo, 5)) { FTRACE(5, "\n--------------------------------------------\n" "regionizeFunc({}): computed regions:\n", funcId); for (auto region : regions) { FTRACE(5, "{}\n\n", show(*region)); } } }
RegionDescPtr selectHotTrace(TransID triggerId, const ProfData* profData, TransCFG& cfg, TransIDSet& selectedSet, TransIDVec* selectedVec) { auto region = std::make_shared<RegionDesc>(); TransID tid = triggerId; TransID prevId = InvalidID; selectedSet.clear(); if (selectedVec) selectedVec->clear(); PostConditions accumPostConds; while (!selectedSet.count(tid)) { RegionDescPtr blockRegion = profData->transRegion(tid); if (blockRegion == nullptr) break; // If the debugger is attached, only allow single-block regions. if (prevId != InvalidID && isDebuggerAttachedProcess()) { FTRACE(2, "selectHotTrace: breaking region at Translation {} " "because of debugger is attached\n", tid); break; } // Break if block is not the first and requires reffiness checks. // Task #2589970: fix translateRegion to support mid-region reffiness checks if (prevId != InvalidID) { auto nRefDeps = blockRegion->blocks[0]->reffinessPreds().size(); if (nRefDeps > 0) { FTRACE(2, "selectHotTrace: breaking region because of refDeps ({}) at " "Translation {}\n", nRefDeps, tid); break; } } // Break if block is not the first and it corresponds to the main // function body entry. This is to prevent creating multiple // large regions containing the function body (starting at various // DV funclets). if (prevId != InvalidID) { const Func* func = profData->transFunc(tid); Offset bcOffset = profData->transStartBcOff(tid); if (func->base() == bcOffset) { FTRACE(2, "selectHotTrace: breaking region because reached the main " "function body entry at Translation {} (BC offset {})\n", tid, bcOffset); break; } } if (prevId != InvalidID) { auto sk = profData->transSrcKey(tid); if (profData->optimized(sk)) { FTRACE(2, "selectHotTrace: breaking region because next sk already " "optimized, for Translation {}\n", tid); break; } } // Break trace if translation tid cannot follow the execution of // the entire translation prevId. This can only happen if the // execution of prevId takes a side exit that leads to the // execution of tid. if (prevId != InvalidID) { Op* lastInstr = profData->transLastInstr(prevId); const Unit* unit = profData->transFunc(prevId)->unit(); OffsetSet succOffs = findSuccOffsets(lastInstr, unit); if (!succOffs.count(profData->transSrcKey(tid).offset())) { if (HPHP::Trace::moduleEnabled(HPHP::Trace::pgo, 2)) { FTRACE(2, "selectHotTrace: WARNING: Breaking region @: {}\n", JIT::show(*region)); FTRACE(2, "selectHotTrace: next translation selected: tid = {}\n{}\n", tid, JIT::show(*blockRegion)); FTRACE(2, "\nsuccOffs = {}\n", folly::join(", ", succOffs)); } break; } } region->blocks.insert(region->blocks.end(), blockRegion->blocks.begin(), blockRegion->blocks.end()); selectedSet.insert(tid); if (selectedVec) selectedVec->push_back(tid); Op lastOp = *(profData->transLastInstr(tid)); if (breaksRegion(lastOp)) { FTRACE(2, "selectHotTrace: breaking region because of last instruction " "in Translation {}: {}\n", tid, opcodeToName(lastOp)); break; } auto outArcs = cfg.outArcs(tid); if (outArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because there's no successor " "for Translation {}\n", tid); break; } auto lastNewBlock = blockRegion->blocks.back(); discardPoppedTypes(accumPostConds, blockRegion->blocks[0]->initialSpOffset()); mergePostConds(accumPostConds, lastNewBlock->postConds()); TransCFG::ArcPtrVec possibleOutArcs; for (auto arc : outArcs) { RegionDesc::BlockPtr possibleNext = profData->transRegion(arc->dst())->blocks[0]; if (preCondsAreSatisfied(possibleNext, accumPostConds)) { possibleOutArcs.emplace_back(arc); } } if (possibleOutArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because postcondition check " "pruned all successors of Translation {}\n", tid); break; } auto maxWeight = std::numeric_limits<int64_t>::min(); TransCFG::Arc* maxArc = nullptr; for (auto arc : possibleOutArcs) { if (arc->weight() >= maxWeight) { maxWeight = arc->weight(); maxArc = arc; } } assert(maxArc != nullptr); prevId = tid; tid = maxArc->dst(); } return region; }
RegionDescPtr selectHotTrace(TransID triggerId, const ProfData* profData, TransCFG& cfg, TransIDSet& selectedSet, TransIDVec* selectedVec) { auto region = std::make_shared<RegionDesc>(); TransID tid = triggerId; TransID prevId = kInvalidTransID; selectedSet.clear(); if (selectedVec) selectedVec->clear(); PostConditions accumPostConds; // Maps BlockIds to the set of BC offsets for its successor blocks. // Used to prevent multiple successors with the same SrcKey for now. // This can go away once task #4157613 is done. hphp_hash_map<RegionDesc::BlockId, SrcKeySet> succSKSet; // Maps from BlockIds to accumulated post conditions for that block. // Used to determine if we can add branch-over edges by checking the // pre-conditions of the successor block. hphp_hash_map<RegionDesc::BlockId, PostConditions> blockPostConds; while (!selectedSet.count(tid)) { RegionDescPtr blockRegion = profData->transRegion(tid); if (blockRegion == nullptr) break; // If the debugger is attached, only allow single-block regions. if (prevId != kInvalidTransID && isDebuggerAttachedProcess()) { FTRACE(2, "selectHotTrace: breaking region at Translation {} " "because of debugger is attached\n", tid); break; } // Break if block is not the first and it corresponds to the main // function body entry. This is to prevent creating multiple // large regions containing the function body (starting at various // DV funclets). if (prevId != kInvalidTransID) { const Func* func = profData->transFunc(tid); Offset bcOffset = profData->transStartBcOff(tid); if (func->base() == bcOffset) { FTRACE(2, "selectHotTrace: breaking region because reached the main " "function body entry at Translation {} (BC offset {})\n", tid, bcOffset); break; } } if (prevId != kInvalidTransID) { auto sk = profData->transSrcKey(tid); if (profData->optimized(sk)) { FTRACE(2, "selectHotTrace: breaking region because next sk already " "optimized, for Translation {}\n", tid); break; } } bool hasPredBlock = !region->empty(); RegionDesc::BlockId predBlockId = (hasPredBlock ? region->blocks().back().get()->id() : 0); auto const& newFirstBlock = blockRegion->entry(); auto newFirstBlockId = newFirstBlock->id(); auto newFirstBlockSk = newFirstBlock->start(); auto newLastBlockId = blockRegion->blocks().back()->id(); // Make sure we don't end up with multiple successors for the same // SrcKey. Task #4157613 will allow the following check to go away. // This needs to be done before we insert blockRegion into region, // to avoid creating unreachable blocks. if (RuntimeOption::EvalHHIRBytecodeControlFlow && hasPredBlock && succSKSet[predBlockId].count(newFirstBlockSk)) { break; } // Add blockRegion's blocks and arcs to region. region->append(*blockRegion); if (hasPredBlock) { if (RuntimeOption::EvalHHIRBytecodeControlFlow) { // This is checked above. assert(succSKSet[predBlockId].count(newFirstBlockSk) == 0); succSKSet[predBlockId].insert(newFirstBlockSk); } region->addArc(predBlockId, newFirstBlockId); } // With bytecode control-flow, we add all forward arcs in the TransCFG // that are induced by the blocks in the region, as a simple way // to expose control-flow for now. // This can go away once Task #4075822 is done. if (RuntimeOption::EvalHHIRBytecodeControlFlow) { assert(hasTransId(newFirstBlockId)); auto newTransId = getTransId(newFirstBlockId); auto& blocks = region->blocks(); for (auto iOther = 0; iOther < blocks.size(); iOther++) { auto other = blocks[iOther]; auto otherFirstBlockId = other.get()->id(); if (!hasTransId(otherFirstBlockId)) continue; auto otherTransId = getTransId(otherFirstBlockId); auto otherFirstBlockSk = other.get()->start(); auto otherRegion = profData->transRegion(otherTransId); auto otherLastBlockId = otherRegion->blocks().back()->id(); // When loops are off, stop once we hit the newTransId we just inserted. if (!RuntimeOption::EvalJitLoops && otherTransId == newTransId) break; if (cfg.hasArc(otherTransId, newTransId) && // Task #4157613 will allow the following check to go away !succSKSet[otherLastBlockId].count(newFirstBlockSk) && preCondsAreSatisfied(newFirstBlock, blockPostConds[otherLastBlockId])) { region->addArc(otherLastBlockId, newFirstBlockId); succSKSet[otherLastBlockId].insert(newFirstBlockSk); } // When Eval.JitLoops is set, insert back-edges in the // region if they exist in the TransCFG. if (RuntimeOption::EvalJitLoops && cfg.hasArc(newTransId, otherTransId) && // Task #4157613 will allow the following check to go away !succSKSet[newLastBlockId].count(otherFirstBlockSk)) { region->addArc(newLastBlockId, otherFirstBlockId); succSKSet[newLastBlockId].insert(otherFirstBlockSk); } } } if (cfg.outArcs(tid).size() > 1) { region->setSideExitingBlock(blockRegion->entry()->id()); } selectedSet.insert(tid); if (selectedVec) selectedVec->push_back(tid); Op lastOp = *(profData->transLastInstr(tid)); if (breaksRegion(lastOp)) { FTRACE(2, "selectHotTrace: breaking region because of last instruction " "in Translation {}: {}\n", tid, opcodeToName(lastOp)); break; } auto outArcs = cfg.outArcs(tid); if (outArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because there's no successor " "for Translation {}\n", tid); break; } auto newLastBlock = blockRegion->blocks().back(); discardPoppedTypes(accumPostConds, blockRegion->entry()->initialSpOffset()); mergePostConds(accumPostConds, newLastBlock->postConds()); blockPostConds[newLastBlock->id()] = accumPostConds; TransCFG::ArcPtrVec possibleOutArcs; for (auto arc : outArcs) { RegionDesc::BlockPtr possibleNext = profData->transRegion(arc->dst())->entry(); if (preCondsAreSatisfied(possibleNext, accumPostConds)) { possibleOutArcs.emplace_back(arc); } } if (possibleOutArcs.size() == 0) { FTRACE(2, "selectHotTrace: breaking region because postcondition check " "pruned all successors of Translation {}\n", tid); break; } auto maxWeight = std::numeric_limits<int64_t>::min(); TransCFG::Arc* maxArc = nullptr; for (auto arc : possibleOutArcs) { if (arc->weight() >= maxWeight) { maxWeight = arc->weight(); maxArc = arc; } } assert(maxArc != nullptr); prevId = tid; tid = maxArc->dst(); } return region; }
RegionDescPtr selectHotTrace(TransID triggerId, const ProfData* profData, TransCFG& cfg, TransIDSet& selectedSet) { JIT::RegionDescPtr region = std::make_shared<JIT::RegionDesc>(); TransID tid = triggerId; TransID prevId = InvalidID; selectedSet.clear(); PostConditions accumPostConds; while (!setContains(selectedSet, tid)) { RegionDescPtr blockRegion = profData->transRegion(tid); if (blockRegion == nullptr) break; // If the debugger is attached, only allow single-block regions. if (prevId != InvalidID && isDebuggerAttachedProcess()) { FTRACE(5, "selectHotRegion: breaking region at Translation {} " "because of debugger is attached\n", tid); break; } // Break if block is not the first and requires reffiness checks. // Task #2589970: fix translateRegion to support mid-region reffiness checks if (prevId != InvalidID) { auto nRefDeps = blockRegion->blocks[0]->reffinessPreds().size(); if (nRefDeps > 0) { FTRACE(5, "selectHotRegion: breaking region because of refDeps ({}) at " "Translation {}\n", nRefDeps, tid); break; } } // Break trace if translation tid cannot follow the execution of // the entire translation prevTd. This can only happen if the // execution of prevId takes a side exit that leads to the // execution of tid. if (prevId != InvalidID) { Op* lastInstr = profData->transLastInstr(prevId); const Unit* unit = profData->transFunc(prevId)->unit(); OffsetSet succOffs = findSuccOffsets(lastInstr, unit); if (!setContains(succOffs, profData->transSrcKey(tid).offset())) { if (HPHP::Trace::moduleEnabled(HPHP::Trace::pgo, 5)) { FTRACE(5, "selectHotTrace: WARNING: Breaking region @: {}\n", JIT::show(*region)); FTRACE(5, "selectHotTrace: next translation selected: tid = {}\n{}\n", tid, JIT::show(*blockRegion)); std::string succStr("succOffs = "); for (auto succ : succOffs) { succStr += lexical_cast<std::string>(succ); } FTRACE(5, "\n{}\n", succStr); } break; } } region->blocks.insert(region->blocks.end(), blockRegion->blocks.begin(), blockRegion->blocks.end()); selectedSet.insert(tid); Op lastOp = *(profData->transLastInstr(tid)); if (breaksRegion(lastOp)) { FTRACE(5, "selectHotTrace: breaking region because of last instruction " "in Translation {}: {}\n", tid, opcodeToName(lastOp)); break; } auto outArcs = cfg.outArcs(tid); if (outArcs.size() == 0) { FTRACE(5, "selectHotTrace: breaking region because there's no successor " "for Translation {}\n", tid); break; } auto lastNewBlock = blockRegion->blocks.back(); mergePostConds(accumPostConds, lastNewBlock->postConds()); TransCFG::ArcPtrVec possibleOutArcs; for (auto arc : outArcs) { RegionDesc::BlockPtr possibleNext = profData->transRegion(arc->dst())->blocks[0]; if (preCondsAreSatisfied(possibleNext, accumPostConds)) { possibleOutArcs.emplace_back(arc); } } if (possibleOutArcs.size() == 0) { FTRACE(5, "selectHotTrace: breaking region because postcondition check " "pruned all successors of Translation {}\n", tid); break; } auto maxWeight = std::numeric_limits<int64_t>::min(); TransCFG::Arc* maxArc = nullptr; for (auto arc : possibleOutArcs) { if (arc->weight() >= maxWeight) { maxWeight = arc->weight(); maxArc = arc; } } assert(maxArc != nullptr); prevId = tid; tid = maxArc->dst(); } return region; }