static void recordActRecPush(const SrcKey sk,
const StringData* name,
const StringData* clsName,
bool staticCall) {
auto unit = sk.unit();
FTRACE(2, "annotation: recordActRecPush: {}@{} {}{}{} ({}static)\n",
unit->filepath()->data(),
sk.offset(),
clsName ? clsName->data() : "",
clsName ? "::" : "",
name,
!staticCall ? "non" : "");
SrcKey next(sk);
next.advance(unit);
const FPIEnt *fpi = sk.func()->findFPI(next.offset());
assert(fpi);
assert(name->isStatic());
assert(sk.offset() == fpi->m_fpushOff);
auto const fcall = SrcKey { sk.func(), fpi->m_fcallOff, sk.resumed() };
assert(isFCallStar(*reinterpret_cast<const Op*>(unit->at(fcall.offset()))));
auto const func = lookupDirectFunc(sk, name, clsName, staticCall);
if (func) {
recordFunc(fcall, func);
}
}
bool InliningDecider::canInlineAt(SrcKey callSK, const Func* callee) const {
if (!callee ||
!RuntimeOption::EvalHHIREnableGenTimeInlining ||
RuntimeOption::EvalJitEnableRenameFunction ||
callee->attrs() & AttrInterceptable) {
return false;
}
if (callee->cls()) {
if (!classHasPersistentRDS(callee->cls())) {
// if the callee's class is not persistent, its still ok
// to use it if we're jitting into a method of a subclass
auto ctx = callSK.func()->cls();
if (!ctx || !ctx->classof(callee->cls())) {
return false;
}
}
} else {
auto const handle = callee->funcHandle();
if (handle == rds::kInvalidHandle || !rds::isPersistentHandle(handle)) {
// if the callee isn't persistent, its still ok to
// use it if its defined at the top level in the same
// unit as the caller
if (callee->unit() != callSK.unit() || !callee->top()) {
return false;
}
}
}
// If inlining was disabled... don't inline.
if (m_disabled) return false;
// TODO(#3331014): We have this hack until more ARM codegen is working.
if (arch() == Arch::ARM) return false;
// We can only inline at normal FCalls.
if (callSK.op() != Op::FCall &&
callSK.op() != Op::FCallD) {
return false;
}
// Don't inline from resumed functions. The inlining mechanism doesn't have
// support for these---it has no way to redefine stack pointers relative to
// the frame pointer, because in a resumed function the frame pointer points
// into the heap instead of into the eval stack.
if (callSK.resumed()) return false;
// TODO(#4238160): Inlining into pseudomain callsites is still buggy.
if (callSK.func()->isPseudoMain()) return false;
if (!isCalleeInlinable(callSK, callee) || !checkNumArgs(callSK, callee)) {
return false;
}
return true;
}
bool InliningDecider::canInlineAt(SrcKey callSK, const Func* callee) const {
if (m_disabled ||
!callee ||
!RuntimeOption::EvalHHIREnableGenTimeInlining ||
RuntimeOption::EvalJitEnableRenameFunction ||
callee->attrs() & AttrInterceptable) {
return false;
}
// We can only inline at normal FCalls.
if (callSK.op() != Op::FCall &&
callSK.op() != Op::FCallD) {
return false;
}
// Don't inline from resumed functions. The inlining mechanism doesn't have
// support for these---it has no way to redefine stack pointers relative to
// the frame pointer, because in a resumed function the frame pointer points
// into the heap instead of into the eval stack.
if (callSK.resumed()) return false;
// TODO(#4238160): Inlining into pseudomain callsites is still buggy.
if (callSK.func()->isPseudoMain()) return false;
if (!isCalleeInlinable(callSK, callee) || !checkNumArgs(callSK, callee)) {
return false;
}
return true;
}
void prepareForNextHHBC(IRGS& env,
const NormalizedInstruction* ni,
SrcKey newSk,
bool lastBcInst) {
FTRACE(1, "------------------- prepareForNextHHBC ------------------\n");
env.currentNormalizedInstruction = ni;
always_assert_flog(
IMPLIES(isInlining(env), !env.lastBcInst),
"Tried to end trace while inlining."
);
always_assert_flog(
IMPLIES(isInlining(env), !env.firstBcInst),
"Inlining while still at the first region instruction."
);
always_assert(env.bcStateStack.size() >= env.inlineLevel + 1);
auto pops = env.bcStateStack.size() - 1 - env.inlineLevel;
while (pops--) env.bcStateStack.pop_back();
always_assert_flog(env.bcStateStack.back().func() == newSk.func(),
"Tried to update current SrcKey with a different func");
env.bcStateStack.back().setOffset(newSk.offset());
updateMarker(env);
env.lastBcInst = lastBcInst;
env.catchCreator = nullptr;
env.irb->prepareForNextHHBC();
}
TransRec::TransRec(SrcKey _src,
TransID transID,
TransKind _kind,
TCA _aStart,
uint32_t _aLen,
TCA _acoldStart,
uint32_t _acoldLen,
TCA _afrozenStart,
uint32_t _afrozenLen,
RegionDescPtr region,
std::vector<TransBCMapping> _bcMapping,
Annotations&& _annotations,
bool _hasLoop)
: bcMapping(_bcMapping)
, annotations(std::move(_annotations))
, funcName(_src.func()->fullName()->data())
, src(_src)
, md5(_src.func()->unit()->md5())
, aStart(_aStart)
, acoldStart(_acoldStart)
, afrozenStart(_afrozenStart)
, aLen(_aLen)
, acoldLen(_acoldLen)
, afrozenLen(_afrozenLen)
, bcStart(_src.offset())
, id(transID)
, kind(_kind)
, hasLoop(_hasLoop)
{
if (funcName.empty()) funcName = "Pseudo-main";
if (!region) return;
assertx(!region->empty());
for (auto& block : region->blocks()) {
auto sk = block->start();
blocks.emplace_back(Block{sk.unit()->md5(), sk.offset(),
block->last().advanced().offset()});
}
auto& firstBlock = *region->blocks().front();
for (auto const& pred : firstBlock.typePreConditions()) {
guards.emplace_back(show(pred));
}
}
TransRec::TransRec(SrcKey _src,
TransKind _kind,
TCA _aStart,
uint32_t _aLen,
TCA _acoldStart,
uint32_t _acoldLen,
TCA _afrozenStart,
uint32_t _afrozenLen,
RegionDescPtr region,
std::vector<TransBCMapping> _bcMapping,
bool _isLLVM)
: bcMapping(_bcMapping)
, funcName(_src.func()->fullName()->data())
, src(_src)
, md5(_src.func()->unit()->md5())
, aStart(_aStart)
, acoldStart(_acoldStart)
, afrozenStart(_afrozenStart)
, aLen(_aLen)
, acoldLen(_acoldLen)
, afrozenLen(_afrozenLen)
, bcStart(_src.offset())
, id(0)
, kind(_kind)
, isLLVM(_isLLVM)
{
if (funcName.empty()) funcName = "Pseudo-main";
if (!region) return;
assertx(!region->empty());
for (auto& block : region->blocks()) {
auto sk = block->start();
blocks.emplace_back(Block{sk.unit()->md5(), sk.offset(),
block->last().advanced().offset()});
}
auto& firstBlock = *region->blocks().front();
auto guardRange = firstBlock.typePreds().equal_range(firstBlock.start());
for (; guardRange.first != guardRange.second; ++guardRange.first) {
guards.emplace_back(show(guardRange.first->second));
}
}
std::string showShort(SrcKey sk) {
if (!sk.valid()) return "<invalid SrcKey>";
return folly::format(
"{}(id {:#x})@{}{}",
sk.func()->fullName(),
sk.funcID(),
sk.offset(),
sk.resumed() ? "r" : ""
).str();
}
std::string show(SrcKey sk) {
auto func = sk.func();
auto unit = sk.unit();
const char *filepath = "*anonFile*";
if (unit->filepath()->data() && unit->filepath()->size()) {
filepath = unit->filepath()->data();
}
return folly::format("{}:{} in {}(id 0x{:#x})@{: >6}",
filepath, unit->getLineNumber(sk.offset()),
func->isPseudoMain() ? "pseudoMain"
: func->fullName()->data(),
(unsigned long long)sk.getFuncId(), sk.offset()).str();
}
std::string show(SrcKey sk) {
auto func = sk.func();
auto unit = sk.unit();
const char *filepath = "*anonFile*";
if (unit->filepath()->data() && unit->filepath()->size()) {
filepath = unit->filepath()->data();
}
return folly::sformat("{}:{} in {}(id 0x{:#x})@{: >6}{}{}",
filepath, unit->getLineNumber(sk.offset()),
func->isPseudoMain() ? "pseudoMain"
: func->fullName()->data(),
(uint32_t)sk.funcID(), sk.offset(),
sk.resumed() ? "r" : "",
sk.hasThis() ? "t" : "",
sk.prologue() ? "p" : "");
}
bool profileSrcKey(SrcKey sk) {
if (!shouldPGOFunc(*sk.func())) return false;
if (profData()->optimized(sk.funcID())) return false;
if (profData()->profiling(sk.funcID())) return true;
// Don't start profiling new functions if the size of either main or
// prof is already above Eval.JitAMaxUsage and we already filled hot.
auto tcUsage = std::max(code().main().used(), code().prof().used());
if (tcUsage >= CodeCache::AMaxUsage && !code().hotEnabled()) {
return false;
}
// We have two knobs to control the number of functions we're allowed to
// profile: Eval.JitProfileRequests and Eval.JitProfileBCSize. We profile new
// functions until either of these limits is exceeded. In practice we expect
// to hit the bytecode size limit first but we keep the request limit around
// as a safety net.
if (RuntimeOption::EvalJitProfileBCSize > 0 &&
profData()->profilingBCSize() >= RuntimeOption::EvalJitProfileBCSize) {
return false;
}
return requestCount() <= RuntimeOption::EvalJitProfileRequests;
}
bool InliningDecider::canInlineAt(SrcKey callSK, const Func* callee,
const RegionDesc& region) const {
if (!RuntimeOption::RepoAuthoritative ||
!RuntimeOption::EvalHHIREnableGenTimeInlining) {
return false;
}
// If inlining was disabled... don't inline.
if (m_disabled) return false;
// TODO(#3331014): We have this hack until more ARM codegen is working.
if (arch() == Arch::ARM) return false;
// We can only inline at normal FCalls.
if (callSK.op() != Op::FCall &&
callSK.op() != Op::FCallD) {
return false;
}
// Don't inline from resumed functions. The inlining mechanism doesn't have
// support for these---it has no way to redefine stack pointers relative to
// the frame pointer, because in a resumed function the frame pointer points
// into the heap instead of into the eval stack.
if (callSK.resumed()) return false;
// TODO(#4238160): Inlining into pseudomain callsites is still buggy.
if (callSK.func()->isPseudoMain()) return false;
if (!isCalleeInlinable(callSK, callee) ||
!checkNumArgs(callSK, callee) ||
!checkFPIRegion(callSK, callee, region)) {
return false;
}
return true;
}
/*
* Checks if the given region is well-formed, which entails the
* following properties:
*
* 1) The region has at least one block.
*
* 2) Each block in the region has a different id.
*
* 3) All arcs involve blocks within the region.
*
* 4) For each arc, the bytecode offset of the dst block must
* possibly follow the execution of the src block.
*
* 5) Each block contains at most one successor corresponding to a
* given SrcKey.
*
* 6) The region doesn't contain any loops, unless JitLoops is
* enabled.
*
* 7) All blocks are reachable from the entry block.
*
* 8) For each block, there must be a path from the entry to it that
* includes only earlier blocks in the region.
*
* 9) The region is topologically sorted unless loops are enabled.
*
* 10) The block-retranslation chains cannot have cycles.
*
*/
bool check(const RegionDesc& region, std::string& error) {
auto bad = [&](const std::string& errorMsg) {
error = errorMsg;
return false;
};
// 1) The region has at least one block.
if (region.empty()) return bad("empty region");
RegionDesc::BlockIdSet blockSet;
for (auto b : region.blocks()) {
auto bid = b->id();
// 2) Each block in the region has a different id.
if (blockSet.count(bid)) {
return bad(folly::sformat("many blocks with id {}", bid));
}
blockSet.insert(bid);
}
for (auto b : region.blocks()) {
auto bid = b->id();
SrcKey lastSk = region.block(bid)->last();
OffsetSet validSuccOffsets = lastSk.succOffsets();
OffsetSet succOffsets;
for (auto succ : region.succs(bid)) {
SrcKey succSk = region.block(succ)->start();
Offset succOffset = succSk.offset();
// 3) All arcs involve blocks within the region.
if (blockSet.count(succ) == 0) {
return bad(folly::sformat("arc with dst not in the region: {} -> {}",
bid, succ));
}
// Checks 4) and 5) below don't make sense for arcs corresponding
// to inlined calls and returns, so skip them in such cases.
// This won't be possible once task #4076399 is done.
if (lastSk.func() != succSk.func()) continue;
// 4) For each arc, the bytecode offset of the dst block must
// possibly follow the execution of the src block.
if (validSuccOffsets.count(succOffset) == 0) {
return bad(folly::sformat("arc with impossible control flow: {} -> {}",
bid, succ));
}
// 5) Each block contains at most one successor corresponding to a
// given SrcKey.
if (succOffsets.count(succOffset) > 0) {
return bad(folly::sformat("block {} has multiple successors with SK {}",
bid, show(succSk)));
}
succOffsets.insert(succOffset);
}
for (auto pred : region.preds(bid)) {
if (blockSet.count(pred) == 0) {
return bad(folly::sformat("arc with src not in the region: {} -> {}",
pred, bid));
}
}
}
// 6) is checked by dfsCheck.
DFSChecker dfsCheck(region);
if (!dfsCheck.check(region.entry()->id())) {
return bad("region is cyclic");
}
// 7) All blocks are reachable from the entry (first) block.
if (dfsCheck.numVisited() != blockSet.size()) {
return bad("region has unreachable blocks");
}
// 8) and 9) are checked below.
RegionDesc::BlockIdSet visited;
auto& blocks = region.blocks();
for (unsigned i = 0; i < blocks.size(); i++) {
auto bid = blocks[i]->id();
unsigned nVisited = 0;
for (auto pred : region.preds(bid)) {
nVisited += visited.count(pred);
}
// 8) For each block, there must be a path from the entry to it that
// includes only earlier blocks in the region.
if (nVisited == 0 && i != 0) {
return bad(folly::sformat("block {} appears before all its predecessors",
bid));
}
// 9) The region is topologically sorted unless loops are enabled.
if (!RuntimeOption::EvalJitLoops && nVisited != region.preds(bid).size()) {
return bad(folly::sformat("non-topological order (bid: {})", bid));
}
visited.insert(bid);
}
// 10) The block-retranslation chains cannot have cycles.
for (auto b : blocks) {
auto bid = b->id();
RegionDesc::BlockIdSet chainSet;
chainSet.insert(bid);
while (auto next = region.nextRetrans(bid)) {
auto nextId = next.value();
if (chainSet.count(nextId)) {
return bad(folly::sformat("cyclic retranslation chain for block {}",
bid));
}
chainSet.insert(nextId);
bid = nextId;
}
}
return true;
}
std::string showShort(SrcKey sk) {
return folly::format("{}(id 0x{:#x})@{}",
sk.func()->fullName()->data(), sk.getFuncId(),
sk.offset()).str();
}
std::string showShort(SrcKey sk) {
if (!sk.valid()) return "<invalid SrcKey>";
return folly::format("{}(id {:#x})@{}",
sk.func()->fullName()->data(), sk.getFuncId(),
sk.offset()).str();
}
