void predictTypeStack(IRGS& env, BCSPOffset offset, Type type) {
FTRACE(1, "predictTypeStack {}: {}\n", offset.offset, type);
assert(type <= TGen);
auto const irSPOff = offsetFromIRSP(env, offset);
env.irb->fs().refineStackPredictedType(irSPOff, type);
}
void checkTypeStack(IRGS& env,
BCSPOffset idx,
Type type,
Offset dest,
bool outerOnly) {
assertx(type <= TCell || type <= TBoxedInitCell);
auto exit = env.irb->guardFailBlock();
if (exit == nullptr) exit = makeExit(env, dest);
if (type <= TBoxedInitCell) {
auto const soff = RelOffsetData { idx, offsetFromIRSP(env, idx) };
profiledGuard(env, TBoxedInitCell, ProfGuard::CheckStk,
idx.offset, exit);
env.irb->constrainStack(soff.irSpOffset, DataTypeSpecific);
gen(env, HintStkInner, type, soff, sp(env));
// Check inner type eargerly only at the beginning of a region.
if (!outerOnly && type.inner() < TInitCell) {
auto stk = gen(env, LdStk, TBoxedInitCell,
IRSPOffsetData{soff.irSpOffset}, sp(env));
gen(env, CheckRefInner,
env.irb->predictedStackInnerType(soff.irSpOffset),
exit, stk);
}
return;
}
FTRACE(1, "checkTypeStack({}): no tmp: {}\n", idx.offset, type.toString());
// Just like CheckType, CheckStk only cares about its input type if the
// simplifier does something with it.
profiledGuard(env, type, ProfGuard::CheckStk, idx.offset, exit);
}
void checkRefs(IRGS& env,
int64_t entryArDelta,
const std::vector<bool>& mask,
const std::vector<bool>& vals,
Offset dest) {
auto const actRecOff = entryArDelta + offsetFromIRSP(env, BCSPOffset{0});
auto const funcPtr = gen(env, LdARFuncPtr,
IRSPOffsetData { actRecOff }, sp(env));
SSATmp* nParams = nullptr;
for (unsigned i = 0; i < mask.size(); i += 64) {
assertx(i < vals.size());
uint64_t mask64 = packBitVec(mask, i);
if (mask64 == 0) {
continue;
}
if (i == 0) {
nParams = cns(env, 64);
} else if (!nParams || nParams->hasConstVal()) {
nParams = gen(env, LdFuncNumParams, funcPtr);
}
auto const vals64 = packBitVec(vals, i);
auto failBlock = env.irb->guardFailBlock();
if (failBlock == nullptr) failBlock = makeExit(env, dest);
gen(env, CheckRefs, failBlock, funcPtr, nParams,
cns(env, i), cns(env, mask64), cns(env, vals64));
}
}
void emitContEnter(IRGS& env) {
auto const returnOffset = nextBcOff(env);
assertx(curClass(env));
assertx(curClass(env)->classof(c_AsyncGenerator::classof()) ||
curClass(env)->classof(c_Generator::classof()));
assertx(curFunc(env)->contains(returnOffset));
auto isAsync = curClass(env)->classof(c_AsyncGenerator::classof());
// Load generator's FP and resume address.
auto const genObj = ldThis(env);
auto const genFp = gen(env, LdContActRec, IsAsyncData(isAsync), genObj);
auto resumeAddr = gen(env, LdContResumeAddr, IsAsyncData(isAsync), genObj);
// Make sure function enter hook is called if needed.
auto const exitSlow = makeExitSlow(env);
gen(env, CheckSurpriseFlags, exitSlow, fp(env));
// Exit to interpreter if resume address is not known.
resumeAddr = gen(env, CheckNonNull, exitSlow, resumeAddr);
spillStack(env);
env.irb->exceptionStackBoundary();
auto returnBcOffset = returnOffset - curFunc(env)->base();
gen(
env,
ContEnter,
ContEnterData { offsetFromIRSP(env, BCSPOffset{0}), returnBcOffset },
sp(env),
fp(env),
genFp,
resumeAddr
);
}
IRSPOffset offsetToReturnSlot(IRGS& env) {
return offsetFromIRSP(
env,
BCSPOffset{
logicalStackDepth(env).offset +
AROFF(m_r) / int32_t{sizeof(Cell)}
}
);
}
void endRegion(IRGS& env, SrcKey nextSk) {
FTRACE(1, "------------------- endRegion ---------------------------\n");
if (!fp(env)) {
// The function already returned. There's no reason to generate code to
// try to go to the next part of it.
return;
}
spillStack(env);
gen(env, AdjustSP, IRSPOffsetData { offsetFromIRSP(env, BCSPOffset{0}) },
sp(env));
gen(env, ReqBindJmp, ReqBindJmpData { nextSk }, sp(env));
}
// All accesses to the stack and locals in this function use DataTypeGeneric so
// this function should only be used for inspecting state; when the values are
// actually used they must be constrained further.
Type predictedTypeFromLocation(IRGS& env, const Location& loc) {
switch (loc.space) {
case Location::Stack: {
auto i = loc.bcRelOffset;
assertx(i >= 0);
if (i < env.irb->evalStack().size()) {
return topType(env, i, DataTypeGeneric);
} else {
auto stackTy = env.irb->stackType(
offsetFromIRSP(env, i),
DataTypeGeneric
);
if (stackTy <= Type::BoxedCell) {
return env.irb->stackInnerTypePrediction(
offsetFromIRSP(env, i)).box();
}
return stackTy;
}
} break;
case Location::Local:
return env.irb->predictedLocalType(loc.offset);
case Location::Litstr:
return Type::cns(curUnit(env)->lookupLitstrId(loc.offset));
case Location::Litint:
return Type::cns(loc.offset);
case Location::This:
// Don't specialize $this for cloned closures which may have been re-bound
if (curFunc(env)->hasForeignThis()) return Type::Obj;
if (auto const cls = curFunc(env)->cls()) {
return Type::SubObj(cls);
}
return Type::Obj;
case Location::Iter:
case Location::Invalid:
break;
}
not_reached();
}
Block* makeExitOpt(IRGS& env, TransID transId) {
assertx(!isInlining(env));
auto const targetBcOff = bcOff(env);
auto const exit = env.unit.defBlock(Block::Hint::Unlikely);
BlockPusher blockPusher(*env.irb, makeMarker(env, targetBcOff), exit);
spillStack(env);
auto const offset = offsetFromIRSP(env, BCSPOffset{0});
gen(env, AdjustSP, IRSPOffsetData { offset }, sp(env));
gen(env,
ReqRetranslateOpt,
ReqRetransOptData{transId, SrcKey{curSrcKey(env), targetBcOff}},
sp(env));
return exit;
}
void emitSSwitch(HTS& env, const ImmVector& iv) {
const int numCases = iv.size() - 1;
/*
* We use a fast path translation with a hashtable if none of the
* cases are numeric strings and if the input is actually a string.
*
* Otherwise we do a linear search through the cases calling string
* conversion routines.
*/
const bool fastPath =
topC(env)->isA(Type::Str) &&
std::none_of(iv.strvec(), iv.strvec() + numCases,
[&](const StrVecItem& item) {
return curUnit(env)->lookupLitstrId(item.str)->isNumeric();
}
);
auto const testVal = popC(env);
std::vector<LdSSwitchData::Elm> cases(numCases);
for (int i = 0; i < numCases; ++i) {
auto const& kv = iv.strvec()[i];
cases[i].str = curUnit(env)->lookupLitstrId(kv.str);
cases[i].dest = SrcKey{curSrcKey(env), bcOff(env) + kv.dest};
}
LdSSwitchData data;
data.numCases = numCases;
data.cases = &cases[0];
data.defaultSk = SrcKey{curSrcKey(env),
bcOff(env) + iv.strvec()[iv.size() - 1].dest};
auto const dest = gen(env,
fastPath ? LdSSwitchDestFast
: LdSSwitchDestSlow,
data,
testVal);
gen(env, DecRef, testVal);
gen(env, AdjustSP, IRSPOffsetData { offsetFromIRSP(env, BCSPOffset{0}) },
sp(env));
gen(env, JmpSSwitchDest, dest, sp(env));
}
void emitNewPackedArray(IRGS& env, int32_t numArgs) {
if (numArgs > kPackedCapCodeThreshold) {
PUNT(NewPackedArray-UnrealisticallyHuge);
}
auto const array = gen(
env,
AllocPackedArray,
PackedArrayData { static_cast<uint32_t>(numArgs) }
);
static constexpr auto kMaxUnrolledInitArray = 8;
if (numArgs > kMaxUnrolledInitArray) {
spillStack(env);
gen(
env,
InitPackedArrayLoop,
InitPackedArrayLoopData {
offsetFromIRSP(env, BCSPOffset{0}),
static_cast<uint32_t>(numArgs)
},
array,
sp(env)
);
discard(env, numArgs);
push(env, array);
return;
}
for (int i = 0; i < numArgs; ++i) {
gen(
env,
InitPackedArray,
IndexData { static_cast<uint32_t>(numArgs - i - 1) },
array,
popC(env)
);
}
push(env, array);
}
void emitNewStructArray(IRGS& env, const ImmVector& immVec) {
auto const numArgs = immVec.size();
auto const ids = immVec.vec32();
// The NewPackedArray opcode's helper needs array values passed to it
// via the stack. We use spillStack() to flush the eval stack and
// obtain a pointer to the topmost item; if over-flushing becomes
// a problem then we should refactor the NewPackedArray opcode to
// take its values directly as SSA operands.
spillStack(env);
NewStructData extra;
extra.offset = offsetFromIRSP(env, BCSPOffset{0});
extra.numKeys = numArgs;
extra.keys = new (env.unit.arena()) StringData*[numArgs];
for (auto i = size_t{0}; i < numArgs; ++i) {
extra.keys[i] = curUnit(env)->lookupLitstrId(ids[i]);
}
discard(env, numArgs);
push(env, gen(env, NewStructArray, extra, sp(env)));
}
/*
* Attempts to begin inlining, and returns whether or not it successed.
*
* When doing gen-time inlining, we set up a series of IR instructions
* that looks like this:
*
* fp0 = DefFP
* sp = DefSP<offset>
*
* // ... normal stuff happens ...
*
* // FPI region:
* SpillFrame sp, ...
* // ... probably some StStks due to argument expressions
* fp2 = DefInlineFP<func,retBC,retSP,off> sp
*
* // ... callee body ...
*
* InlineReturn fp2
*
* In DCE we attempt to remove the InlineReturn and DefInlineFP instructions if
* they aren't needed.
*/
bool beginInlining(IRGS& env,
unsigned numParams,
const Func* target,
Offset returnBcOffset) {
auto const& fpiStack = env.irb->fpiStack();
assertx(!fpiStack.empty() &&
"Inlining does not support calls with the FPush* in a different Tracelet");
assertx(returnBcOffset >= 0 && "returnBcOffset before beginning of caller");
assertx(curFunc(env)->base() + returnBcOffset < curFunc(env)->past() &&
"returnBcOffset past end of caller");
FTRACE(1, "[[[ begin inlining: {}\n", target->fullName()->data());
SSATmp** params = (SSATmp**)alloca(sizeof(SSATmp*) * numParams);
for (unsigned i = 0; i < numParams; ++i) {
params[numParams - i - 1] = popF(env);
}
auto const prevSP = fpiStack.front().returnSP;
auto const prevSPOff = fpiStack.front().returnSPOff;
spillStack(env);
auto const calleeSP = sp(env);
always_assert_flog(
prevSP == calleeSP,
"FPI stack pointer and callee stack pointer didn't match in beginInlining"
);
auto const& info = fpiStack.front();
always_assert(!isFPushCuf(info.fpushOpc) && !info.interp);
auto ctx = [&] {
if (info.ctx || isFPushFunc(info.fpushOpc)) {
return info.ctx;
}
constexpr int32_t adjust = offsetof(ActRec, m_r) - offsetof(ActRec, m_this);
IRSPOffset ctxOff{invSPOff(env) - info.returnSPOff - adjust};
return gen(env, LdStk, TCtx, IRSPOffsetData{ctxOff}, sp(env));
}();
DefInlineFPData data;
data.target = target;
data.retBCOff = returnBcOffset;
data.fromFPushCtor = isFPushCtor(info.fpushOpc);
data.ctx = ctx;
data.retSPOff = prevSPOff;
data.spOffset = offsetFromIRSP(env, BCSPOffset{0});
// Push state and update the marker before emitting any instructions so
// they're all given markers in the callee.
auto const key = SrcKey {
target,
target->getEntryForNumArgs(numParams),
false
};
env.bcStateStack.emplace_back(key);
env.inlineLevel++;
updateMarker(env);
auto const calleeFP = gen(env, DefInlineFP, data, calleeSP, fp(env));
for (unsigned i = 0; i < numParams; ++i) {
stLocRaw(env, i, calleeFP, params[i]);
}
const bool hasVariadicArg = target->hasVariadicCaptureParam();
for (unsigned i = numParams; i < target->numLocals() - hasVariadicArg; ++i) {
/*
* Here we need to be generating hopefully-dead stores to initialize
* non-parameter locals to KindOfUninit in case we have to leave the trace.
*/
stLocRaw(env, i, calleeFP, cns(env, TUninit));
}
if (hasVariadicArg) {
auto argNum = target->numLocals() - 1;
always_assert(numParams <= argNum);
stLocRaw(env, argNum, calleeFP, cns(env, staticEmptyArray()));
}
return true;
}
void interpOne(IRGS& env, Type outType, int popped) {
InterpOneData idata { offsetFromIRSP(env, BCSPOffset{0}) };
interpOne(env, outType, popped, 1, idata);
}
void assertTypeStack(IRGS& env, BCSPOffset idx, Type type) {
gen(env, AssertStk, type,
IRSPOffsetData { offsetFromIRSP(env, idx) }, sp(env));
}
void emitSwitch(HTS& env,
const ImmVector& iv,
int64_t base,
int32_t bounded) {
int nTargets = bounded ? iv.size() - 2 : iv.size();
SSATmp* const switchVal = popC(env);
Type type = switchVal->type();
assertx(IMPLIES(!(type <= (Type::Int | Type::Null)), bounded));
assertx(IMPLIES(bounded, iv.size() > 2));
SSATmp* index;
SSATmp* ssabase = cns(env, base);
SSATmp* ssatargets = cns(env, nTargets);
Offset defaultOff = bcOff(env) + iv.vec32()[iv.size() - 1];
Offset zeroOff = 0;
if (base <= 0 && (base + nTargets) > 0) {
zeroOff = bcOff(env) + iv.vec32()[0 - base];
} else {
zeroOff = defaultOff;
}
if (type <= Type::Null) {
gen(env, Jmp, makeExit(env, zeroOff));
return;
}
if (type <= Type::Bool) {
Offset nonZeroOff = bcOff(env) + iv.vec32()[iv.size() - 2];
gen(env, JmpNZero, makeExit(env, nonZeroOff), switchVal);
gen(env, Jmp, makeExit(env, zeroOff));
return;
}
if (type <= Type::Int) {
// No special treatment needed
index = switchVal;
} else if (type <= Type::Dbl) {
// switch(Double|String|Obj)Helper do bounds-checking for us, so
// we need to make sure the default case is in the jump table,
// and don't emit our own bounds-checking code
bounded = false;
index = gen(env, LdSwitchDblIndex, switchVal, ssabase, ssatargets);
} else if (type <= Type::Str) {
bounded = false;
index = gen(env, LdSwitchStrIndex, switchVal, ssabase, ssatargets);
} else if (type <= Type::Obj) {
// switchObjHelper can throw exceptions and reenter the VM so we use the
// catch block here.
bounded = false;
index = gen(env, LdSwitchObjIndex, switchVal, ssabase, ssatargets);
} else if (type <= Type::Arr) {
gen(env, DecRef, switchVal);
gen(env, Jmp, makeExit(env, defaultOff));
return;
} else {
PUNT(Switch-UnknownType);
}
std::vector<SrcKey> targets(iv.size());
for (int i = 0; i < iv.size(); i++) {
targets[i] = SrcKey{curSrcKey(env), bcOff(env) + iv.vec32()[i]};
}
JmpSwitchData data;
data.base = base;
data.bounded = bounded;
data.cases = iv.size();
data.defaultSk = {curSrcKey(env), defaultOff};
data.targets = &targets[0];
spillStack(env);
gen(env, AdjustSP, IRSPOffsetData { offsetFromIRSP(env, BCSPOffset{0}) },
sp(env));
gen(env, JmpSwitchDest, data, index, sp(env));
}
/*
* Attempts to begin inlining, and returns whether or not it successed.
*
* When doing gen-time inlining, we set up a series of IR instructions
* that looks like this:
*
* fp0 = DefFP
* sp = DefSP<offset>
*
* // ... normal stuff happens ...
*
* // FPI region:
* SpillFrame sp, ...
* // ... probably some StStks due to argument expressions
* fp2 = DefInlineFP<func,retBC,retSP,off> sp
*
* // ... callee body ...
*
* InlineReturn fp2
*
* In DCE we attempt to remove the InlineReturn and DefInlineFP instructions if
* they aren't needed.
*/
bool beginInlining(IRGS& env,
unsigned numParams,
const Func* target,
Offset returnBcOffset) {
assertx(!env.fpiStack.empty() &&
"Inlining does not support calls with the FPush* in a different Tracelet");
assertx(returnBcOffset >= 0 && "returnBcOffset before beginning of caller");
assertx(curFunc(env)->base() + returnBcOffset < curFunc(env)->past() &&
"returnBcOffset past end of caller");
FTRACE(1, "[[[ begin inlining: {}\n", target->fullName()->data());
SSATmp* params[numParams];
for (unsigned i = 0; i < numParams; ++i) {
params[numParams - i - 1] = popF(env);
}
auto const prevSP = env.fpiStack.top().returnSP;
auto const prevSPOff = env.fpiStack.top().returnSPOff;
spillStack(env);
auto const calleeSP = sp(env);
always_assert_flog(
prevSP == calleeSP,
"FPI stack pointer and callee stack pointer didn't match in beginInlining"
);
// This can only happen if the code is unreachable, in which case
// the FPush* can punt if it gets a TBottom.
if (env.fpiStack.top().spillFrame == nullptr) return false;
auto const sframe = env.fpiStack.top().spillFrame;
DefInlineFPData data;
data.target = target;
data.retBCOff = returnBcOffset;
data.fromFPushCtor = sframe->extra<ActRecInfo>()->fromFPushCtor;
data.ctx = sframe->src(2);
data.retSPOff = prevSPOff;
data.spOffset = offsetFromIRSP(env, BCSPOffset{0});
// Push state and update the marker before emitting any instructions so
// they're all given markers in the callee.
auto const key = SrcKey {
target,
target->getEntryForNumArgs(numParams),
false
};
env.bcStateStack.emplace_back(key);
env.inlineLevel++;
updateMarker(env);
auto const calleeFP = gen(env, DefInlineFP, data, calleeSP, fp(env));
for (unsigned i = 0; i < numParams; ++i) {
stLocRaw(env, i, calleeFP, params[i]);
}
const bool hasVariadicArg = target->hasVariadicCaptureParam();
for (unsigned i = numParams; i < target->numLocals() - hasVariadicArg; ++i) {
/*
* Here we need to be generating hopefully-dead stores to initialize
* non-parameter locals to KindOfUninit in case we have to leave the trace.
*/
stLocRaw(env, i, calleeFP, cns(env, TUninit));
}
if (hasVariadicArg) {
auto argNum = target->numLocals() - 1;
always_assert(numParams <= argNum);
stLocRaw(env, argNum, calleeFP, cns(env, staticEmptyArray()));
}
env.fpiActiveStack.push(std::make_pair(env.fpiStack.top().returnSP,
env.fpiStack.top().returnSPOff));
env.fpiStack.pop();
return true;
}
std::string show(const IRGS& irgs) {
std::ostringstream out;
auto header = [&](const std::string& str) {
out << folly::format("+{:-^102}+\n", str);
};
const int32_t frameCells = resumed(irgs)
? 0
: curFunc(irgs)->numSlotsInFrame();
auto const stackDepth = irgs.irb->fs().bcSPOff().offset - frameCells;
assertx(stackDepth >= 0);
auto spOffset = stackDepth;
auto elem = [&](const std::string& str) {
out << folly::format("| {:<100} |\n",
folly::format("{:>2}: {}",
stackDepth - spOffset, str));
assertx(spOffset > 0);
--spOffset;
};
auto fpi = curFunc(irgs)->findFPI(bcOff(irgs));
auto checkFpi = [&]() {
if (fpi && spOffset + frameCells == fpi->m_fpOff) {
auto fpushOff = fpi->m_fpushOff;
auto after = fpushOff + instrLen(curUnit(irgs)->at(fpushOff));
std::ostringstream msg;
msg << "ActRec from ";
curUnit(irgs)->prettyPrint(
msg,
Unit::PrintOpts().range(fpushOff, after)
.noLineNumbers()
.indent(0)
.noFuncs()
);
auto msgStr = msg.str();
assertx(msgStr.back() == '\n');
msgStr.erase(msgStr.size() - 1);
for (unsigned i = 0; i < kNumActRecCells; ++i) elem(msgStr);
fpi = fpi->m_parentIndex != -1
? &curFunc(irgs)->fpitab()[fpi->m_parentIndex]
: nullptr;
return true;
}
return false;
};
header(folly::format(" {} stack element(s): ",
stackDepth).str());
assertx(spOffset <= curFunc(irgs)->maxStackCells());
for (auto i = 0; i < spOffset; ) {
if (checkFpi()) {
i += kNumActRecCells;
continue;
}
auto const spRel = offsetFromIRSP(irgs, BCSPRelOffset{i});
auto const stkTy = irgs.irb->stack(spRel, DataTypeGeneric).type;
auto const stkVal = irgs.irb->stack(spRel, DataTypeGeneric).value;
std::string elemStr;
if (stkTy == TStkElem) {
elemStr = "unknown";
} else if (stkVal) {
elemStr = stkVal->inst()->toString();
} else {
elemStr = stkTy.toString();
}
auto const irSPRel = BCSPRelOffset{i}
.to<FPInvOffset>(irgs.irb->fs().bcSPOff());
auto const predicted = predictedType(irgs, Location::Stack { irSPRel });
if (predicted < stkTy) {
elemStr += folly::sformat(" (predict: {})", predicted);
}
elem(elemStr);
++i;
}
header("");
out << "\n";
header(folly::format(" {} local(s) ",
curFunc(irgs)->numLocals()).str());
for (unsigned i = 0; i < curFunc(irgs)->numLocals(); ++i) {
auto const localValue = irgs.irb->local(i, DataTypeGeneric).value;
auto const localTy = localValue ? localValue->type()
: irgs.irb->local(i, DataTypeGeneric).type;
auto str = localValue ? localValue->inst()->toString()
: localTy.toString();
auto const predicted = irgs.irb->fs().local(i).predictedType;
if (predicted < localTy) str += folly::sformat(" (predict: {})", predicted);
if (localTy <= TBoxedCell) {
auto const pred = irgs.irb->predictedLocalInnerType(i);
if (pred != TBottom) {
str += folly::sformat(" (predict inner: {})", pred.toString());
}
}
out << folly::format("| {:<100} |\n",
folly::format("{:>2}: {}", i, str));
}
header("");
return out.str();
}
FPAbsOffset logicalStackDepth(const IRGS& env) {
// Negate the offsetFromIRSP because it is an offset from the actual StkPtr
// (so negative values go deeper on the stack), but this function deals with
// logical stack depths (where more positive values are deeper).
return env.irb->spOffset() - offsetFromIRSP(env, BCSPOffset{0});
}
