void cgOrdStrIdx(IRLS& env, const IRInstruction* inst) {
auto const sd = srcLoc(env, inst, 0).reg();
auto const idx = srcLoc(env, inst, 1).reg();
auto& v = vmain(env);
auto const sf = v.makeReg();
auto const length = v.makeReg();
v << loadzlq{sd[StringData::sizeOff()], length};
v << cmpq{idx, length, sf};
unlikelyCond(v, vcold(env), CC_B, sf, dstLoc(env, inst, 0).reg(),
[&] (Vout& v) {
auto const args = argGroup(env, inst).ssa(0).ssa(1);
cgCallHelper(v, env, CallSpec::direct(MInstrHelpers::stringGetI),
kVoidDest, SyncOptions::Sync, args);
return v.cns(0);
},
[&] (Vout& v) {
auto const dst = v.makeReg();
auto const data = v.makeReg();
#ifdef NO_M_DATA
v << lea{sd[sizeof(StringData)], data};
#else
v << load{sd[StringData::dataOff()], data};
#endif
v << loadzbq{data[idx], dst};
return dst;
}
);
}
void cgStLocRange(IRLS& env, const IRInstruction* inst) {
auto const range = inst->extra<StLocRange>();
if (range->start >= range->end) return;
auto const fp = srcLoc(env, inst, 0).reg();
auto const loc = srcLoc(env, inst, 1);
auto const val = inst->src(1);
auto& v = vmain(env);
auto ireg = v.makeReg();
auto nreg = v.makeReg();
v << lea{fp[localOffset(range->start)], ireg};
v << lea{fp[localOffset(range->end)], nreg};
doWhile(v, CC_NE, {ireg},
[&] (const VregList& in, const VregList& out) {
auto const i = in[0];
auto const res = out[0];
auto const sf = v.makeReg();
storeTV(v, i[0], loc, val);
v << subqi{int32_t{sizeof(Cell)}, i, res, v.makeReg()};
v << cmpq{res, nreg, sf};
return sf;
}
);
}
void cgContEnter(IRLS& env, const IRInstruction* inst) {
auto const sp = srcLoc(env, inst, 0).reg();
auto const fp = srcLoc(env, inst, 1).reg();
auto const genFP = srcLoc(env, inst, 2).reg();
auto const target = srcLoc(env, inst, 3).reg();
auto const extra = inst->extra<ContEnter>();
auto const spOff = extra->spOffset;
auto const returnOff = extra->returnBCOffset;
auto& v = vmain(env);
auto const next = v.makeBlock();
v << store{fp, genFP[AROFF(m_sfp)]};
v << storeli{returnOff, genFP[AROFF(m_soff)]};
v << copy{genFP, fp};
auto const sync_sp = v.makeReg();
v << lea{sp[cellsToBytes(spOff.offset)], sync_sp};
v << syncvmsp{sync_sp};
v << contenter{fp, target, cross_trace_regs_resumed(),
{next, label(env, inst->taken())}};
v = next;
auto const dst = dstLoc(env, inst, 0);
auto const type = inst->dst()->type();
if (!type.admitsSingleVal()) {
v << defvmretdata{dst.reg(0)};
}
if (type.needsReg()) {
v << defvmrettype{dst.reg(1)};
}
}
void cgStRef(IRLS& env, const IRInstruction* inst) {
auto const ptr = srcLoc(env, inst, 0).reg();
auto const valLoc = srcLoc(env, inst, 1);
always_assert(!srcLoc(env, inst, 1).isFullSIMD());
storeTV(vmain(env), ptr[RefData::tvOffset()], valLoc, inst->src(1));
}
void cgInitClosureStaticLoc(IRLS& env, const IRInstruction* inst) {
auto const ref = srcLoc(env, inst, 0).reg();
auto& v = vmain(env);
always_assert(!srcLoc(env, inst, 1).isFullSIMD());
storeTV(v, ref[RefData::tvOffset()], srcLoc(env, inst, 1), inst->src(1));
}
void cgAFWHBlockOn(IRLS& env, const IRInstruction* inst) {
auto parentAR = srcLoc(env, inst, 0).reg();
auto child = srcLoc(env, inst, 1).reg();
auto& v = vmain(env);
// parent->setState(STATE_BLOCKED);
v << storebi{
AFWH::toKindState(WH::Kind::AsyncFunction, AFWH::STATE_BLOCKED),
parentAR[ar_rel(AFWH::stateOff())]
};
auto const blockableOff = AFWH::childrenOff() + AFWH::Node::blockableOff();
// parent->m_blockable.m_bits = child->m_parentChain.m_firstParent|Kind::AFWH;
static_assert(
uint8_t(AsioBlockable::Kind::AsyncFunctionWaitHandleNode) == 0,
"AFWH kind must be 0."
);
auto const firstParentOff = c_WaitableWaitHandle::parentChainOff() +
AsioBlockableChain::firstParentOff();
auto const nextParentOff = blockableOff + AsioBlockable::bitsOff();
auto const firstParent = v.makeReg();
v << load{child[firstParentOff], firstParent};
v << store{firstParent, parentAR[ar_rel(nextParentOff)]};
// child->m_parentChain.m_firstParent = &parent->m_blockable;
auto objToAR = v.makeReg();
v << lea{parentAR[ar_rel(blockableOff)], objToAR};
v << store{objToAR, child[firstParentOff]};
// parent->m_child = child;
auto const childOff = AFWH::childrenOff() + AFWH::Node::childOff();
v << store{child, parentAR[ar_rel(childOff)]};
}
void cgStOutValue(IRLS& env, const IRInstruction* inst) {
auto const fp = srcLoc(env, inst, 0).reg();
auto const off = cellsToBytes(
inst->extra<StOutValue>()->index + kNumActRecCells
);
storeTV(vmain(env), fp[off], srcLoc(env, inst, 1), inst->src(1));
}
void cgContEnter(IRLS& env, const IRInstruction* inst) {
auto const sp = srcLoc(env, inst, 0).reg();
auto const fp = srcLoc(env, inst, 1).reg();
auto const genFP = srcLoc(env, inst, 2).reg();
auto const target = srcLoc(env, inst, 3).reg();
auto const extra = inst->extra<ContEnter>();
auto const spOff = extra->spOffset;
auto const returnOff = extra->returnBCOffset;
auto& v = vmain(env);
auto const next = v.makeBlock();
v << store{fp, genFP[AROFF(m_sfp)]};
v << storeli{returnOff, genFP[AROFF(m_soff)]};
v << copy{genFP, fp};
auto const sync_sp = v.makeReg();
v << lea{sp[cellsToBytes(spOff.offset)], sync_sp};
v << syncvmsp{sync_sp};
v << contenter{fp, target, cross_trace_regs_resumed(),
{next, label(env, inst->taken())}};
env.catch_calls[inst->taken()] = CatchCall::PHP;
v = next;
}
void cgJmpSSwitchDest(IRLS& env, const IRInstruction* inst) {
auto const extra = inst->extra<JmpSSwitchDest>();
auto const marker = inst->marker();
auto& v = vmain(env);
maybe_syncsp(v, marker, srcLoc(env, inst, 1).reg(), extra->offset);
v << jmpr{srcLoc(env, inst, 0).reg(), cross_trace_args(marker)};
}
void cgMod(IRLS& env, const IRInstruction* inst) {
auto const d = dstLoc(env, inst, 0).reg();
auto const dividend = srcLoc(env, inst, 0).reg();
auto const divisor = srcLoc(env, inst, 1).reg();
auto& v = vmain(env);
v << srem{dividend, divisor, d};
}
void cgDbgAssertFunc(IRLS& env, const IRInstruction* inst) {
auto const s0 = srcLoc(env, inst, 0).reg(0);
auto const s1 = srcLoc(env, inst, 1).reg(0);
auto& v = vmain(env);
auto const sf = v.makeReg();
v << cmpq{s0, s1, sf};
ifThen(v, CC_NE, sf, [&](Vout& v) { v << ud2{}; });
}
void cgStClsRef(IRLS& env, const IRInstruction* inst) {
auto const fp = srcLoc(env, inst, 0).reg();
auto const src = srcLoc(env, inst, 1).reg();
auto const off = frame_clsref_offset(
funcFromFp(inst->src(0)),
inst->extra<ClsRefSlotData>()->slot
);
emitStLowPtr(vmain(env), src, fp[off], sizeof(LowPtr<Class>));
}
void cgSyncReturnBC(IRLS& env, const IRInstruction* inst) {
auto const extra = inst->extra<SyncReturnBC>();
auto const spOffset = cellsToBytes(extra->spOffset.offset);
auto const bcOffset = extra->bcOffset;
auto const sp = srcLoc(env, inst, 0).reg();
auto const fp = srcLoc(env, inst, 1).reg();
auto& v = vmain(env);
v << storeli{safe_cast<int32_t>(bcOffset), sp[spOffset + AROFF(m_soff)]};
v << store{fp, sp[spOffset + AROFF(m_sfp)]};
}
void cgStElem(IRLS& env, const IRInstruction* inst) {
auto const rbase = srcLoc(env, inst, 0).reg();
auto const ridx = srcLoc(env, inst, 1).reg();
auto const idx = inst->src(1);
auto& v = vmain(env);
if (idx->hasConstVal() && deltaFits(idx->intVal(), sz::dword)) {
storeTV(v, rbase[idx->intVal()], srcLoc(env, inst, 2), inst->src(2));
} else {
storeTV(v, rbase[ridx], srcLoc(env, inst, 2), inst->src(2));
}
}
void cgLdElem(IRLS& env, const IRInstruction* inst) {
auto const rbase = srcLoc(env, inst, 0).reg();
auto const ridx = srcLoc(env, inst, 1).reg();
auto const idx = inst->src(1);
auto& v = vmain(env);
if (idx->hasConstVal() && deltaFits(idx->intVal(), sz::dword)) {
loadTV(v, inst->dst(), dstLoc(env, inst, 0), rbase[idx->intVal()]);
} else {
loadTV(v, inst->dst(), dstLoc(env, inst, 0), rbase[ridx]);
}
}
void cgDbgAssertARFunc(IRLS& env, const IRInstruction* inst) {
auto const sp = srcLoc(env, inst, 0).reg();
auto const func = srcLoc(env, inst, 1).reg(0);
auto const off = cellsToBytes(inst->extra<DbgAssertARFunc>()->offset.offset);
auto& v = vmain(env);
auto const sf = v.makeReg();
v << cmpqm{func, sp[off + AROFF(m_func)], sf};
ifThen(v, CC_NE, sf, [&](Vout& v) { v << ud2{}; });
}
void cgStringIsset(IRLS& env, const IRInstruction* inst) {
auto const dst = dstLoc(env, inst, 0).reg();
auto const sd = srcLoc(env, inst, 0).reg();
auto const idx = srcLoc(env, inst, 1).reg();
auto& v = vmain(env);
auto const length = v.makeReg();
auto const sf = v.makeReg();
v << loadzlq{sd[StringData::sizeOff()], length};
v << cmpq{idx, length, sf};
v << setcc{CC_NBE, sf, dst};
}
void cgContArUpdateIdx(IRLS& env, const IRInstruction* inst) {
auto contAR = srcLoc(env, inst, 0).reg();
auto newIdx = srcLoc(env, inst, 1).reg();
auto& v = vmain(env);
auto mem_index = v.makeReg();
auto const idxOff = GENDATAOFF(m_index) - Generator::arOff();
v << load{contAR[idxOff], mem_index};
auto const sf = v.makeReg();
auto res = v.makeReg();
v << cmpq{mem_index, newIdx, sf};
v << cmovq{CC_G, sf, mem_index, newIdx, res};
v << store{res, contAR[idxOff]};
}
void cgDbgTrashFrame(IRLS& env, const IRInstruction* inst) {
auto const fp = srcLoc(env, inst, 0).reg();
auto const off = cellsToBytes(inst->extra<DbgTrashFrame>()->offset.offset);
for (auto i = 0; i < kNumActRecCells; ++i) {
trashTV(vmain(env), fp, off + cellsToBytes(i), kTVTrashJITFrame);
}
}
void cgInlineReturn(IRLS& env, const IRInstruction* inst) {
auto& v = vmain(env);
auto const fp = srcLoc(env, inst, 0).reg();
auto const callerFPOff = inst->extra<InlineReturn>()->offset;
v << lea{fp[cellsToBytes(callerFPOff.offset)], rvmfp()};
v << inlineend{};
}
void cgContArIncKey(IRLS& env, const IRInstruction* inst) {
auto const contAR = srcLoc(env, inst, 0).reg();
auto& v = vmain(env);
auto const keyOff = GENDATAOFF(m_key) - Generator::arOff();
v << incqm{contAR[keyOff + TVOFF(m_data)], v.makeReg()};
}
void cgContPreNext(IRLS& env, const IRInstruction* inst) {
auto const cont = srcLoc(env, inst, 0).reg();
auto const checkStarted = inst->src(1)->boolVal();
auto const isAsync = inst->extra<IsAsyncData>()->isAsync;
auto& v = vmain(env);
auto const sf = v.makeReg();
// These asserts make sure that the startedCheck work.
static_assert(uint8_t(BaseGenerator::State::Created) == 0, "used below");
static_assert(uint8_t(BaseGenerator::State::Started) == 1, "used below");
static_assert(uint8_t(BaseGenerator::State::Done) > 3, "");
// These asserts ensure that the state transition works. If we're in the
// Created state we want to transition to Priming, and if we're in the
// Started state we want to transition to Running. By laying out the enum
// this way we can avoid the branch and just transition by adding 2 to the
// current state.
static_assert(uint8_t(BaseGenerator::State::Priming) ==
uint8_t(BaseGenerator::State::Created) + 2, "used below");
static_assert(uint8_t(BaseGenerator::State::Running) ==
uint8_t(BaseGenerator::State::Started) + 2, "used below");
// Take exit if state != 1 (checkStarted) or if state > 1 (!checkStarted).
auto stateOff = BaseGenerator::stateOff() - genOffset(isAsync);
v << cmpbim{int8_t(BaseGenerator::State::Started), cont[stateOff], sf};
fwdJcc(v, env, checkStarted ? CC_NE : CC_A, sf, inst->taken());
// Transition the generator into either the Priming state (if we were just
// created) or the Running state (if we were started). Due to the way the
// enum is layed out, we can model this by just adding 2.
v << addlim{int8_t(2), cont[stateOff], v.makeReg()};
}
void cgLdContResumeAddr(IRLS& env, const IRInstruction* inst) {
auto const dst = dstLoc(env, inst, 0).reg();
auto const cont = srcLoc(env, inst, 0).reg();
auto const isAsync = inst->extra<IsAsyncData>()->isAsync;
auto const addrOff = BaseGenerator::resumeAddrOff() - genOffset(isAsync);
vmain(env) << load{cont[addrOff], dst};
}
void cgCoerceStk(IRLS& env, const IRInstruction *inst) {
auto const extra = inst->extra<CoerceStk>();
auto const sp = srcLoc(env, inst, 0).reg();
auto const offset = cellsToBytes(extra->offset.offset);
implCoerce(env, inst, sp, offset, extra->callee, extra->argNum);
}
void cgKillClsRef(IRLS& env, const IRInstruction* inst) {
if (!debug) return;
auto& v = vmain(env);
auto const fp = srcLoc(env, inst, 0).reg();
auto const off = frame_clsref_offset(
funcFromFp(inst->src(0)),
inst->extra<ClsRefSlotData>()->slot
);
LowPtr<Class> trash;
memset(&trash, kTrashClsRef, sizeof(trash));
Immed64 immed(trash.get());
if (sizeof(trash) == 4) {
v << storeli{immed.l(), fp[off]};
} else if (sizeof(trash) == 8) {
if (immed.fits(sz::dword)) {
v << storeqi{immed.l(), fp[off]};
} else {
v << store{v.cns(immed.q()), fp[off]};
}
} else {
not_implemented();
}
}
void cgInstanceOfIfaceVtable(IRLS& env, const IRInstruction* inst) {
auto const iface = inst->extra<InstanceOfIfaceVtable>()->cls;
auto const slot = iface->preClass()->ifaceVtableSlot();
auto const dst = dstLoc(env, inst, 0).reg();
auto const rcls = srcLoc(env, inst, 0).reg();
auto& v = vmain(env);
auto const sf = v.makeReg();
emitCmpVecLen(v, sf, static_cast<int32_t>(slot),
rcls[Class::vtableVecLenOff()]);
cond(
v, CC_A, sf, dst,
[&] (Vout& v) {
auto const vtableVec = v.makeReg();
emitLdLowPtr(v, rcls[Class::vtableVecOff()], vtableVec,
sizeof(LowPtr<Class::VtableVecSlot>));
auto const ifaceOff = slot * sizeof(Class::VtableVecSlot) +
offsetof(Class::VtableVecSlot, iface);
auto const sf = v.makeReg();
emitCmpLowPtr<Class>(v, sf, iface, vtableVec[ifaceOff]);
auto tmp = v.makeReg();
v << setcc{CC_E, sf, tmp};
return tmp;
},
[&] (Vout& v) { return v.cns(false); }
);
}
void cgFwdCtxStaticCall(IRLS& env, const IRInstruction* inst) {
auto const dstCtx = dstLoc(env, inst, 0).reg();
auto const srcCtx = srcLoc(env, inst, 0).reg();
auto const ty = inst->src(0)->type();
auto& v = vmain(env);
auto ctx_from_this = [] (Vout& v, Vreg rthis, Vreg dst) {
// Load (this->m_cls | 0x1) into `dst'.
auto const cls = emitLdObjClass(v, rthis, v.makeReg());
v << orqi{ActRec::kHasClassBit, cls, dst, v.makeReg()};
return dst;
};
if (ty <= TCctx) {
v << copy{srcCtx, dstCtx};
} else if (ty <= TObj) {
ctx_from_this(v, srcCtx, dstCtx);
} else {
// If we don't know whether we have a $this, we need to check dynamically.
auto const sf = v.makeReg();
v << testqi{ActRec::kHasClassBit, srcCtx, sf};
unlikelyCond(v, vcold(env), CC_NZ, sf, dstCtx,
[&] (Vout& v) { return srcCtx; },
[&] (Vout& v) { return ctx_from_this(v, srcCtx, v.makeReg()); }
);
}
}
void cgLookupClsMethodFCache(IRLS& env, const IRInstruction* inst) {
auto const extra = inst->extra<ClsMethodData>();
auto const dst = dstLoc(env, inst, 0).reg(0);
auto const cls = inst->src(0)->clsVal();
auto const fp = srcLoc(env, inst, 1).reg();
auto& v = vmain(env);
auto const ch = StaticMethodFCache::alloc(
cls->name(),
extra->methodName,
ctxName(inst->marker())
);
assertx(rds::isNormalHandle(ch));
const Func* (*lookup)(rds::Handle, const Class*,
const StringData*, TypedValue*) =
StaticMethodFCache::lookup;
auto const args = argGroup(env, inst)
.imm(ch)
.immPtr(cls)
.immPtr(extra->methodName)
.reg(fp);
cgCallHelper(v, env, CallSpec::direct(lookup),
callDest(dst), SyncOptions::Sync, args);
}
void cgLookupClsMethodCache(IRLS& env, const IRInstruction* inst) {
auto const extra = inst->extra<ClsMethodData>();
auto const dst = dstLoc(env, inst, 0).reg();
auto const fp = srcLoc(env, inst, 0).reg();
auto& v = vmain(env);
auto const ch = StaticMethodCache::alloc(
extra->clsName,
extra->methodName,
ctxName(inst->marker())
);
if (false) { // typecheck
UNUSED TypedValue* fake_fp = nullptr;
const UNUSED Func* f = StaticMethodCache::lookup(
ch,
extra->namedEntity,
extra->clsName,
extra->methodName,
fake_fp
);
}
auto const args = argGroup(env, inst)
.imm(ch)
.immPtr(extra->namedEntity)
.immPtr(extra->clsName)
.immPtr(extra->methodName)
.reg(fp);
// May raise an error if the class is undefined.
cgCallHelper(v, env, CallSpec::direct(StaticMethodCache::lookup),
callDest(dst), SyncOptions::Sync, args);
}
void cgCallArray(IRLS& env, const IRInstruction* inst) {
auto const extra = inst->extra<CallArray>();
auto const sp = srcLoc(env, inst, 0).reg();
auto& v = vmain(env);
auto const syncSP = v.makeReg();
v << lea{sp[cellsToBytes(extra->spOffset.offset)], syncSP};
v << syncvmsp{syncSP};
auto const target = extra->numParams == 0
? mcg->ustubs().fcallArrayHelper
: mcg->ustubs().fcallUnpackHelper;
auto const pc = v.cns(extra->pc);
auto const after = v.cns(extra->after);
auto const args = extra->numParams == 0
? v.makeTuple({pc, after})
: v.makeTuple({pc, after, v.cns(extra->numParams)});
auto const done = v.makeBlock();
v << vcallarray{target, fcall_array_regs(), args,
{done, label(env, inst->taken())}};
env.catch_calls[inst->taken()] = CatchCall::PHP;
v = done;
auto const dst = dstLoc(env, inst, 0);
v << defvmret{dst.reg(0), dst.reg(1)};
}