void JSCallBase::emit(CCallHelpers& jit, State& /*state*/, int32_t osrExitFromGenericUnwindStackSpillSlot)
{
RELEASE_ASSERT(!!m_callSiteIndex);
#if FTL_USES_B3
UNUSED_PARAM(osrExitFromGenericUnwindStackSpillSlot);
#else // FTL_USES_B3
if (m_correspondingGenericUnwindOSRExit)
m_correspondingGenericUnwindOSRExit->spillRegistersToSpillSlot(jit, osrExitFromGenericUnwindStackSpillSlot);
#endif // FTL_USES_B3
jit.store32(CCallHelpers::TrustedImm32(m_callSiteIndex.bits()), CCallHelpers::tagFor(static_cast<VirtualRegister>(JSStack::ArgumentCount)));
m_callLinkInfo = jit.codeBlock()->addCallLinkInfo();
if (CallLinkInfo::callModeFor(m_type) == CallMode::Tail)
jit.emitRestoreCalleeSaves();
CCallHelpers::Jump slowPath = jit.branchPtrWithPatch(
CCallHelpers::NotEqual, GPRInfo::regT0, m_targetToCheck,
CCallHelpers::TrustedImmPtr(0));
CCallHelpers::Jump done;
if (CallLinkInfo::callModeFor(m_type) == CallMode::Tail) {
jit.prepareForTailCallSlow();
m_fastCall = jit.nearTailCall();
} else {
m_fastCall = jit.nearCall();
done = jit.jump();
}
slowPath.link(&jit);
jit.move(CCallHelpers::TrustedImmPtr(m_callLinkInfo), GPRInfo::regT2);
m_slowCall = jit.nearCall();
if (CallLinkInfo::callModeFor(m_type) == CallMode::Tail)
jit.abortWithReason(JITDidReturnFromTailCall);
else
done.link(&jit);
m_callLinkInfo->setUpCall(m_type, m_semanticeOrigin, GPRInfo::regT0);
}
void JSCallVarargs::emit(CCallHelpers& jit, State& state, int32_t spillSlotsOffset, int32_t osrExitFromGenericUnwindSpillSlots)
{
// We are passed three pieces of information:
// - The callee.
// - The arguments object, if it's not a forwarding call.
// - The "this" value, if it's a constructor call.
CallVarargsData* data = m_node->callVarargsData();
GPRReg calleeGPR = GPRInfo::argumentGPR0;
GPRReg argumentsGPR = InvalidGPRReg;
GPRReg thisGPR = InvalidGPRReg;
bool forwarding = false;
switch (m_node->op()) {
case CallVarargs:
case TailCallVarargs:
case TailCallVarargsInlinedCaller:
case ConstructVarargs:
argumentsGPR = GPRInfo::argumentGPR1;
thisGPR = GPRInfo::argumentGPR2;
break;
case CallForwardVarargs:
case TailCallForwardVarargs:
case TailCallForwardVarargsInlinedCaller:
case ConstructForwardVarargs:
thisGPR = GPRInfo::argumentGPR1;
forwarding = true;
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
const unsigned calleeSpillSlot = 0;
const unsigned argumentsSpillSlot = 1;
const unsigned thisSpillSlot = 2;
const unsigned stackPointerSpillSlot = 3;
// Get some scratch registers.
RegisterSet usedRegisters;
usedRegisters.merge(RegisterSet::stackRegisters());
usedRegisters.merge(RegisterSet::reservedHardwareRegisters());
usedRegisters.merge(RegisterSet::calleeSaveRegisters());
usedRegisters.set(calleeGPR);
if (argumentsGPR != InvalidGPRReg)
usedRegisters.set(argumentsGPR);
ASSERT(thisGPR);
usedRegisters.set(thisGPR);
ScratchRegisterAllocator allocator(usedRegisters);
GPRReg scratchGPR1 = allocator.allocateScratchGPR();
GPRReg scratchGPR2 = allocator.allocateScratchGPR();
GPRReg scratchGPR3 = allocator.allocateScratchGPR();
RELEASE_ASSERT(!allocator.numberOfReusedRegisters());
auto computeUsedStack = [&] (GPRReg targetGPR, unsigned extra) {
if (isARM64()) {
// Have to do this the weird way because $sp on ARM64 means zero when used in a subtraction.
jit.move(CCallHelpers::stackPointerRegister, targetGPR);
jit.negPtr(targetGPR);
jit.addPtr(GPRInfo::callFrameRegister, targetGPR);
} else {
jit.move(GPRInfo::callFrameRegister, targetGPR);
jit.subPtr(CCallHelpers::stackPointerRegister, targetGPR);
}
if (extra)
jit.subPtr(CCallHelpers::TrustedImm32(extra), targetGPR);
jit.urshiftPtr(CCallHelpers::Imm32(3), targetGPR);
};
auto callWithExceptionCheck = [&] (void* callee) {
jit.move(CCallHelpers::TrustedImmPtr(callee), GPRInfo::nonPreservedNonArgumentGPR);
jit.call(GPRInfo::nonPreservedNonArgumentGPR);
m_exceptions.append(jit.emitExceptionCheck(AssemblyHelpers::NormalExceptionCheck, AssemblyHelpers::FarJumpWidth));
};
if (isARM64()) {
jit.move(CCallHelpers::stackPointerRegister, scratchGPR1);
jit.storePtr(scratchGPR1, CCallHelpers::addressFor(spillSlotsOffset + stackPointerSpillSlot));
} else
jit.storePtr(CCallHelpers::stackPointerRegister, CCallHelpers::addressFor(spillSlotsOffset + stackPointerSpillSlot));
unsigned extraStack = sizeof(CallerFrameAndPC) +
WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(void*));
if (forwarding) {
CCallHelpers::JumpList slowCase;
computeUsedStack(scratchGPR2, 0);
emitSetupVarargsFrameFastCase(jit, scratchGPR2, scratchGPR1, scratchGPR2, scratchGPR3, m_node->child2()->origin.semantic.inlineCallFrame, data->firstVarArgOffset, slowCase);
CCallHelpers::Jump done = jit.jump();
slowCase.link(&jit);
jit.subPtr(CCallHelpers::TrustedImm32(extraStack), CCallHelpers::stackPointerRegister);
jit.setupArgumentsExecState();
callWithExceptionCheck(bitwise_cast<void*>(operationThrowStackOverflowForVarargs));
jit.abortWithReason(DFGVarargsThrowingPathDidNotThrow);
done.link(&jit);
jit.move(calleeGPR, GPRInfo::regT0);
} else {
// Gotta spill the callee, arguments, and this because we will need them later and we will have some
// calls that clobber them.
jit.store64(calleeGPR, CCallHelpers::addressFor(spillSlotsOffset + calleeSpillSlot));
jit.store64(argumentsGPR, CCallHelpers::addressFor(spillSlotsOffset + argumentsSpillSlot));
jit.store64(thisGPR, CCallHelpers::addressFor(spillSlotsOffset + thisSpillSlot));
computeUsedStack(scratchGPR1, 0);
jit.subPtr(CCallHelpers::TrustedImm32(extraStack), CCallHelpers::stackPointerRegister);
jit.setupArgumentsWithExecState(argumentsGPR, scratchGPR1, CCallHelpers::TrustedImm32(data->firstVarArgOffset));
callWithExceptionCheck(bitwise_cast<void*>(operationSizeFrameForVarargs));
jit.move(GPRInfo::returnValueGPR, scratchGPR1);
computeUsedStack(scratchGPR2, extraStack);
jit.load64(CCallHelpers::addressFor(spillSlotsOffset + argumentsSpillSlot), argumentsGPR);
emitSetVarargsFrame(jit, scratchGPR1, false, scratchGPR2, scratchGPR2);
jit.addPtr(CCallHelpers::TrustedImm32(-extraStack), scratchGPR2, CCallHelpers::stackPointerRegister);
jit.setupArgumentsWithExecState(scratchGPR2, argumentsGPR, CCallHelpers::TrustedImm32(data->firstVarArgOffset), scratchGPR1);
callWithExceptionCheck(bitwise_cast<void*>(operationSetupVarargsFrame));
jit.move(GPRInfo::returnValueGPR, scratchGPR2);
jit.load64(CCallHelpers::addressFor(spillSlotsOffset + thisSpillSlot), thisGPR);
jit.load64(CCallHelpers::addressFor(spillSlotsOffset + calleeSpillSlot), GPRInfo::regT0);
}
jit.addPtr(CCallHelpers::TrustedImm32(sizeof(CallerFrameAndPC)), scratchGPR2, CCallHelpers::stackPointerRegister);
jit.store64(thisGPR, CCallHelpers::calleeArgumentSlot(0));
// Henceforth we make the call. The base FTL call machinery expects the callee in regT0 and for the
// stack frame to already be set up, which it is.
jit.store64(GPRInfo::regT0, CCallHelpers::calleeFrameSlot(JSStack::Callee));
m_callBase.emit(jit, state, osrExitFromGenericUnwindSpillSlots);
// Undo the damage we've done.
if (isARM64()) {
GPRReg scratchGPRAtReturn = CCallHelpers::selectScratchGPR(GPRInfo::returnValueGPR);
jit.loadPtr(CCallHelpers::addressFor(spillSlotsOffset + stackPointerSpillSlot), scratchGPRAtReturn);
jit.move(scratchGPRAtReturn, CCallHelpers::stackPointerRegister);
} else
jit.loadPtr(CCallHelpers::addressFor(spillSlotsOffset + stackPointerSpillSlot), CCallHelpers::stackPointerRegister);
}
void JSTailCall::emit(JITCode& jitCode, CCallHelpers& jit)
{
StackMaps::Record* record { nullptr };
for (unsigned i = jitCode.stackmaps.records.size(); i--;) {
record = &jitCode.stackmaps.records[i];
if (record->patchpointID == m_stackmapID)
break;
}
RELEASE_ASSERT(record->patchpointID == m_stackmapID);
m_callLinkInfo = jit.codeBlock()->addCallLinkInfo();
CallFrameShuffleData shuffleData;
// The callee was the first passed argument, and must be in a GPR because
// we used the "anyregcc" calling convention
auto calleeLocation =
FTL::Location::forStackmaps(nullptr, record->locations[0]);
GPRReg calleeGPR = calleeLocation.directGPR();
shuffleData.callee = ValueRecovery::inGPR(calleeGPR, DataFormatJS);
// The tag type number was the second argument, if there was one
auto tagTypeNumberLocation =
FTL::Location::forStackmaps(&jitCode.stackmaps, record->locations[1]);
if (tagTypeNumberLocation.isGPR() && !tagTypeNumberLocation.addend())
shuffleData.tagTypeNumber = tagTypeNumberLocation.directGPR();
shuffleData.args.grow(numArguments());
HashMap<Reg, Vector<std::pair<ValueRecovery*, int32_t>>> withAddend;
size_t numAddends { 0 };
for (size_t i = 0; i < numArguments(); ++i) {
shuffleData.args[i] = recoveryFor(m_arguments[i], *record, jitCode.stackmaps);
if (FTL::Location addend = getRegisterWithAddend(m_arguments[i], *record, jitCode.stackmaps)) {
withAddend.add(
addend.reg(),
Vector<std::pair<ValueRecovery*, int32_t>>()).iterator->value.append(
std::make_pair(&shuffleData.args[i], addend.addend()));
numAddends++;
}
}
numAddends = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), numAddends);
shuffleData.numLocals = static_cast<int64_t>(jitCode.stackmaps.stackSizeForLocals()) / sizeof(void*) + numAddends;
ASSERT(!numAddends == withAddend.isEmpty());
if (!withAddend.isEmpty()) {
jit.subPtr(MacroAssembler::TrustedImm32(numAddends * sizeof(void*)), MacroAssembler::stackPointerRegister);
VirtualRegister spillBase { 1 - static_cast<int>(shuffleData.numLocals) };
for (auto entry : withAddend) {
for (auto pair : entry.value) {
ASSERT(numAddends > 0);
VirtualRegister spillSlot { spillBase + --numAddends };
ASSERT(entry.key.isGPR());
jit.addPtr(MacroAssembler::TrustedImm32(pair.second), entry.key.gpr());
jit.storePtr(entry.key.gpr(), CCallHelpers::addressFor(spillSlot));
jit.subPtr(MacroAssembler::TrustedImm32(pair.second), entry.key.gpr());
*pair.first = ValueRecovery::displacedInJSStack(spillSlot, pair.first->dataFormat());
}
}
ASSERT(numAddends < stackAlignmentRegisters());
}
shuffleData.args.resize(numArguments());
for (size_t i = 0; i < numArguments(); ++i)
shuffleData.args[i] = recoveryFor(m_arguments[i], *record, jitCode.stackmaps);
shuffleData.setupCalleeSaveRegisters(jit.codeBlock());
CCallHelpers::Jump slowPath = jit.branchPtrWithPatch(
CCallHelpers::NotEqual, calleeGPR, m_targetToCheck,
CCallHelpers::TrustedImmPtr(0));
m_callLinkInfo->setFrameShuffleData(shuffleData);
CallFrameShuffler(jit, shuffleData).prepareForTailCall();
m_fastCall = jit.nearTailCall();
slowPath.link(&jit);
CallFrameShuffler slowPathShuffler(jit, shuffleData);
slowPathShuffler.setCalleeJSValueRegs(JSValueRegs { GPRInfo::regT0 });
slowPathShuffler.prepareForSlowPath();
jit.move(CCallHelpers::TrustedImmPtr(m_callLinkInfo), GPRInfo::regT2);
m_slowCall = jit.nearCall();
jit.abortWithReason(JITDidReturnFromTailCall);
m_callLinkInfo->setUpCall(m_type, m_semanticeOrigin, calleeGPR);
}
