MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(JSGlobalData* globalData)
{
CCallHelpers jit(globalData);
// We will jump to here if the JIT code thinks it's making a call, but the
// linking helper (C++ code) decided to throw an exception instead. We will
// have saved the callReturnIndex in the first arguments of JITStackFrame.
// Note that the return address will be on the stack at this point, so we
// need to remove it and drop it on the floor, since we don't care about it.
// Finally note that the call frame register points at the callee frame, so
// we need to pop it.
jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
jit.loadPtr(
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::CallerFrame),
GPRInfo::callFrameRegister);
jit.peek(GPRInfo::nonPreservedNonReturnGPR, JITSTACKFRAME_ARGS_INDEX);
jit.setupArgumentsWithExecState(GPRInfo::nonPreservedNonReturnGPR);
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(lookupExceptionHandler)), GPRInfo::nonArgGPR0);
emitPointerValidation(jit, GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
emitPointerValidation(jit, GPRInfo::returnValueGPR2);
jit.jump(GPRInfo::returnValueGPR2);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("DFG throw exception from call slow path thunk"));
}
MacroAssemblerCodeRef baselineSetterReturnThunkGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
unsigned numberOfParameters = 0;
numberOfParameters++; // The 'this' argument.
numberOfParameters++; // The value to set.
numberOfParameters++; // The true return PC.
unsigned numberOfRegsForCall =
JSStack::CallFrameHeaderSize + numberOfParameters;
unsigned numberOfBytesForCall =
numberOfRegsForCall * sizeof(Register) - sizeof(CallerFrameAndPC);
unsigned alignedNumberOfBytesForCall =
WTF::roundUpToMultipleOf(stackAlignmentBytes(), numberOfBytesForCall);
// The real return address is stored above the arguments. We passed two arguments, so
// the argument at index 2 is the return address.
jit.loadPtr(
AssemblyHelpers::Address(
AssemblyHelpers::stackPointerRegister,
(virtualRegisterForArgument(2).offset() - JSStack::CallerFrameAndPCSize) * sizeof(Register)),
GPRInfo::regT2);
jit.addPtr(
AssemblyHelpers::TrustedImm32(alignedNumberOfBytesForCall),
AssemblyHelpers::stackPointerRegister);
jit.jump(GPRInfo::regT2);
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("baseline setter return thunk"));
}
MacroAssemblerCodeRef stringLengthTrampolineGenerator(VM* vm)
{
JSInterfaceJIT jit;
#if USE(JSVALUE64)
// Check eax is a string
JSInterfaceJIT::Jump failureCases1 = jit.emitJumpIfNotJSCell(JSInterfaceJIT::regT0);
JSInterfaceJIT::Jump failureCases2 = jit.branchPtr(
JSInterfaceJIT::NotEqual, JSInterfaceJIT::Address(
JSInterfaceJIT::regT0, JSCell::structureOffset()),
JSInterfaceJIT::TrustedImmPtr(vm->stringStructure.get()));
// Checks out okay! - get the length from the Ustring.
jit.load32(
JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSString::offsetOfLength()),
JSInterfaceJIT::regT0);
JSInterfaceJIT::Jump failureCases3 = jit.branch32(
JSInterfaceJIT::LessThan, JSInterfaceJIT::regT0, JSInterfaceJIT::TrustedImm32(0));
// regT0 contains a 64 bit value (is positive, is zero extended) so we don't need sign extend here.
jit.emitFastArithIntToImmNoCheck(JSInterfaceJIT::regT0, JSInterfaceJIT::regT0);
#else // USE(JSVALUE64)
// regT0 holds payload, regT1 holds tag
JSInterfaceJIT::Jump failureCases1 = jit.branch32(
JSInterfaceJIT::NotEqual, JSInterfaceJIT::regT1,
JSInterfaceJIT::TrustedImm32(JSValue::CellTag));
JSInterfaceJIT::Jump failureCases2 = jit.branchPtr(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSCell::structureOffset()),
JSInterfaceJIT::TrustedImmPtr(vm->stringStructure.get()));
// Checks out okay! - get the length from the Ustring.
jit.load32(
JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSString::offsetOfLength()),
JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump failureCases3 = jit.branch32(
JSInterfaceJIT::Above, JSInterfaceJIT::regT2, JSInterfaceJIT::TrustedImm32(INT_MAX));
jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::Int32Tag), JSInterfaceJIT::regT1);
#endif // USE(JSVALUE64)
jit.ret();
JSInterfaceJIT::Call failureCases1Call = jit.makeTailRecursiveCall(failureCases1);
JSInterfaceJIT::Call failureCases2Call = jit.makeTailRecursiveCall(failureCases2);
JSInterfaceJIT::Call failureCases3Call = jit.makeTailRecursiveCall(failureCases3);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
patchBuffer.link(failureCases1Call, FunctionPtr(cti_op_get_by_id_string_fail));
patchBuffer.link(failureCases2Call, FunctionPtr(cti_op_get_by_id_string_fail));
patchBuffer.link(failureCases3Call, FunctionPtr(cti_op_get_by_id_string_fail));
return FINALIZE_CODE(patchBuffer, ("string length trampoline"));
}
MacroAssemblerCodeRef osrExitGenerationThunkGenerator(VM* vm)
{
MacroAssembler jit;
// This needs to happen before we use the scratch buffer because this function also uses the scratch buffer.
adjustFrameAndStackInOSRExitCompilerThunk<DFG::JITCode>(jit, vm, JITCode::DFGJIT);
size_t scratchSize = sizeof(EncodedJSValue) * (GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters);
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(scratchSize);
EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.store64(GPRInfo::toRegister(i), buffer + i);
#else
jit.store32(GPRInfo::toRegister(i), buffer + i);
#endif
}
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.storeDouble(FPRInfo::toRegister(i), MacroAssembler::Address(GPRInfo::regT0));
}
// Tell GC mark phase how much of the scratch buffer is active during call.
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->addressOfActiveLength()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(scratchSize), MacroAssembler::Address(GPRInfo::regT0));
// Set up one argument.
#if CPU(X86)
jit.poke(GPRInfo::callFrameRegister, 0);
#else
jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
#endif
MacroAssembler::Call functionCall = jit.call();
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->addressOfActiveLength()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(0), MacroAssembler::Address(GPRInfo::regT0));
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.loadDouble(MacroAssembler::Address(GPRInfo::regT0), FPRInfo::toRegister(i));
}
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.load64(buffer + i, GPRInfo::toRegister(i));
#else
jit.load32(buffer + i, GPRInfo::toRegister(i));
#endif
}
jit.jump(MacroAssembler::AbsoluteAddress(&vm->osrExitJumpDestination));
LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID);
patchBuffer.link(functionCall, OSRExit::compileOSRExit);
return FINALIZE_CODE(patchBuffer, ("DFG OSR exit generation thunk"));
}
// For closure optimizations, we only include calls, since if you're using closures for
// object construction then you're going to lose big time anyway.
MacroAssemblerCodeRef linkClosureCallThunkGenerator(VM* vm)
{
CCallHelpers jit(vm);
slowPathFor(jit, vm, operationLinkClosureCall);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("Link closure call slow path thunk"));
}
MacroAssemblerCodeRef unreachableGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
jit.breakpoint();
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("unreachable thunk"));
}
// For closure optimizations, we only include calls, since if you're using closures for
// object construction then you're going to lose big time anyway.
MacroAssemblerCodeRef linkClosureCallThunkGenerator(JSGlobalData* globalData)
{
CCallHelpers jit(globalData);
slowPathFor(jit, globalData, operationLinkClosureCall);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("DFG link closure call slow path thunk"));
}
void compileOSRExit(ExecState* exec)
{
CodeBlock* codeBlock = exec->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->getJITType() == JITCode::DFGJIT);
JSGlobalData* globalData = &exec->globalData();
uint32_t exitIndex = globalData->osrExitIndex;
OSRExit& exit = codeBlock->osrExit(exitIndex);
// Make sure all code on our inline stack is JIT compiled. This is necessary since
// we may opt to inline a code block even before it had ever been compiled by the
// JIT, but our OSR exit infrastructure currently only works if the target of the
// OSR exit is JIT code. This could be changed since there is nothing particularly
// hard about doing an OSR exit into the interpreter, but for now this seems to make
// sense in that if we're OSR exiting from inlined code of a DFG code block, then
// probably it's a good sign that the thing we're exiting into is hot. Even more
// interestingly, since the code was inlined, it may never otherwise get JIT
// compiled since the act of inlining it may ensure that it otherwise never runs.
for (CodeOrigin codeOrigin = exit.m_codeOrigin; codeOrigin.inlineCallFrame; codeOrigin = codeOrigin.inlineCallFrame->caller) {
static_cast<FunctionExecutable*>(codeOrigin.inlineCallFrame->executable.get())
->baselineCodeBlockFor(codeOrigin.inlineCallFrame->isCall ? CodeForCall : CodeForConstruct)
->jitCompile(*globalData);
}
SpeculationRecovery* recovery = 0;
if (exit.m_recoveryIndex)
recovery = &codeBlock->speculationRecovery(exit.m_recoveryIndex - 1);
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLog("Generating OSR exit #%u (bc#%u, @%u, %s) for code block %p.\n", exitIndex, exit.m_codeOrigin.bytecodeIndex, exit.m_nodeIndex, exitKindToString(exit.m_kind), codeBlock);
#endif
{
AssemblyHelpers jit(globalData, codeBlock);
OSRExitCompiler exitCompiler(jit);
jit.jitAssertHasValidCallFrame();
exitCompiler.compileExit(exit, recovery);
LinkBuffer patchBuffer(*globalData, &jit, codeBlock);
exit.m_code = patchBuffer.finalizeCode();
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLog("OSR exit code at [%p, %p).\n", patchBuffer.debugAddress(), static_cast<char*>(patchBuffer.debugAddress()) + patchBuffer.debugSize());
#endif
}
{
RepatchBuffer repatchBuffer(codeBlock);
repatchBuffer.relink(exit.m_check.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
}
globalData->osrExitJumpDestination = exit.m_code.code().executableAddress();
}
MacroAssemblerCodeRef osrExitGenerationThunkGenerator(JSGlobalData* globalData)
{
MacroAssembler jit;
size_t scratchSize = sizeof(EncodedJSValue) * (GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters);
ScratchBuffer* scratchBuffer = globalData->scratchBufferForSize(scratchSize);
EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.store64(GPRInfo::toRegister(i), buffer + i);
#else
jit.store32(GPRInfo::toRegister(i), buffer + i);
#endif
}
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.storeDouble(FPRInfo::toRegister(i), GPRInfo::regT0);
}
// Tell GC mark phase how much of the scratch buffer is active during call.
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->activeLengthPtr()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(scratchSize), GPRInfo::regT0);
// Set up one argument.
#if CPU(X86)
jit.poke(GPRInfo::callFrameRegister, 0);
#else
jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
#endif
MacroAssembler::Call functionCall = jit.call();
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->activeLengthPtr()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(0), GPRInfo::regT0);
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.loadDouble(GPRInfo::regT0, FPRInfo::toRegister(i));
}
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.load64(buffer + i, GPRInfo::toRegister(i));
#else
jit.load32(buffer + i, GPRInfo::toRegister(i));
#endif
}
jit.jump(MacroAssembler::AbsoluteAddress(&globalData->osrExitJumpDestination));
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
patchBuffer.link(functionCall, compileOSRExit);
return FINALIZE_CODE(patchBuffer, ("DFG OSR exit generation thunk"));
}
static MacroAssemblerCodeRef linkClosureCallForThunkGenerator(
VM* vm, RegisterPreservationMode registers)
{
CCallHelpers jit(vm);
slowPathFor(jit, vm, operationLinkClosureCallFor(registers));
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("Link closure call %s slow path thunk", registers == MustPreserveRegisters ? " that preserves registers" : ""));
}
static MacroAssemblerCodeRef generateThunkWithJumpTo(VM* vm, void (*target)(), const char *thunkKind)
{
JSInterfaceJIT jit(vm);
// FIXME: there's probably a better way to do it on X86, but I'm not sure I care.
jit.move(JSInterfaceJIT::TrustedImmPtr(bitwise_cast<void*>(target)), JSInterfaceJIT::regT0);
jit.jump(JSInterfaceJIT::regT0);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("LLInt %s prologue thunk", thunkKind));
}
MacroAssemblerCodeRef linkCallThunkGenerator(VM* vm)
{
// The return address is on the stack or in the link register. We will hence
// save the return address to the call frame while we make a C++ function call
// to perform linking and lazy compilation if necessary. We expect the callee
// to be in regT0/regT1 (payload/tag), the CallFrame to have already
// been adjusted, and all other registers to be available for use.
CCallHelpers jit(vm);
slowPathFor(jit, vm, operationLinkCall);
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("Link call slow path thunk"));
}
// We will jump here if the JIT code tries to make a call, but the
// linking helper (C++ code) decides to throw an exception instead.
MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(VM* vm)
{
CCallHelpers jit(vm);
// The call pushed a return address, so we need to pop it back off to re-align the stack,
// even though we won't use it.
jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
jit.setupArguments(CCallHelpers::TrustedImmPtr(vm), GPRInfo::callFrameRegister);
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(lookupExceptionHandler)), GPRInfo::nonArgGPR0);
emitPointerValidation(jit, GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
jit.jumpToExceptionHandler();
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("Throw exception from call slow path thunk"));
}
void JIT::privateCompileClosureCall(CallLinkInfo* callLinkInfo, CodeBlock* calleeCodeBlock, Structure* expectedStructure, ExecutableBase* expectedExecutable, MacroAssemblerCodePtr codePtr)
{
JumpList slowCases;
slowCases.append(branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag)));
slowCases.append(branchPtr(NotEqual, Address(regT0, JSCell::structureOffset()), TrustedImmPtr(expectedStructure)));
slowCases.append(branchPtr(NotEqual, Address(regT0, JSFunction::offsetOfExecutable()), TrustedImmPtr(expectedExecutable)));
loadPtr(Address(regT0, JSFunction::offsetOfScopeChain()), regT1);
emitPutCellToCallFrameHeader(regT1, JSStack::ScopeChain);
Call call = nearCall();
Jump done = jump();
slowCases.link(this);
move(TrustedImmPtr(callLinkInfo->callReturnLocation.executableAddress()), regT2);
restoreReturnAddressBeforeReturn(regT2);
Jump slow = jump();
LinkBuffer patchBuffer(*m_vm, this, m_codeBlock);
patchBuffer.link(call, FunctionPtr(codePtr.executableAddress()));
patchBuffer.link(done, callLinkInfo->hotPathOther.labelAtOffset(0));
patchBuffer.link(slow, CodeLocationLabel(m_vm->getCTIStub(virtualCallThunkGenerator).code()));
RefPtr<ClosureCallStubRoutine> stubRoutine = adoptRef(new ClosureCallStubRoutine(
FINALIZE_CODE(
patchBuffer,
("Baseline closure call stub for %s, return point %p, target %p (%s)",
toCString(*m_codeBlock).data(),
callLinkInfo->hotPathOther.labelAtOffset(0).executableAddress(),
codePtr.executableAddress(),
toCString(pointerDump(calleeCodeBlock)).data())),
*m_vm, m_codeBlock->ownerExecutable(), expectedStructure, expectedExecutable,
callLinkInfo->codeOrigin));
RepatchBuffer repatchBuffer(m_codeBlock);
repatchBuffer.replaceWithJump(
RepatchBuffer::startOfBranchPtrWithPatchOnRegister(callLinkInfo->hotPathBegin),
CodeLocationLabel(stubRoutine->code().code()));
repatchBuffer.relink(callLinkInfo->callReturnLocation, m_vm->getCTIStub(virtualCallThunkGenerator).code());
callLinkInfo->stub = stubRoutine.release();
}
static MacroAssemblerCodeRef linkForThunkGenerator(
VM* vm, CodeSpecializationKind kind, RegisterPreservationMode registers)
{
// The return address is on the stack or in the link register. We will hence
// save the return address to the call frame while we make a C++ function call
// to perform linking and lazy compilation if necessary. We expect the callee
// to be in regT0/regT1 (payload/tag), the CallFrame to have already
// been adjusted, and all other registers to be available for use.
CCallHelpers jit(vm);
slowPathFor(jit, vm, operationLinkFor(kind, registers));
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("Link %s%s slow path thunk", kind == CodeForCall ? "call" : "construct", registers == MustPreserveRegisters ? " that preserves registers" : ""));
}
static MacroAssemblerCodeRef virtualForGenerator(VM* vm, FunctionPtr compile, FunctionPtr notJSFunction, const char* name, CodeSpecializationKind kind)
{
JSInterfaceJIT jit;
JSInterfaceJIT::JumpList slowCase;
#if USE(JSVALUE64)
slowCase.append(jit.emitJumpIfNotJSCell(JSInterfaceJIT::regT0));
#else // USE(JSVALUE64)
slowCase.append(jit.branch32(JSInterfaceJIT::NotEqual, JSInterfaceJIT::regT1, JSInterfaceJIT::TrustedImm32(JSValue::CellTag)));
#endif // USE(JSVALUE64)
slowCase.append(jit.emitJumpIfNotType(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1, JSFunctionType));
// Finish canonical initialization before JS function call.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSFunction::offsetOfScopeChain()), JSInterfaceJIT::regT1);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump hasCodeBlock1 = jit.branch32(JSInterfaceJIT::GreaterThanOrEqual, JSInterfaceJIT::Address(JSInterfaceJIT::regT2, FunctionExecutable::offsetOfNumParametersFor(kind)), JSInterfaceJIT::TrustedImm32(0));
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
jit.restoreArgumentReference();
JSInterfaceJIT::Call callCompile = jit.call();
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
hasCodeBlock1.link(&jit);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, FunctionExecutable::offsetOfJITCodeWithArityCheckFor(kind)), JSInterfaceJIT::regT0);
#if !ASSERT_DISABLED
JSInterfaceJIT::Jump ok = jit.branchTestPtr(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT0);
jit.breakpoint();
ok.link(&jit);
#endif
jit.jump(JSInterfaceJIT::regT0);
slowCase.link(&jit);
JSInterfaceJIT::Call callNotJSFunction = generateSlowCaseFor(vm, jit);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
patchBuffer.link(callCompile, compile);
patchBuffer.link(callNotJSFunction, notJSFunction);
return FINALIZE_CODE(patchBuffer, ("virtual %s trampoline", name));
}
MacroAssemblerCodeRef osrEntryThunkGenerator(VM* vm)
{
AssemblyHelpers jit(nullptr);
// We get passed the address of a scratch buffer. The first 8-byte slot of the buffer
// is the frame size. The second 8-byte slot is the pointer to where we are supposed to
// jump. The remaining bytes are the new call frame header followed by the locals.
ptrdiff_t offsetOfFrameSize = 0; // This is the DFG frame count.
ptrdiff_t offsetOfTargetPC = offsetOfFrameSize + sizeof(EncodedJSValue);
ptrdiff_t offsetOfPayload = offsetOfTargetPC + sizeof(EncodedJSValue);
ptrdiff_t offsetOfLocals = offsetOfPayload + sizeof(Register) * CallFrame::headerSizeInRegisters;
jit.move(GPRInfo::returnValueGPR2, GPRInfo::regT0);
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfFrameSize), GPRInfo::regT1); // Load the frame size.
jit.move(GPRInfo::regT1, GPRInfo::regT2);
jit.lshiftPtr(MacroAssembler::Imm32(3), GPRInfo::regT2);
jit.move(GPRInfo::callFrameRegister, MacroAssembler::stackPointerRegister);
jit.subPtr(GPRInfo::regT2, MacroAssembler::stackPointerRegister);
MacroAssembler::Label loop = jit.label();
jit.subPtr(MacroAssembler::TrustedImm32(1), GPRInfo::regT1);
jit.move(GPRInfo::regT1, GPRInfo::regT4);
jit.negPtr(GPRInfo::regT4);
jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals), GPRInfo::regT2);
jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals + sizeof(int32_t)), GPRInfo::regT3);
jit.store32(GPRInfo::regT2, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register))));
jit.store32(GPRInfo::regT3, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register)) + static_cast<intptr_t>(sizeof(int32_t))));
jit.branchPtr(MacroAssembler::NotEqual, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(-static_cast<intptr_t>(CallFrame::headerSizeInRegisters)))).linkTo(loop, &jit);
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfTargetPC), GPRInfo::regT1);
MacroAssembler::Jump ok = jit.branchPtr(MacroAssembler::Above, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(static_cast<intptr_t>(1000))));
jit.abortWithReason(DFGUnreasonableOSREntryJumpDestination);
ok.link(&jit);
jit.restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(*vm);
jit.emitMaterializeTagCheckRegisters();
jit.jump(GPRInfo::regT1);
LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("DFG OSR entry thunk"));
}
static MacroAssemblerCodeRef linkForThunkGenerator(
JSGlobalData* globalData, CodeSpecializationKind kind)
{
// The return address is on the stack or in the link register. We will hence
// save the return address to the call frame while we make a C++ function call
// to perform linking and lazy compilation if necessary. We expect the callee
// to be in nonArgGPR0/nonArgGPR1 (payload/tag), the call frame to have already
// been adjusted, nonPreservedNonReturnGPR holds the exception handler index,
// and all other registers to be available for use. We use JITStackFrame::args
// to save important information across calls.
CCallHelpers jit(globalData);
slowPathFor(jit, globalData, kind == CodeForCall ? operationLinkCall : operationLinkConstruct);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("DFG link %s slow path thunk", kind == CodeForCall ? "call" : "construct"));
}
static MacroAssemblerCodeRef linkForGenerator(VM* vm, FunctionPtr lazyLink, FunctionPtr notJSFunction, const char* name)
{
JSInterfaceJIT jit;
JSInterfaceJIT::JumpList slowCase;
#if USE(JSVALUE64)
slowCase.append(jit.emitJumpIfNotJSCell(JSInterfaceJIT::regT0));
slowCase.append(jit.emitJumpIfNotType(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1, JSFunctionType));
#else // USE(JSVALUE64)
slowCase.append(jit.branch32(JSInterfaceJIT::NotEqual, JSInterfaceJIT::regT1, JSInterfaceJIT::TrustedImm32(JSValue::CellTag)));
slowCase.append(jit.emitJumpIfNotType(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1, JSFunctionType));
#endif // USE(JSVALUE64)
// Finish canonical initialization before JS function call.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSFunction::offsetOfScopeChain()), JSInterfaceJIT::regT1);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
// Also initialize ReturnPC for use by lazy linking and exceptions.
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3);
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
jit.restoreArgumentReference();
JSInterfaceJIT::Call callLazyLink = jit.call();
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
jit.jump(JSInterfaceJIT::regT0);
slowCase.link(&jit);
JSInterfaceJIT::Call callNotJSFunction = generateSlowCaseFor(vm, jit);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
patchBuffer.link(callLazyLink, lazyLink);
patchBuffer.link(callNotJSFunction, notJSFunction);
return FINALIZE_CODE(patchBuffer, ("link %s trampoline", name));
}
MacroAssemblerCodeRef arityFixup(VM* vm)
{
JSInterfaceJIT jit;
// We enter with fixup count in regT0
#if USE(JSVALUE64)
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.addPtr(JSInterfaceJIT::TrustedImm32(-8), JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT1);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register to account for missing args
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT0);
jit.addPtr(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister);
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.addPtr(JSInterfaceJIT::TrustedImm32(-8), JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, 4), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register to account for missing args
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT0);
jit.addPtr(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister);
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
static MacroAssemblerCodeRef virtualForThunkGenerator(
VM* vm, CodeSpecializationKind kind, RegisterPreservationMode registers)
{
// The callee is in regT0 (for JSVALUE32_64, the tag is in regT1).
// The return address is on the stack, or in the link register. We will hence
// jump to the callee, or save the return address to the call frame while we
// make a C++ function call to the appropriate JIT operation.
CCallHelpers jit(vm);
CCallHelpers::JumpList slowCase;
// FIXME: we should have a story for eliminating these checks. In many cases,
// the DFG knows that the value is definitely a cell, or definitely a function.
#if USE(JSVALUE64)
jit.move(CCallHelpers::TrustedImm64(TagMask), GPRInfo::regT2);
slowCase.append(
jit.branchTest64(
CCallHelpers::NonZero, GPRInfo::regT0, GPRInfo::regT2));
#else
slowCase.append(
jit.branch32(
CCallHelpers::NotEqual, GPRInfo::regT1,
CCallHelpers::TrustedImm32(JSValue::CellTag)));
#endif
AssemblyHelpers::emitLoadStructure(jit, GPRInfo::regT0, GPRInfo::regT2, GPRInfo::regT1);
slowCase.append(
jit.branchPtr(
CCallHelpers::NotEqual,
CCallHelpers::Address(GPRInfo::regT2, Structure::classInfoOffset()),
CCallHelpers::TrustedImmPtr(JSFunction::info())));
// Now we know we have a JSFunction.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfExecutable()),
GPRInfo::regT2);
jit.loadPtr(
CCallHelpers::Address(
GPRInfo::regT2, ExecutableBase::offsetOfJITCodeWithArityCheckFor(kind, registers)),
GPRInfo::regT2);
slowCase.append(jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT2));
// Now we know that we have a CodeBlock, and we're committed to making a fast
// call.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfScopeChain()),
GPRInfo::regT1);
#if USE(JSVALUE64)
jit.emitPutToCallFrameHeaderBeforePrologue(GPRInfo::regT1, JSStack::ScopeChain);
#else
jit.emitPutPayloadToCallFrameHeaderBeforePrologue(GPRInfo::regT1, JSStack::ScopeChain);
jit.emitPutTagToCallFrameHeaderBeforePrologue(CCallHelpers::TrustedImm32(JSValue::CellTag),
JSStack::ScopeChain);
#endif
// Make a tail call. This will return back to JIT code.
emitPointerValidation(jit, GPRInfo::regT2);
jit.jump(GPRInfo::regT2);
slowCase.link(&jit);
// Here we don't know anything, so revert to the full slow path.
slowPathFor(jit, vm, operationVirtualFor(kind, registers));
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("Virtual %s%s slow path thunk", kind == CodeForCall ? "call" : "construct", registers == MustPreserveRegisters ? " that preserves registers" : ""));
}
void compileOSRExit(ExecState* exec)
{
SamplingRegion samplingRegion("DFG OSR Exit Compilation");
CodeBlock* codeBlock = exec->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->jitType() == JITCode::DFGJIT);
VM* vm = &exec->vm();
// It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
// really be profitable.
DeferGCForAWhile deferGC(vm->heap);
uint32_t exitIndex = vm->osrExitIndex;
OSRExit& exit = codeBlock->jitCode()->dfg()->osrExit[exitIndex];
prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
// Compute the value recoveries.
Operands<ValueRecovery> operands;
codeBlock->jitCode()->dfg()->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->jitCode()->dfg()->minifiedDFG, exit.m_streamIndex, operands);
// There may be an override, for forward speculations.
if (!!exit.m_valueRecoveryOverride) {
operands.setOperand(
exit.m_valueRecoveryOverride->operand, exit.m_valueRecoveryOverride->recovery);
}
SpeculationRecovery* recovery = 0;
if (exit.m_recoveryIndex != UINT_MAX)
recovery = &codeBlock->jitCode()->dfg()->speculationRecovery[exit.m_recoveryIndex];
{
CCallHelpers jit(vm, codeBlock);
OSRExitCompiler exitCompiler(jit);
jit.jitAssertHasValidCallFrame();
if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
Profiler::Database& database = *vm->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
exitCompiler.compileExit(exit, operands, recovery);
LinkBuffer patchBuffer(*vm, jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldShowDisassembly() || Options::verboseOSR(),
patchBuffer,
("DFG OSR exit #%u (%s, %s) from %s, with operands = %s",
exitIndex, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(operands)).data()));
}
{
RepatchBuffer repatchBuffer(codeBlock);
repatchBuffer.relink(exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
}
vm->osrExitJumpDestination = exit.m_code.code().executableAddress();
}
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind)
{
int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
JSInterfaceJIT jit;
jit.emitPutImmediateToCallFrameHeader(0, JSStack::CodeBlock);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.peek(JSInterfaceJIT::regT1);
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ReturnPC);
// Calling convention: f(ecx, edx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister); // Align stack after call.
// call the function
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister);
#elif CPU(X86_64)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.peek(JSInterfaceJIT::regT1);
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ReturnPC);
#if !OS(WINDOWS)
// Calling convention: f(edi, esi, edx, ecx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); // Align stack after call.
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#else
// Calling convention: f(ecx, edx, r8, r9, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
// Leave space for the callee parameter home addresses and align the stack.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
#elif CPU(ARM)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT2);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC);
// Calling convention: f(r0 == regT0, r1 == regT1, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, ARMRegisters::r0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARMRegisters::r1);
jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.loadPtr(JSInterfaceJIT::Address(ARMRegisters::r1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
#elif CPU(SH4)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT2);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC);
// Calling convention: f(r0 == regT4, r1 == regT5, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT4);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT5);
jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT5, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction), JSInterfaceJIT::regT0);
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
#elif CPU(MIPS)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC);
// Calling convention: f(a0, a1, a2, a3);
// Host function signature: f(ExecState*);
// Allocate stack space for 16 bytes (8-byte aligned)
// 16 bytes (unused) for 4 arguments
jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
// Setup arg0
jit.move(JSInterfaceJIT::callFrameRegister, MIPSRegisters::a0);
// Call
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, MIPSRegisters::a2);
jit.loadPtr(JSInterfaceJIT::Address(MIPSRegisters::a2, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
// Restore stack space
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
#else
#error "JIT not supported on this platform."
UNUSED_PARAM(executableOffsetToFunction);
breakpoint();
#endif
// Check for an exception
#if USE(JSVALUE64)
jit.load64(&(vm->exception), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
#else
JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::AbsoluteAddress(reinterpret_cast<char*>(&vm->exception) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
JSInterfaceJIT::TrustedImm32(JSValue::EmptyValueTag));
#endif
// Return.
jit.ret();
// Handle an exception
exceptionHandler.link(&jit);
// Grab the return address.
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT1);
jit.move(JSInterfaceJIT::TrustedImmPtr(&vm->exceptionLocation), JSInterfaceJIT::regT2);
jit.storePtr(JSInterfaceJIT::regT1, JSInterfaceJIT::regT2);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(ctiVMThrowTrampolineSlowpath).value()), JSInterfaceJIT::regT1);
jit.jump(JSInterfaceJIT::regT1);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("native %s trampoline", toCString(kind).data()));
}
MacroAssemblerCodeRef arityFixupGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
// We enter with fixup count in argumentGPR0
// We have the guarantee that a0, a1, a2, t3, t4 and t5 (or t0 for Windows) are all distinct :-)
#if USE(JSVALUE64)
#if OS(WINDOWS)
const GPRReg extraTemp = JSInterfaceJIT::regT0;
#else
const GPRReg extraTemp = JSInterfaceJIT::regT5;
#endif
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
// Check to see if we have extra slots we can use
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
JSInterfaceJIT::Label fillExtraSlots(jit.label());
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight));
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
noExtraSlot.link(&jit);
jit.neg64(JSInterfaceJIT::argumentGPR0);
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT3, extraTemp);
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
// Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::argumentGPR0, extraTemp);
jit.lshift64(JSInterfaceJIT::TrustedImm32(3), extraTemp);
jit.addPtr(extraTemp, JSInterfaceJIT::callFrameRegister);
jit.addPtr(extraTemp, JSInterfaceJIT::stackPointerRegister);
done.link(&jit);
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
// Check to see if we have extra slots we can use
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Label fillExtraSlots(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, TagOffset));
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
noExtraSlot.link(&jit);
jit.neg32(JSInterfaceJIT::argumentGPR0);
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, PayloadOffset), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, TagOffset), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
// Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::regT5);
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT5);
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::stackPointerRegister);
done.link(&jit);
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall)
{
int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
JSInterfaceJIT jit(vm);
if (entryType == EnterViaCall)
jit.emitFunctionPrologue();
#if USE(JSVALUE64)
else if (entryType == EnterViaJump) {
// We're coming from a specialized thunk that has saved the prior tag registers' contents.
// Restore them now.
#if CPU(ARM64)
jit.popPair(JSInterfaceJIT::tagTypeNumberRegister, JSInterfaceJIT::tagMaskRegister);
#else
jit.pop(JSInterfaceJIT::tagMaskRegister);
jit.pop(JSInterfaceJIT::tagTypeNumberRegister);
#endif
}
#endif
jit.emitPutToCallFrameHeader(0, JSStack::CodeBlock);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86)
// Calling convention: f(ecx, edx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue.
// call the function
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister);
#elif CPU(X86_64)
#if !OS(WINDOWS)
// Calling convention: f(edi, esi, edx, ecx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
#else
// Calling convention: f(ecx, edx, r8, r9, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
// Leave space for the callee parameter home addresses.
// At this point the stack is aligned to 16 bytes, but if this changes at some point, we need to emit code to align it.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
#elif CPU(ARM64)
COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0);
COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1);
COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1);
jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2);
jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction));
#elif CPU(ARM) || CPU(SH4) || CPU(MIPS)
#if CPU(MIPS)
// Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments.
jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
// Calling convention is f(argumentGPR0, argumentGPR1, ...).
// Host function signature is f(ExecState*).
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
#if CPU(MIPS)
// Restore stack space
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
#else
#error "JIT not supported on this platform."
UNUSED_PARAM(executableOffsetToFunction);
abortWithReason(TGNotSupported);
#endif
// Check for an exception
#if USE(JSVALUE64)
jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
#else
JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::AbsoluteAddress(vm->addressOfException()),
JSInterfaceJIT::TrustedImm32(0));
#endif
jit.emitFunctionEpilogue();
// Return.
jit.ret();
// Handle an exception
exceptionHandler.link(&jit);
jit.copyCalleeSavesToVMCalleeSavesBuffer();
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT0);
jit.push(JSInterfaceJIT::regT0);
#else
#if OS(WINDOWS)
// Allocate space on stack for the 4 parameter registers.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
#endif
jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3);
jit.call(JSInterfaceJIT::regT3);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#elif OS(WINDOWS)
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
jit.jumpToExceptionHandler();
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJump ? "Tail " : "", toCString(kind).data()));
}
MacroAssemblerCodeRef arityFixup(VM* vm)
{
JSInterfaceJIT jit(vm);
// We enter with fixup count, in aligned stack units, in regT0 and the return thunk in
// regT5 on 32-bit and regT7 on 64-bit.
#if USE(JSVALUE64)
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.lshift32(JSInterfaceJIT::TrustedImm32(logStackAlignmentRegisters()), JSInterfaceJIT::regT0);
jit.neg64(JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT6);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame down regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT6, JSInterfaceJIT::regT1);
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT6, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT6);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 - 1 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT1);
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT6, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT6);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1);
jit.lshift64(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT1);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::stackPointerRegister);
// Save the original return PC.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()), GPRInfo::regT1);
jit.storePtr(GPRInfo::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT6, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
// Install the new return PC.
jit.storePtr(GPRInfo::regT7, JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()));
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.lshift32(JSInterfaceJIT::TrustedImm32(logStackAlignmentRegisters()), JSInterfaceJIT::regT0);
jit.neg32(JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame down regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, 4), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 - 1 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1);
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT1);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::stackPointerRegister);
// Save the original return PC.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()), GPRInfo::regT1);
jit.storePtr(GPRInfo::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
// Install the new return PC.
jit.storePtr(GPRInfo::regT5, JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()));
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
static MacroAssemblerCodeRef virtualForThunkGenerator(
JSGlobalData* globalData, CodeSpecializationKind kind)
{
// The return address is on the stack, or in the link register. We will hence
// jump to the callee, or save the return address to the call frame while we
// make a C++ function call to the appropriate DFG operation.
CCallHelpers jit(globalData);
CCallHelpers::JumpList slowCase;
// FIXME: we should have a story for eliminating these checks. In many cases,
// the DFG knows that the value is definitely a cell, or definitely a function.
#if USE(JSVALUE64)
slowCase.append(
jit.branchTestPtr(
CCallHelpers::NonZero, GPRInfo::nonArgGPR0, GPRInfo::tagMaskRegister));
#else
slowCase.append(
jit.branch32(
CCallHelpers::NotEqual, GPRInfo::nonArgGPR1,
CCallHelpers::TrustedImm32(JSValue::CellTag)));
#endif
jit.loadPtr(CCallHelpers::Address(GPRInfo::nonArgGPR0, JSCell::structureOffset()), GPRInfo::nonArgGPR2);
slowCase.append(
jit.branchPtr(
CCallHelpers::NotEqual,
CCallHelpers::Address(GPRInfo::nonArgGPR2, Structure::classInfoOffset()),
CCallHelpers::TrustedImmPtr(&JSFunction::s_info)));
// Now we know we have a JSFunction.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::nonArgGPR0, JSFunction::offsetOfExecutable()),
GPRInfo::nonArgGPR2);
slowCase.append(
jit.branch32(
CCallHelpers::LessThan,
CCallHelpers::Address(
GPRInfo::nonArgGPR2, ExecutableBase::offsetOfNumParametersFor(kind)),
CCallHelpers::TrustedImm32(0)));
// Now we know that we have a CodeBlock, and we're committed to making a fast
// call.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::nonArgGPR0, JSFunction::offsetOfScopeChain()),
GPRInfo::nonArgGPR1);
#if USE(JSVALUE64)
jit.storePtr(
GPRInfo::nonArgGPR1,
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::ScopeChain));
#else
jit.storePtr(
GPRInfo::nonArgGPR1,
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::ScopeChain +
OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
jit.store32(
CCallHelpers::TrustedImm32(JSValue::CellTag),
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::ScopeChain +
OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
#endif
jit.loadPtr(
CCallHelpers::Address(GPRInfo::nonArgGPR2, ExecutableBase::offsetOfJITCodeWithArityCheckFor(kind)),
GPRInfo::regT0);
// Make a tail call. This will return back to DFG code.
emitPointerValidation(jit, GPRInfo::regT0);
jit.jump(GPRInfo::regT0);
slowCase.link(&jit);
// Here we don't know anything, so revert to the full slow path.
slowPathFor(jit, globalData, kind == CodeForCall ? operationVirtualCall : operationVirtualConstruct);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("DFG virtual %s slow path thunk", kind == CodeForCall ? "call" : "construct"));
}
// FIXME: We should distinguish between a megamorphic virtual call vs. a slow
// path virtual call so that we can enable fast tail calls for megamorphic
// virtual calls by using the shuffler.
// https://bugs.webkit.org/show_bug.cgi?id=148831
MacroAssemblerCodeRef virtualThunkFor(VM* vm, CallLinkInfo& callLinkInfo)
{
// The callee is in regT0 (for JSVALUE32_64, the tag is in regT1).
// The return address is on the stack, or in the link register. We will hence
// jump to the callee, or save the return address to the call frame while we
// make a C++ function call to the appropriate JIT operation.
CCallHelpers jit(vm);
CCallHelpers::JumpList slowCase;
// This is a slow path execution, and regT2 contains the CallLinkInfo. Count the
// slow path execution for the profiler.
jit.add32(
CCallHelpers::TrustedImm32(1),
CCallHelpers::Address(GPRInfo::regT2, CallLinkInfo::offsetOfSlowPathCount()));
// FIXME: we should have a story for eliminating these checks. In many cases,
// the DFG knows that the value is definitely a cell, or definitely a function.
#if USE(JSVALUE64)
jit.move(CCallHelpers::TrustedImm64(TagMask), GPRInfo::regT4);
slowCase.append(
jit.branchTest64(
CCallHelpers::NonZero, GPRInfo::regT0, GPRInfo::regT4));
#else
slowCase.append(
jit.branch32(
CCallHelpers::NotEqual, GPRInfo::regT1,
CCallHelpers::TrustedImm32(JSValue::CellTag)));
#endif
AssemblyHelpers::emitLoadStructure(jit, GPRInfo::regT0, GPRInfo::regT4, GPRInfo::regT1);
slowCase.append(
jit.branchPtr(
CCallHelpers::NotEqual,
CCallHelpers::Address(GPRInfo::regT4, Structure::classInfoOffset()),
CCallHelpers::TrustedImmPtr(JSFunction::info())));
// Now we know we have a JSFunction.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfExecutable()),
GPRInfo::regT4);
jit.loadPtr(
CCallHelpers::Address(
GPRInfo::regT4, ExecutableBase::offsetOfJITCodeWithArityCheckFor(
callLinkInfo.specializationKind())),
GPRInfo::regT4);
slowCase.append(jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT4));
// Now we know that we have a CodeBlock, and we're committed to making a fast
// call.
// Make a tail call. This will return back to JIT code.
emitPointerValidation(jit, GPRInfo::regT4);
if (callLinkInfo.isTailCall()) {
jit.preserveReturnAddressAfterCall(GPRInfo::regT0);
jit.prepareForTailCallSlow(GPRInfo::regT4);
}
jit.jump(GPRInfo::regT4);
slowCase.link(&jit);
// Here we don't know anything, so revert to the full slow path.
slowPathFor(jit, vm, operationVirtualCall);
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("Virtual %s slow path thunk",
callLinkInfo.callMode() == CallMode::Regular ? "call" : callLinkInfo.callMode() == CallMode::Tail ? "tail call" : "construct"));
}
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall)
{
int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
JSInterfaceJIT jit(vm);
if (entryType == EnterViaCall)
jit.emitFunctionPrologue();
jit.emitPutImmediateToCallFrameHeader(0, JSStack::CodeBlock);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
// Calling convention: f(ecx, edx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue.
// call the function
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister);
#elif CPU(X86_64)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
#if !OS(WINDOWS)
// Calling convention: f(edi, esi, edx, ecx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
#else
// Calling convention: f(ecx, edx, r8, r9, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
// Leave space for the callee parameter home addresses and align the stack.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
#elif CPU(ARM64)
COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT1, prev_callframe_not_trampled_by_T1);
COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT3, prev_callframe_not_trampled_by_T3);
COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0);
COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1);
COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2);
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(ARM64Registers::x3);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, ARM64Registers::x3);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1);
jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2);
jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction));
#elif CPU(ARM) || CPU(SH4) || CPU(MIPS)
// Load caller frame's scope chain into this callframe so that whatever we call can get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT2);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
#if CPU(MIPS)
// Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments.
jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
// Calling convention is f(argumentGPR0, argumentGPR1, ...).
// Host function signature is f(ExecState*).
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
#if CPU(MIPS)
// Restore stack space
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
#else
#error "JIT not supported on this platform."
UNUSED_PARAM(executableOffsetToFunction);
breakpoint();
#endif
// Check for an exception
#if USE(JSVALUE64)
jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
#else
JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::AbsoluteAddress(reinterpret_cast<char*>(vm->addressOfException()) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
JSInterfaceJIT::TrustedImm32(JSValue::EmptyValueTag));
#endif
jit.emitFunctionEpilogue();
// Return.
jit.ret();
// Handle an exception
exceptionHandler.link(&jit);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister), JSInterfaceJIT::regT0);
jit.push(JSInterfaceJIT::regT0);
#else
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister), JSInterfaceJIT::argumentGPR0);
#endif
jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3);
jit.call(JSInterfaceJIT::regT3);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
jit.jumpToExceptionHandler();
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJump ? "Tail " : "", toCString(kind).data()));
}
void compileOSRExit(ExecState* exec)
{
SamplingRegion samplingRegion("DFG OSR Exit Compilation");
CodeBlock* codeBlock = exec->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->getJITType() == JITCode::DFGJIT);
JSGlobalData* globalData = &exec->globalData();
uint32_t exitIndex = globalData->osrExitIndex;
OSRExit& exit = codeBlock->osrExit(exitIndex);
// Make sure all code on our inline stack is JIT compiled. This is necessary since
// we may opt to inline a code block even before it had ever been compiled by the
// JIT, but our OSR exit infrastructure currently only works if the target of the
// OSR exit is JIT code. This could be changed since there is nothing particularly
// hard about doing an OSR exit into the interpreter, but for now this seems to make
// sense in that if we're OSR exiting from inlined code of a DFG code block, then
// probably it's a good sign that the thing we're exiting into is hot. Even more
// interestingly, since the code was inlined, it may never otherwise get JIT
// compiled since the act of inlining it may ensure that it otherwise never runs.
for (CodeOrigin codeOrigin = exit.m_codeOrigin; codeOrigin.inlineCallFrame; codeOrigin = codeOrigin.inlineCallFrame->caller) {
static_cast<FunctionExecutable*>(codeOrigin.inlineCallFrame->executable.get())
->baselineCodeBlockFor(codeOrigin.inlineCallFrame->isCall ? CodeForCall : CodeForConstruct)
->jitCompile(exec);
}
// Compute the value recoveries.
Operands<ValueRecovery> operands;
codeBlock->variableEventStream().reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->minifiedDFG(), exit.m_streamIndex, operands);
// There may be an override, for forward speculations.
if (!!exit.m_valueRecoveryOverride) {
operands.setOperand(
exit.m_valueRecoveryOverride->operand, exit.m_valueRecoveryOverride->recovery);
}
SpeculationRecovery* recovery = 0;
if (exit.m_recoveryIndex)
recovery = &codeBlock->speculationRecovery(exit.m_recoveryIndex - 1);
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLog(
"Generating OSR exit #", exitIndex, " (seq#", exit.m_streamIndex,
", bc#", exit.m_codeOrigin.bytecodeIndex, ", @", exit.m_nodeIndex, ", ",
exit.m_kind, ") for ", *codeBlock, ".\n");
#endif
{
CCallHelpers jit(globalData, codeBlock);
OSRExitCompiler exitCompiler(jit);
jit.jitAssertHasValidCallFrame();
if (globalData->m_perBytecodeProfiler && codeBlock->compilation()) {
Profiler::Database& database = *globalData->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->compilation();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind,
exit.m_watchpointIndex != std::numeric_limits<unsigned>::max());
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
exitCompiler.compileExit(exit, operands, recovery);
LinkBuffer patchBuffer(*globalData, &jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldShowDisassembly(),
patchBuffer,
("DFG OSR exit #%u (bc#%u, @%u, %s) from %s",
exitIndex, exit.m_codeOrigin.bytecodeIndex, exit.m_nodeIndex,
exitKindToString(exit.m_kind), toCString(*codeBlock).data()));
}
{
RepatchBuffer repatchBuffer(codeBlock);
repatchBuffer.relink(exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
}
globalData->osrExitJumpDestination = exit.m_code.code().executableAddress();
}
