MacroAssemblerCodeRef registerRestorationThunkGenerator(VM* vm)
{
AssemblyHelpers jit(vm, 0);
generateRegisterRestoration(jit);
LinkBuffer linkBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(linkBuffer, ("Register restoration thunk"));
}
MacroAssemblerCodeRef baselineSetterReturnThunkGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
unsigned numberOfParameters = 0;
numberOfParameters++; // The 'this' argument.
numberOfParameters++; // The value to set.
numberOfParameters++; // The true return PC.
unsigned numberOfRegsForCall =
JSStack::CallFrameHeaderSize + numberOfParameters;
unsigned numberOfBytesForCall =
numberOfRegsForCall * sizeof(Register) - sizeof(CallerFrameAndPC);
unsigned alignedNumberOfBytesForCall =
WTF::roundUpToMultipleOf(stackAlignmentBytes(), numberOfBytesForCall);
// The real return address is stored above the arguments. We passed two arguments, so
// the argument at index 2 is the return address.
jit.loadPtr(
AssemblyHelpers::Address(
AssemblyHelpers::stackPointerRegister,
(virtualRegisterForArgument(2).offset() - JSStack::CallerFrameAndPCSize) * sizeof(Register)),
GPRInfo::regT2);
jit.addPtr(
AssemblyHelpers::TrustedImm32(alignedNumberOfBytesForCall),
AssemblyHelpers::stackPointerRegister);
jit.jump(GPRInfo::regT2);
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("baseline setter return thunk"));
}
MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(JSGlobalData* globalData)
{
CCallHelpers jit(globalData);
// We will jump to here if the JIT code thinks it's making a call, but the
// linking helper (C++ code) decided to throw an exception instead. We will
// have saved the callReturnIndex in the first arguments of JITStackFrame.
// Note that the return address will be on the stack at this point, so we
// need to remove it and drop it on the floor, since we don't care about it.
// Finally note that the call frame register points at the callee frame, so
// we need to pop it.
jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
jit.loadPtr(
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::CallerFrame),
GPRInfo::callFrameRegister);
jit.peek(GPRInfo::nonPreservedNonReturnGPR, JITSTACKFRAME_ARGS_INDEX);
jit.setupArgumentsWithExecState(GPRInfo::nonPreservedNonReturnGPR);
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(lookupExceptionHandler)), GPRInfo::nonArgGPR0);
emitPointerValidation(jit, GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
emitPointerValidation(jit, GPRInfo::returnValueGPR2);
jit.jump(GPRInfo::returnValueGPR2);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("DFG throw exception from call slow path thunk"));
}
MacroAssemblerCodeRef stringLengthTrampolineGenerator(VM* vm)
{
JSInterfaceJIT jit;
#if USE(JSVALUE64)
// Check eax is a string
JSInterfaceJIT::Jump failureCases1 = jit.emitJumpIfNotJSCell(JSInterfaceJIT::regT0);
JSInterfaceJIT::Jump failureCases2 = jit.branchPtr(
JSInterfaceJIT::NotEqual, JSInterfaceJIT::Address(
JSInterfaceJIT::regT0, JSCell::structureOffset()),
JSInterfaceJIT::TrustedImmPtr(vm->stringStructure.get()));
// Checks out okay! - get the length from the Ustring.
jit.load32(
JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSString::offsetOfLength()),
JSInterfaceJIT::regT0);
JSInterfaceJIT::Jump failureCases3 = jit.branch32(
JSInterfaceJIT::LessThan, JSInterfaceJIT::regT0, JSInterfaceJIT::TrustedImm32(0));
// regT0 contains a 64 bit value (is positive, is zero extended) so we don't need sign extend here.
jit.emitFastArithIntToImmNoCheck(JSInterfaceJIT::regT0, JSInterfaceJIT::regT0);
#else // USE(JSVALUE64)
// regT0 holds payload, regT1 holds tag
JSInterfaceJIT::Jump failureCases1 = jit.branch32(
JSInterfaceJIT::NotEqual, JSInterfaceJIT::regT1,
JSInterfaceJIT::TrustedImm32(JSValue::CellTag));
JSInterfaceJIT::Jump failureCases2 = jit.branchPtr(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSCell::structureOffset()),
JSInterfaceJIT::TrustedImmPtr(vm->stringStructure.get()));
// Checks out okay! - get the length from the Ustring.
jit.load32(
JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSString::offsetOfLength()),
JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump failureCases3 = jit.branch32(
JSInterfaceJIT::Above, JSInterfaceJIT::regT2, JSInterfaceJIT::TrustedImm32(INT_MAX));
jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::Int32Tag), JSInterfaceJIT::regT1);
#endif // USE(JSVALUE64)
jit.ret();
JSInterfaceJIT::Call failureCases1Call = jit.makeTailRecursiveCall(failureCases1);
JSInterfaceJIT::Call failureCases2Call = jit.makeTailRecursiveCall(failureCases2);
JSInterfaceJIT::Call failureCases3Call = jit.makeTailRecursiveCall(failureCases3);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
patchBuffer.link(failureCases1Call, FunctionPtr(cti_op_get_by_id_string_fail));
patchBuffer.link(failureCases2Call, FunctionPtr(cti_op_get_by_id_string_fail));
patchBuffer.link(failureCases3Call, FunctionPtr(cti_op_get_by_id_string_fail));
return FINALIZE_CODE(patchBuffer, ("string length trampoline"));
}
MacroAssemblerCodeRef osrExitGenerationThunkGenerator(VM* vm)
{
MacroAssembler jit;
// This needs to happen before we use the scratch buffer because this function also uses the scratch buffer.
adjustFrameAndStackInOSRExitCompilerThunk<DFG::JITCode>(jit, vm, JITCode::DFGJIT);
size_t scratchSize = sizeof(EncodedJSValue) * (GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters);
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(scratchSize);
EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.store64(GPRInfo::toRegister(i), buffer + i);
#else
jit.store32(GPRInfo::toRegister(i), buffer + i);
#endif
}
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.storeDouble(FPRInfo::toRegister(i), MacroAssembler::Address(GPRInfo::regT0));
}
// Tell GC mark phase how much of the scratch buffer is active during call.
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->addressOfActiveLength()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(scratchSize), MacroAssembler::Address(GPRInfo::regT0));
// Set up one argument.
#if CPU(X86)
jit.poke(GPRInfo::callFrameRegister, 0);
#else
jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
#endif
MacroAssembler::Call functionCall = jit.call();
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->addressOfActiveLength()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(0), MacroAssembler::Address(GPRInfo::regT0));
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.loadDouble(MacroAssembler::Address(GPRInfo::regT0), FPRInfo::toRegister(i));
}
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.load64(buffer + i, GPRInfo::toRegister(i));
#else
jit.load32(buffer + i, GPRInfo::toRegister(i));
#endif
}
jit.jump(MacroAssembler::AbsoluteAddress(&vm->osrExitJumpDestination));
LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID);
patchBuffer.link(functionCall, OSRExit::compileOSRExit);
return FINALIZE_CODE(patchBuffer, ("DFG OSR exit generation thunk"));
}
// For closure optimizations, we only include calls, since if you're using closures for
// object construction then you're going to lose big time anyway.
MacroAssemblerCodeRef linkClosureCallThunkGenerator(VM* vm)
{
CCallHelpers jit(vm);
slowPathFor(jit, vm, operationLinkClosureCall);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("Link closure call slow path thunk"));
}
// For closure optimizations, we only include calls, since if you're using closures for
// object construction then you're going to lose big time anyway.
MacroAssemblerCodeRef linkClosureCallThunkGenerator(JSGlobalData* globalData)
{
CCallHelpers jit(globalData);
slowPathFor(jit, globalData, operationLinkClosureCall);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("DFG link closure call slow path thunk"));
}
MacroAssemblerCodeRef unreachableGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
jit.breakpoint();
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("unreachable thunk"));
}
Compilation::Compilation(VM& vm, Procedure& proc, unsigned optLevel)
{
CCallHelpers jit(&vm);
generate(proc, jit, optLevel);
LinkBuffer linkBuffer(vm, jit, nullptr);
m_codeRef = FINALIZE_CODE(linkBuffer, ("B3::Compilation"));
m_byproducts = proc.takeByproducts();
}
MacroAssemblerCodeRef osrExitGenerationThunkGenerator(JSGlobalData* globalData)
{
MacroAssembler jit;
size_t scratchSize = sizeof(EncodedJSValue) * (GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters);
ScratchBuffer* scratchBuffer = globalData->scratchBufferForSize(scratchSize);
EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.store64(GPRInfo::toRegister(i), buffer + i);
#else
jit.store32(GPRInfo::toRegister(i), buffer + i);
#endif
}
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.storeDouble(FPRInfo::toRegister(i), GPRInfo::regT0);
}
// Tell GC mark phase how much of the scratch buffer is active during call.
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->activeLengthPtr()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(scratchSize), GPRInfo::regT0);
// Set up one argument.
#if CPU(X86)
jit.poke(GPRInfo::callFrameRegister, 0);
#else
jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
#endif
MacroAssembler::Call functionCall = jit.call();
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->activeLengthPtr()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(0), GPRInfo::regT0);
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.loadDouble(GPRInfo::regT0, FPRInfo::toRegister(i));
}
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.load64(buffer + i, GPRInfo::toRegister(i));
#else
jit.load32(buffer + i, GPRInfo::toRegister(i));
#endif
}
jit.jump(MacroAssembler::AbsoluteAddress(&globalData->osrExitJumpDestination));
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
patchBuffer.link(functionCall, compileOSRExit);
return FINALIZE_CODE(patchBuffer, ("DFG OSR exit generation thunk"));
}
static MacroAssemblerCodeRef linkClosureCallForThunkGenerator(
VM* vm, RegisterPreservationMode registers)
{
CCallHelpers jit(vm);
slowPathFor(jit, vm, operationLinkClosureCallFor(registers));
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("Link closure call %s slow path thunk", registers == MustPreserveRegisters ? " that preserves registers" : ""));
}
static MacroAssemblerCodeRef generateThunkWithJumpTo(VM* vm, void (*target)(), const char *thunkKind)
{
JSInterfaceJIT jit(vm);
// FIXME: there's probably a better way to do it on X86, but I'm not sure I care.
jit.move(JSInterfaceJIT::TrustedImmPtr(bitwise_cast<void*>(target)), JSInterfaceJIT::regT0);
jit.jump(JSInterfaceJIT::regT0);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("LLInt %s prologue thunk", thunkKind));
}
Compilation compile(VM& vm, Procedure& proc, unsigned optLevel)
{
TimingScope timingScope("Compilation");
prepareForGeneration(proc, optLevel);
CCallHelpers jit(&vm);
generate(proc, jit);
LinkBuffer linkBuffer(vm, jit, nullptr);
return Compilation(FINALIZE_CODE(linkBuffer, ("B3::Compilation")), proc.releaseByproducts());
}
MacroAssemblerCodeRef linkCallThunkGenerator(VM* vm)
{
// The return address is on the stack or in the link register. We will hence
// save the return address to the call frame while we make a C++ function call
// to perform linking and lazy compilation if necessary. We expect the callee
// to be in regT0/regT1 (payload/tag), the CallFrame to have already
// been adjusted, and all other registers to be available for use.
CCallHelpers jit(vm);
slowPathFor(jit, vm, operationLinkCall);
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("Link call slow path thunk"));
}
// We will jump here if the JIT code tries to make a call, but the
// linking helper (C++ code) decides to throw an exception instead.
MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(VM* vm)
{
CCallHelpers jit(vm);
// The call pushed a return address, so we need to pop it back off to re-align the stack,
// even though we won't use it.
jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
jit.setupArguments(CCallHelpers::TrustedImmPtr(vm), GPRInfo::callFrameRegister);
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(lookupExceptionHandler)), GPRInfo::nonArgGPR0);
emitPointerValidation(jit, GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
jit.jumpToExceptionHandler();
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("Throw exception from call slow path thunk"));
}
void JIT::privateCompileClosureCall(CallLinkInfo* callLinkInfo, CodeBlock* calleeCodeBlock, Structure* expectedStructure, ExecutableBase* expectedExecutable, MacroAssemblerCodePtr codePtr)
{
JumpList slowCases;
slowCases.append(branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag)));
slowCases.append(branchPtr(NotEqual, Address(regT0, JSCell::structureOffset()), TrustedImmPtr(expectedStructure)));
slowCases.append(branchPtr(NotEqual, Address(regT0, JSFunction::offsetOfExecutable()), TrustedImmPtr(expectedExecutable)));
loadPtr(Address(regT0, JSFunction::offsetOfScopeChain()), regT1);
emitPutCellToCallFrameHeader(regT1, JSStack::ScopeChain);
Call call = nearCall();
Jump done = jump();
slowCases.link(this);
move(TrustedImmPtr(callLinkInfo->callReturnLocation.executableAddress()), regT2);
restoreReturnAddressBeforeReturn(regT2);
Jump slow = jump();
LinkBuffer patchBuffer(*m_vm, this, m_codeBlock);
patchBuffer.link(call, FunctionPtr(codePtr.executableAddress()));
patchBuffer.link(done, callLinkInfo->hotPathOther.labelAtOffset(0));
patchBuffer.link(slow, CodeLocationLabel(m_vm->getCTIStub(virtualCallThunkGenerator).code()));
RefPtr<ClosureCallStubRoutine> stubRoutine = adoptRef(new ClosureCallStubRoutine(
FINALIZE_CODE(
patchBuffer,
("Baseline closure call stub for %s, return point %p, target %p (%s)",
toCString(*m_codeBlock).data(),
callLinkInfo->hotPathOther.labelAtOffset(0).executableAddress(),
codePtr.executableAddress(),
toCString(pointerDump(calleeCodeBlock)).data())),
*m_vm, m_codeBlock->ownerExecutable(), expectedStructure, expectedExecutable,
callLinkInfo->codeOrigin));
RepatchBuffer repatchBuffer(m_codeBlock);
repatchBuffer.replaceWithJump(
RepatchBuffer::startOfBranchPtrWithPatchOnRegister(callLinkInfo->hotPathBegin),
CodeLocationLabel(stubRoutine->code().code()));
repatchBuffer.relink(callLinkInfo->callReturnLocation, m_vm->getCTIStub(virtualCallThunkGenerator).code());
callLinkInfo->stub = stubRoutine.release();
}
static MacroAssemblerCodeRef linkForThunkGenerator(
VM* vm, CodeSpecializationKind kind, RegisterPreservationMode registers)
{
// The return address is on the stack or in the link register. We will hence
// save the return address to the call frame while we make a C++ function call
// to perform linking and lazy compilation if necessary. We expect the callee
// to be in regT0/regT1 (payload/tag), the CallFrame to have already
// been adjusted, and all other registers to be available for use.
CCallHelpers jit(vm);
slowPathFor(jit, vm, operationLinkFor(kind, registers));
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("Link %s%s slow path thunk", kind == CodeForCall ? "call" : "construct", registers == MustPreserveRegisters ? " that preserves registers" : ""));
}
static MacroAssemblerCodeRef virtualForGenerator(VM* vm, FunctionPtr compile, FunctionPtr notJSFunction, const char* name, CodeSpecializationKind kind)
{
JSInterfaceJIT jit;
JSInterfaceJIT::JumpList slowCase;
#if USE(JSVALUE64)
slowCase.append(jit.emitJumpIfNotJSCell(JSInterfaceJIT::regT0));
#else // USE(JSVALUE64)
slowCase.append(jit.branch32(JSInterfaceJIT::NotEqual, JSInterfaceJIT::regT1, JSInterfaceJIT::TrustedImm32(JSValue::CellTag)));
#endif // USE(JSVALUE64)
slowCase.append(jit.emitJumpIfNotType(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1, JSFunctionType));
// Finish canonical initialization before JS function call.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSFunction::offsetOfScopeChain()), JSInterfaceJIT::regT1);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump hasCodeBlock1 = jit.branch32(JSInterfaceJIT::GreaterThanOrEqual, JSInterfaceJIT::Address(JSInterfaceJIT::regT2, FunctionExecutable::offsetOfNumParametersFor(kind)), JSInterfaceJIT::TrustedImm32(0));
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
jit.restoreArgumentReference();
JSInterfaceJIT::Call callCompile = jit.call();
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
hasCodeBlock1.link(&jit);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, FunctionExecutable::offsetOfJITCodeWithArityCheckFor(kind)), JSInterfaceJIT::regT0);
#if !ASSERT_DISABLED
JSInterfaceJIT::Jump ok = jit.branchTestPtr(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT0);
jit.breakpoint();
ok.link(&jit);
#endif
jit.jump(JSInterfaceJIT::regT0);
slowCase.link(&jit);
JSInterfaceJIT::Call callNotJSFunction = generateSlowCaseFor(vm, jit);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
patchBuffer.link(callCompile, compile);
patchBuffer.link(callNotJSFunction, notJSFunction);
return FINALIZE_CODE(patchBuffer, ("virtual %s trampoline", name));
}
MacroAssemblerCodeRef osrEntryThunkGenerator(VM* vm)
{
AssemblyHelpers jit(nullptr);
// We get passed the address of a scratch buffer. The first 8-byte slot of the buffer
// is the frame size. The second 8-byte slot is the pointer to where we are supposed to
// jump. The remaining bytes are the new call frame header followed by the locals.
ptrdiff_t offsetOfFrameSize = 0; // This is the DFG frame count.
ptrdiff_t offsetOfTargetPC = offsetOfFrameSize + sizeof(EncodedJSValue);
ptrdiff_t offsetOfPayload = offsetOfTargetPC + sizeof(EncodedJSValue);
ptrdiff_t offsetOfLocals = offsetOfPayload + sizeof(Register) * CallFrame::headerSizeInRegisters;
jit.move(GPRInfo::returnValueGPR2, GPRInfo::regT0);
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfFrameSize), GPRInfo::regT1); // Load the frame size.
jit.move(GPRInfo::regT1, GPRInfo::regT2);
jit.lshiftPtr(MacroAssembler::Imm32(3), GPRInfo::regT2);
jit.move(GPRInfo::callFrameRegister, MacroAssembler::stackPointerRegister);
jit.subPtr(GPRInfo::regT2, MacroAssembler::stackPointerRegister);
MacroAssembler::Label loop = jit.label();
jit.subPtr(MacroAssembler::TrustedImm32(1), GPRInfo::regT1);
jit.move(GPRInfo::regT1, GPRInfo::regT4);
jit.negPtr(GPRInfo::regT4);
jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals), GPRInfo::regT2);
jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals + sizeof(int32_t)), GPRInfo::regT3);
jit.store32(GPRInfo::regT2, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register))));
jit.store32(GPRInfo::regT3, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register)) + static_cast<intptr_t>(sizeof(int32_t))));
jit.branchPtr(MacroAssembler::NotEqual, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(-static_cast<intptr_t>(CallFrame::headerSizeInRegisters)))).linkTo(loop, &jit);
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfTargetPC), GPRInfo::regT1);
MacroAssembler::Jump ok = jit.branchPtr(MacroAssembler::Above, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(static_cast<intptr_t>(1000))));
jit.abortWithReason(DFGUnreasonableOSREntryJumpDestination);
ok.link(&jit);
jit.restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(*vm);
jit.emitMaterializeTagCheckRegisters();
jit.jump(GPRInfo::regT1);
LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("DFG OSR entry thunk"));
}
static MacroAssemblerCodeRef linkForThunkGenerator(
JSGlobalData* globalData, CodeSpecializationKind kind)
{
// The return address is on the stack or in the link register. We will hence
// save the return address to the call frame while we make a C++ function call
// to perform linking and lazy compilation if necessary. We expect the callee
// to be in nonArgGPR0/nonArgGPR1 (payload/tag), the call frame to have already
// been adjusted, nonPreservedNonReturnGPR holds the exception handler index,
// and all other registers to be available for use. We use JITStackFrame::args
// to save important information across calls.
CCallHelpers jit(globalData);
slowPathFor(jit, globalData, kind == CodeForCall ? operationLinkCall : operationLinkConstruct);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("DFG link %s slow path thunk", kind == CodeForCall ? "call" : "construct"));
}
static MacroAssemblerCodeRef linkForGenerator(VM* vm, FunctionPtr lazyLink, FunctionPtr notJSFunction, const char* name)
{
JSInterfaceJIT jit;
JSInterfaceJIT::JumpList slowCase;
#if USE(JSVALUE64)
slowCase.append(jit.emitJumpIfNotJSCell(JSInterfaceJIT::regT0));
slowCase.append(jit.emitJumpIfNotType(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1, JSFunctionType));
#else // USE(JSVALUE64)
slowCase.append(jit.branch32(JSInterfaceJIT::NotEqual, JSInterfaceJIT::regT1, JSInterfaceJIT::TrustedImm32(JSValue::CellTag)));
slowCase.append(jit.emitJumpIfNotType(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1, JSFunctionType));
#endif // USE(JSVALUE64)
// Finish canonical initialization before JS function call.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSFunction::offsetOfScopeChain()), JSInterfaceJIT::regT1);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
// Also initialize ReturnPC for use by lazy linking and exceptions.
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3);
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
jit.restoreArgumentReference();
JSInterfaceJIT::Call callLazyLink = jit.call();
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
jit.jump(JSInterfaceJIT::regT0);
slowCase.link(&jit);
JSInterfaceJIT::Call callNotJSFunction = generateSlowCaseFor(vm, jit);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
patchBuffer.link(callLazyLink, lazyLink);
patchBuffer.link(callNotJSFunction, notJSFunction);
return FINALIZE_CODE(patchBuffer, ("link %s trampoline", name));
}
static MacroAssemblerCodeRef virtualForThunkGenerator(
JSGlobalData* globalData, CodeSpecializationKind kind)
{
// The return address is on the stack, or in the link register. We will hence
// jump to the callee, or save the return address to the call frame while we
// make a C++ function call to the appropriate DFG operation.
CCallHelpers jit(globalData);
CCallHelpers::JumpList slowCase;
// FIXME: we should have a story for eliminating these checks. In many cases,
// the DFG knows that the value is definitely a cell, or definitely a function.
#if USE(JSVALUE64)
slowCase.append(
jit.branchTestPtr(
CCallHelpers::NonZero, GPRInfo::nonArgGPR0, GPRInfo::tagMaskRegister));
#else
slowCase.append(
jit.branch32(
CCallHelpers::NotEqual, GPRInfo::nonArgGPR1,
CCallHelpers::TrustedImm32(JSValue::CellTag)));
#endif
jit.loadPtr(CCallHelpers::Address(GPRInfo::nonArgGPR0, JSCell::structureOffset()), GPRInfo::nonArgGPR2);
slowCase.append(
jit.branchPtr(
CCallHelpers::NotEqual,
CCallHelpers::Address(GPRInfo::nonArgGPR2, Structure::classInfoOffset()),
CCallHelpers::TrustedImmPtr(&JSFunction::s_info)));
// Now we know we have a JSFunction.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::nonArgGPR0, JSFunction::offsetOfExecutable()),
GPRInfo::nonArgGPR2);
slowCase.append(
jit.branch32(
CCallHelpers::LessThan,
CCallHelpers::Address(
GPRInfo::nonArgGPR2, ExecutableBase::offsetOfNumParametersFor(kind)),
CCallHelpers::TrustedImm32(0)));
// Now we know that we have a CodeBlock, and we're committed to making a fast
// call.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::nonArgGPR0, JSFunction::offsetOfScopeChain()),
GPRInfo::nonArgGPR1);
#if USE(JSVALUE64)
jit.storePtr(
GPRInfo::nonArgGPR1,
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::ScopeChain));
#else
jit.storePtr(
GPRInfo::nonArgGPR1,
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::ScopeChain +
OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
jit.store32(
CCallHelpers::TrustedImm32(JSValue::CellTag),
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::ScopeChain +
OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
#endif
jit.loadPtr(
CCallHelpers::Address(GPRInfo::nonArgGPR2, ExecutableBase::offsetOfJITCodeWithArityCheckFor(kind)),
GPRInfo::regT0);
// Make a tail call. This will return back to DFG code.
emitPointerValidation(jit, GPRInfo::regT0);
jit.jump(GPRInfo::regT0);
slowCase.link(&jit);
// Here we don't know anything, so revert to the full slow path.
slowPathFor(jit, globalData, kind == CodeForCall ? operationVirtualCall : operationVirtualConstruct);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("DFG virtual %s slow path thunk", kind == CodeForCall ? "call" : "construct"));
}
MacroAssemblerCodeRef arityFixup(VM* vm)
{
JSInterfaceJIT jit;
// We enter with fixup count in regT0
#if USE(JSVALUE64)
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.addPtr(JSInterfaceJIT::TrustedImm32(-8), JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT1);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register to account for missing args
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT0);
jit.addPtr(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister);
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.addPtr(JSInterfaceJIT::TrustedImm32(-8), JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, 4), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register to account for missing args
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT0);
jit.addPtr(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister);
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
// We are creating a bunch of threads that touch the main thread's stack. This will make ASAN unhappy.
// The reason this is OK is that we guarantee that the main thread doesn't continue until all threads
// that could touch its stack are done executing.
SUPPRESS_ASAN
void Plan::run()
{
if (!parseAndValidateModule())
return;
auto tryReserveCapacity = [this] (auto& vector, size_t size, const char* what) {
if (UNLIKELY(!vector.tryReserveCapacity(size))) {
StringBuilder builder;
builder.appendLiteral("Failed allocating enough space for ");
builder.appendNumber(size);
builder.append(what);
m_errorMessage = builder.toString();
return false;
}
return true;
};
if (!tryReserveCapacity(m_wasmExitStubs, m_moduleInformation->importFunctionSignatureIndices.size(), " WebAssembly to JavaScript stubs")
|| !tryReserveCapacity(m_unlinkedWasmToWasmCalls, m_functionLocationInBinary.size(), " unlinked WebAssembly to WebAssembly calls")
|| !tryReserveCapacity(m_wasmInternalFunctions, m_functionLocationInBinary.size(), " WebAssembly functions")
|| !tryReserveCapacity(m_compilationContexts, m_functionLocationInBinary.size(), " compilation contexts"))
return;
m_unlinkedWasmToWasmCalls.resize(m_functionLocationInBinary.size());
m_wasmInternalFunctions.resize(m_functionLocationInBinary.size());
m_compilationContexts.resize(m_functionLocationInBinary.size());
for (unsigned importIndex = 0; importIndex < m_moduleInformation->imports.size(); ++importIndex) {
Import* import = &m_moduleInformation->imports[importIndex];
if (import->kind != ExternalKind::Function)
continue;
unsigned importFunctionIndex = m_wasmExitStubs.size();
if (verbose)
dataLogLn("Processing import function number ", importFunctionIndex, ": ", import->module, ": ", import->field);
SignatureIndex signatureIndex = m_moduleInformation->importFunctionSignatureIndices.at(import->kindIndex);
m_wasmExitStubs.uncheckedAppend(exitStubGenerator(m_vm, m_callLinkInfos, signatureIndex, importFunctionIndex));
}
m_currentIndex = 0;
auto doWork = [this] {
while (true) {
uint32_t functionIndex;
{
auto locker = holdLock(m_lock);
if (m_currentIndex >= m_functionLocationInBinary.size())
return;
functionIndex = m_currentIndex;
++m_currentIndex;
}
const uint8_t* functionStart = m_source + m_functionLocationInBinary[functionIndex].start;
size_t functionLength = m_functionLocationInBinary[functionIndex].end - m_functionLocationInBinary[functionIndex].start;
ASSERT(functionLength <= m_sourceLength);
SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
const Signature* signature = SignatureInformation::get(m_vm, signatureIndex);
unsigned functionIndexSpace = m_wasmExitStubs.size() + functionIndex;
ASSERT_UNUSED(functionIndexSpace, m_moduleInformation->signatureIndexFromFunctionIndexSpace(functionIndexSpace) == signatureIndex);
ASSERT(validateFunction(m_vm, functionStart, functionLength, signature, *m_moduleInformation, m_moduleSignatureIndicesToUniquedSignatureIndices));
m_unlinkedWasmToWasmCalls[functionIndex] = Vector<UnlinkedWasmToWasmCall>();
auto parseAndCompileResult = parseAndCompile(*m_vm, m_compilationContexts[functionIndex], functionStart, functionLength, signature, m_unlinkedWasmToWasmCalls[functionIndex], *m_moduleInformation, m_moduleSignatureIndicesToUniquedSignatureIndices);
if (UNLIKELY(!parseAndCompileResult)) {
auto locker = holdLock(m_lock);
if (!m_errorMessage) {
// Multiple compiles could fail simultaneously. We arbitrarily choose the first.
m_errorMessage = makeString(parseAndCompileResult.error(), ", in function at index ", String::number(functionIndex)); // FIXME make this an Expected.
}
m_currentIndex = m_functionLocationInBinary.size();
// We will terminate on the next execution.
continue;
}
m_wasmInternalFunctions[functionIndex] = WTFMove(*parseAndCompileResult);
}
};
MonotonicTime startTime;
if (verbose || Options::reportCompileTimes())
startTime = MonotonicTime::now();
uint32_t threadCount = Options::useConcurrentJIT() ? WTF::numberOfProcessorCores() : 1;
uint32_t numWorkerThreads = threadCount - 1;
Vector<ThreadIdentifier> threads;
threads.reserveCapacity(numWorkerThreads);
for (uint32_t i = 0; i < numWorkerThreads; i++)
threads.uncheckedAppend(createThread("jsc.wasm-b3-compilation.thread", doWork));
doWork(); // Let the main thread do some work too.
for (uint32_t i = 0; i < numWorkerThreads; i++)
waitForThreadCompletion(threads[i]);
for (uint32_t functionIndex = 0; functionIndex < m_functionLocationInBinary.size(); functionIndex++) {
{
CompilationContext& context = m_compilationContexts[functionIndex];
SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex];
String signatureDescription = SignatureInformation::get(m_vm, signatureIndex)->toString();
{
LinkBuffer linkBuffer(*m_vm, *context.wasmEntrypointJIT, nullptr);
m_wasmInternalFunctions[functionIndex]->wasmEntrypoint.compilation =
std::make_unique<B3::Compilation>(FINALIZE_CODE(linkBuffer, ("WebAssembly function[%i] %s", functionIndex, signatureDescription.ascii().data())), WTFMove(context.wasmEntrypointByproducts));
}
{
LinkBuffer linkBuffer(*m_vm, *context.jsEntrypointJIT, nullptr);
linkBuffer.link(context.jsEntrypointToWasmEntrypointCall, FunctionPtr(m_wasmInternalFunctions[functionIndex]->wasmEntrypoint.compilation->code().executableAddress()));
m_wasmInternalFunctions[functionIndex]->jsToWasmEntrypoint.compilation =
std::make_unique<B3::Compilation>(FINALIZE_CODE(linkBuffer, ("JavaScript->WebAssembly entrypoint[%i] %s", functionIndex, signatureDescription.ascii().data())), WTFMove(context.jsEntrypointByproducts));
}
}
}
if (verbose || Options::reportCompileTimes()) {
dataLogLn("Took ", (MonotonicTime::now() - startTime).microseconds(),
" us to compile and link the module");
}
// Patch the call sites for each WebAssembly function.
for (auto& unlinked : m_unlinkedWasmToWasmCalls) {
for (auto& call : unlinked) {
void* executableAddress;
if (m_moduleInformation->isImportedFunctionFromFunctionIndexSpace(call.functionIndex)) {
// FIXME imports could have been linked in B3, instead of generating a patchpoint. This condition should be replaced by a RELEASE_ASSERT. https://bugs.webkit.org/show_bug.cgi?id=166462
executableAddress = call.target == UnlinkedWasmToWasmCall::Target::ToJs
? m_wasmExitStubs.at(call.functionIndex).wasmToJs.code().executableAddress()
: m_wasmExitStubs.at(call.functionIndex).wasmToWasm.code().executableAddress();
} else {
ASSERT(call.target != UnlinkedWasmToWasmCall::Target::ToJs);
executableAddress = m_wasmInternalFunctions.at(call.functionIndex - m_wasmExitStubs.size())->wasmEntrypoint.compilation->code().executableAddress();
}
MacroAssembler::repatchCall(call.callLocation, CodeLocationLabel(executableAddress));
}
}
m_failed = false;
}
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind)
{
int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
JSInterfaceJIT jit;
jit.emitPutImmediateToCallFrameHeader(0, JSStack::CodeBlock);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.peek(JSInterfaceJIT::regT1);
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ReturnPC);
// Calling convention: f(ecx, edx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister); // Align stack after call.
// call the function
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister);
#elif CPU(X86_64)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.peek(JSInterfaceJIT::regT1);
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ReturnPC);
#if !OS(WINDOWS)
// Calling convention: f(edi, esi, edx, ecx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); // Align stack after call.
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#else
// Calling convention: f(ecx, edx, r8, r9, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
// Leave space for the callee parameter home addresses and align the stack.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
#elif CPU(ARM)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT2);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC);
// Calling convention: f(r0 == regT0, r1 == regT1, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, ARMRegisters::r0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARMRegisters::r1);
jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.loadPtr(JSInterfaceJIT::Address(ARMRegisters::r1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
#elif CPU(SH4)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT2);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC);
// Calling convention: f(r0 == regT4, r1 == regT5, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT4);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT5);
jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT5, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction), JSInterfaceJIT::regT0);
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
#elif CPU(MIPS)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved
jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC);
// Calling convention: f(a0, a1, a2, a3);
// Host function signature: f(ExecState*);
// Allocate stack space for 16 bytes (8-byte aligned)
// 16 bytes (unused) for 4 arguments
jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
// Setup arg0
jit.move(JSInterfaceJIT::callFrameRegister, MIPSRegisters::a0);
// Call
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, MIPSRegisters::a2);
jit.loadPtr(JSInterfaceJIT::Address(MIPSRegisters::a2, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack.
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
// Restore stack space
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3);
#else
#error "JIT not supported on this platform."
UNUSED_PARAM(executableOffsetToFunction);
breakpoint();
#endif
// Check for an exception
#if USE(JSVALUE64)
jit.load64(&(vm->exception), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
#else
JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::AbsoluteAddress(reinterpret_cast<char*>(&vm->exception) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
JSInterfaceJIT::TrustedImm32(JSValue::EmptyValueTag));
#endif
// Return.
jit.ret();
// Handle an exception
exceptionHandler.link(&jit);
// Grab the return address.
jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT1);
jit.move(JSInterfaceJIT::TrustedImmPtr(&vm->exceptionLocation), JSInterfaceJIT::regT2);
jit.storePtr(JSInterfaceJIT::regT1, JSInterfaceJIT::regT2);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(ctiVMThrowTrampolineSlowpath).value()), JSInterfaceJIT::regT1);
jit.jump(JSInterfaceJIT::regT1);
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("native %s trampoline", toCString(kind).data()));
}
MacroAssemblerCodeRef arityFixupGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
// We enter with fixup count in argumentGPR0
// We have the guarantee that a0, a1, a2, t3, t4 and t5 (or t0 for Windows) are all distinct :-)
#if USE(JSVALUE64)
#if OS(WINDOWS)
const GPRReg extraTemp = JSInterfaceJIT::regT0;
#else
const GPRReg extraTemp = JSInterfaceJIT::regT5;
#endif
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
// Check to see if we have extra slots we can use
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
JSInterfaceJIT::Label fillExtraSlots(jit.label());
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight));
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
noExtraSlot.link(&jit);
jit.neg64(JSInterfaceJIT::argumentGPR0);
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT3, extraTemp);
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
// Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::argumentGPR0, extraTemp);
jit.lshift64(JSInterfaceJIT::TrustedImm32(3), extraTemp);
jit.addPtr(extraTemp, JSInterfaceJIT::callFrameRegister);
jit.addPtr(extraTemp, JSInterfaceJIT::stackPointerRegister);
done.link(&jit);
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
// Check to see if we have extra slots we can use
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Label fillExtraSlots(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, TagOffset));
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
noExtraSlot.link(&jit);
jit.neg32(JSInterfaceJIT::argumentGPR0);
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, PayloadOffset), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, TagOffset), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
// Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::regT5);
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT5);
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::stackPointerRegister);
done.link(&jit);
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall)
{
int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
JSInterfaceJIT jit(vm);
if (entryType == EnterViaCall)
jit.emitFunctionPrologue();
#if USE(JSVALUE64)
else if (entryType == EnterViaJump) {
// We're coming from a specialized thunk that has saved the prior tag registers' contents.
// Restore them now.
#if CPU(ARM64)
jit.popPair(JSInterfaceJIT::tagTypeNumberRegister, JSInterfaceJIT::tagMaskRegister);
#else
jit.pop(JSInterfaceJIT::tagMaskRegister);
jit.pop(JSInterfaceJIT::tagTypeNumberRegister);
#endif
}
#endif
jit.emitPutToCallFrameHeader(0, JSStack::CodeBlock);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86)
// Calling convention: f(ecx, edx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue.
// call the function
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister);
#elif CPU(X86_64)
#if !OS(WINDOWS)
// Calling convention: f(edi, esi, edx, ecx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
#else
// Calling convention: f(ecx, edx, r8, r9, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
// Leave space for the callee parameter home addresses.
// At this point the stack is aligned to 16 bytes, but if this changes at some point, we need to emit code to align it.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
#elif CPU(ARM64)
COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0);
COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1);
COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1);
jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2);
jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction));
#elif CPU(ARM) || CPU(SH4) || CPU(MIPS)
#if CPU(MIPS)
// Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments.
jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
// Calling convention is f(argumentGPR0, argumentGPR1, ...).
// Host function signature is f(ExecState*).
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
#if CPU(MIPS)
// Restore stack space
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
#else
#error "JIT not supported on this platform."
UNUSED_PARAM(executableOffsetToFunction);
abortWithReason(TGNotSupported);
#endif
// Check for an exception
#if USE(JSVALUE64)
jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
#else
JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::AbsoluteAddress(vm->addressOfException()),
JSInterfaceJIT::TrustedImm32(0));
#endif
jit.emitFunctionEpilogue();
// Return.
jit.ret();
// Handle an exception
exceptionHandler.link(&jit);
jit.copyCalleeSavesToVMCalleeSavesBuffer();
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT0);
jit.push(JSInterfaceJIT::regT0);
#else
#if OS(WINDOWS)
// Allocate space on stack for the 4 parameter registers.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
#endif
jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3);
jit.call(JSInterfaceJIT::regT3);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#elif OS(WINDOWS)
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
jit.jumpToExceptionHandler();
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJump ? "Tail " : "", toCString(kind).data()));
}
static MacroAssemblerCodeRef virtualForThunkGenerator(
VM* vm, CodeSpecializationKind kind, RegisterPreservationMode registers)
{
// The callee is in regT0 (for JSVALUE32_64, the tag is in regT1).
// The return address is on the stack, or in the link register. We will hence
// jump to the callee, or save the return address to the call frame while we
// make a C++ function call to the appropriate JIT operation.
CCallHelpers jit(vm);
CCallHelpers::JumpList slowCase;
// FIXME: we should have a story for eliminating these checks. In many cases,
// the DFG knows that the value is definitely a cell, or definitely a function.
#if USE(JSVALUE64)
jit.move(CCallHelpers::TrustedImm64(TagMask), GPRInfo::regT2);
slowCase.append(
jit.branchTest64(
CCallHelpers::NonZero, GPRInfo::regT0, GPRInfo::regT2));
#else
slowCase.append(
jit.branch32(
CCallHelpers::NotEqual, GPRInfo::regT1,
CCallHelpers::TrustedImm32(JSValue::CellTag)));
#endif
AssemblyHelpers::emitLoadStructure(jit, GPRInfo::regT0, GPRInfo::regT2, GPRInfo::regT1);
slowCase.append(
jit.branchPtr(
CCallHelpers::NotEqual,
CCallHelpers::Address(GPRInfo::regT2, Structure::classInfoOffset()),
CCallHelpers::TrustedImmPtr(JSFunction::info())));
// Now we know we have a JSFunction.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfExecutable()),
GPRInfo::regT2);
jit.loadPtr(
CCallHelpers::Address(
GPRInfo::regT2, ExecutableBase::offsetOfJITCodeWithArityCheckFor(kind, registers)),
GPRInfo::regT2);
slowCase.append(jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT2));
// Now we know that we have a CodeBlock, and we're committed to making a fast
// call.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfScopeChain()),
GPRInfo::regT1);
#if USE(JSVALUE64)
jit.emitPutToCallFrameHeaderBeforePrologue(GPRInfo::regT1, JSStack::ScopeChain);
#else
jit.emitPutPayloadToCallFrameHeaderBeforePrologue(GPRInfo::regT1, JSStack::ScopeChain);
jit.emitPutTagToCallFrameHeaderBeforePrologue(CCallHelpers::TrustedImm32(JSValue::CellTag),
JSStack::ScopeChain);
#endif
// Make a tail call. This will return back to JIT code.
emitPointerValidation(jit, GPRInfo::regT2);
jit.jump(GPRInfo::regT2);
slowCase.link(&jit);
// Here we don't know anything, so revert to the full slow path.
slowPathFor(jit, vm, operationVirtualFor(kind, registers));
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("Virtual %s%s slow path thunk", kind == CodeForCall ? "call" : "construct", registers == MustPreserveRegisters ? " that preserves registers" : ""));
}
MacroAssemblerCodeRef arityFixup(VM* vm)
{
JSInterfaceJIT jit(vm);
// We enter with fixup count, in aligned stack units, in regT0 and the return thunk in
// regT5 on 32-bit and regT7 on 64-bit.
#if USE(JSVALUE64)
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.lshift32(JSInterfaceJIT::TrustedImm32(logStackAlignmentRegisters()), JSInterfaceJIT::regT0);
jit.neg64(JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT6);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame down regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT6, JSInterfaceJIT::regT1);
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT6, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT6);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 - 1 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT1);
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT6, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT6);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1);
jit.lshift64(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT1);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::stackPointerRegister);
// Save the original return PC.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()), GPRInfo::regT1);
jit.storePtr(GPRInfo::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT6, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
// Install the new return PC.
jit.storePtr(GPRInfo::regT7, JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()));
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.lshift32(JSInterfaceJIT::TrustedImm32(logStackAlignmentRegisters()), JSInterfaceJIT::regT0);
jit.neg32(JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame down regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, 4), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 - 1 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1);
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT1);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::stackPointerRegister);
// Save the original return PC.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()), GPRInfo::regT1);
jit.storePtr(GPRInfo::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
// Install the new return PC.
jit.storePtr(GPRInfo::regT5, JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()));
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall)
{
int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
JSInterfaceJIT jit(vm);
if (entryType == EnterViaCall)
jit.emitFunctionPrologue();
jit.emitPutImmediateToCallFrameHeader(0, JSStack::CodeBlock);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
// Calling convention: f(ecx, edx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue.
// call the function
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister);
#elif CPU(X86_64)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
#if !OS(WINDOWS)
// Calling convention: f(edi, esi, edx, ecx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
#else
// Calling convention: f(ecx, edx, r8, r9, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
// Leave space for the callee parameter home addresses and align the stack.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
#elif CPU(ARM64)
COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT1, prev_callframe_not_trampled_by_T1);
COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT3, prev_callframe_not_trampled_by_T3);
COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0);
COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1);
COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2);
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(ARM64Registers::x3);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, ARM64Registers::x3);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1);
jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2);
jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction));
#elif CPU(ARM) || CPU(SH4) || CPU(MIPS)
// Load caller frame's scope chain into this callframe so that whatever we call can get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT2);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
#if CPU(MIPS)
// Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments.
jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
// Calling convention is f(argumentGPR0, argumentGPR1, ...).
// Host function signature is f(ExecState*).
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
#if CPU(MIPS)
// Restore stack space
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
#else
#error "JIT not supported on this platform."
UNUSED_PARAM(executableOffsetToFunction);
breakpoint();
#endif
// Check for an exception
#if USE(JSVALUE64)
jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
#else
JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::AbsoluteAddress(reinterpret_cast<char*>(vm->addressOfException()) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
JSInterfaceJIT::TrustedImm32(JSValue::EmptyValueTag));
#endif
jit.emitFunctionEpilogue();
// Return.
jit.ret();
// Handle an exception
exceptionHandler.link(&jit);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister), JSInterfaceJIT::regT0);
jit.push(JSInterfaceJIT::regT0);
#else
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister), JSInterfaceJIT::argumentGPR0);
#endif
jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3);
jit.call(JSInterfaceJIT::regT3);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
jit.jumpToExceptionHandler();
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJump ? "Tail " : "", toCString(kind).data()));
}
