void LazySlowPath::generate(CodeBlock* codeBlock)
{
RELEASE_ASSERT(!m_stub);
VM& vm = *codeBlock->vm();
CCallHelpers jit(&vm, codeBlock);
GenerationParams params;
CCallHelpers::JumpList exceptionJumps;
params.exceptionJumps = m_exceptionTarget ? &exceptionJumps : nullptr;
params.lazySlowPath = this;
unsigned bytesSaved = m_scratchRegisterAllocator.preserveReusedRegistersByPushing(jit, ScratchRegisterAllocator::ExtraStackSpace::NoExtraSpace);
// This is needed because LLVM may create a stackmap location that is the register SP.
// But on arm64, SP is also the same register number as ZR, so LLVM is telling us that it has
// proven something is zero. Our MASM isn't universally compatible with arm64's context dependent
// notion of SP meaning ZR. We just make things easier by ensuring we do the necessary move of zero
// into a non-SP register.
if (m_newZeroValueRegister != InvalidGPRReg)
jit.move(CCallHelpers::TrustedImm32(0), m_newZeroValueRegister);
m_generator->run(jit, params);
CCallHelpers::Label doneLabel;
CCallHelpers::Jump jumpToEndOfPatchpoint;
if (bytesSaved) {
doneLabel = jit.label();
m_scratchRegisterAllocator.restoreReusedRegistersByPopping(jit, bytesSaved, ScratchRegisterAllocator::ExtraStackSpace::NoExtraSpace);
jumpToEndOfPatchpoint = jit.jump();
}
LinkBuffer linkBuffer(vm, jit, codeBlock, JITCompilationMustSucceed);
if (bytesSaved) {
linkBuffer.link(params.doneJumps, linkBuffer.locationOf(doneLabel));
linkBuffer.link(jumpToEndOfPatchpoint, m_patchpoint.labelAtOffset(MacroAssembler::maxJumpReplacementSize()));
} else
linkBuffer.link(params.doneJumps, m_patchpoint.labelAtOffset(MacroAssembler::maxJumpReplacementSize()));
if (m_exceptionTarget)
linkBuffer.link(exceptionJumps, m_exceptionTarget);
m_stub = FINALIZE_CODE_FOR(codeBlock, linkBuffer, ("Lazy slow path call stub"));
MacroAssembler::replaceWithJump(m_patchpoint, CodeLocationLabel(m_stub.code()));
}
MacroAssemblerCodeRef osrEntryThunkGenerator(VM* vm)
{
AssemblyHelpers jit(nullptr);
// We get passed the address of a scratch buffer. The first 8-byte slot of the buffer
// is the frame size. The second 8-byte slot is the pointer to where we are supposed to
// jump. The remaining bytes are the new call frame header followed by the locals.
ptrdiff_t offsetOfFrameSize = 0; // This is the DFG frame count.
ptrdiff_t offsetOfTargetPC = offsetOfFrameSize + sizeof(EncodedJSValue);
ptrdiff_t offsetOfPayload = offsetOfTargetPC + sizeof(EncodedJSValue);
ptrdiff_t offsetOfLocals = offsetOfPayload + sizeof(Register) * CallFrame::headerSizeInRegisters;
jit.move(GPRInfo::returnValueGPR2, GPRInfo::regT0);
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfFrameSize), GPRInfo::regT1); // Load the frame size.
jit.move(GPRInfo::regT1, GPRInfo::regT2);
jit.lshiftPtr(MacroAssembler::Imm32(3), GPRInfo::regT2);
jit.move(GPRInfo::callFrameRegister, MacroAssembler::stackPointerRegister);
jit.subPtr(GPRInfo::regT2, MacroAssembler::stackPointerRegister);
MacroAssembler::Label loop = jit.label();
jit.subPtr(MacroAssembler::TrustedImm32(1), GPRInfo::regT1);
jit.move(GPRInfo::regT1, GPRInfo::regT4);
jit.negPtr(GPRInfo::regT4);
jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals), GPRInfo::regT2);
jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals + sizeof(int32_t)), GPRInfo::regT3);
jit.store32(GPRInfo::regT2, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register))));
jit.store32(GPRInfo::regT3, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register)) + static_cast<intptr_t>(sizeof(int32_t))));
jit.branchPtr(MacroAssembler::NotEqual, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(-static_cast<intptr_t>(CallFrame::headerSizeInRegisters)))).linkTo(loop, &jit);
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfTargetPC), GPRInfo::regT1);
MacroAssembler::Jump ok = jit.branchPtr(MacroAssembler::Above, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(static_cast<intptr_t>(1000))));
jit.abortWithReason(DFGUnreasonableOSREntryJumpDestination);
ok.link(&jit);
jit.restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(*vm);
jit.emitMaterializeTagCheckRegisters();
jit.jump(GPRInfo::regT1);
LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("DFG OSR entry thunk"));
}
MacroAssemblerCodeRef absThunkGenerator(VM* vm)
{
SpecializedThunkJIT jit(1);
if (!jit.supportsFloatingPointAbs())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
MacroAssembler::Jump nonIntJump;
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
jit.rshift32(SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(31), SpecializedThunkJIT::regT1);
jit.add32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
jit.xor32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
jit.appendFailure(jit.branch32(MacroAssembler::Equal, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1 << 31)));
jit.returnInt32(SpecializedThunkJIT::regT0);
nonIntJump.link(&jit);
// Shame about the double int conversion here.
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
jit.absDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
jit.returnDouble(SpecializedThunkJIT::fpRegT1);
return jit.finalize(*vm, vm->jitStubs->ctiNativeCall(vm), "abs");
}
MacroAssemblerCodeRef absThunkGenerator(JSGlobalData* globalData)
{
SpecializedThunkJIT jit(1, globalData);
if (!jit.supportsDoubleBitops())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(globalData->jitStubs->ctiNativeCall());
MacroAssembler::Jump nonIntJump;
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
jit.rshift32(SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(31), SpecializedThunkJIT::regT1);
jit.add32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
jit.xor32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
jit.appendFailure(jit.branch32(MacroAssembler::Equal, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1 << 31)));
jit.returnInt32(SpecializedThunkJIT::regT0);
nonIntJump.link(&jit);
// Shame about the double int conversion here.
jit.loadDouble(&negativeZeroConstant, SpecializedThunkJIT::fpRegT1);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
jit.andnotDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
jit.returnDouble(SpecializedThunkJIT::fpRegT1);
return jit.finalize(*globalData, globalData->jitStubs->ctiNativeCall());
}
static MacroAssemblerCodeRef linkForThunkGenerator(
JSGlobalData* globalData, CodeSpecializationKind kind)
{
// The return address is on the stack or in the link register. We will hence
// save the return address to the call frame while we make a C++ function call
// to perform linking and lazy compilation if necessary. We expect the callee
// to be in nonArgGPR0/nonArgGPR1 (payload/tag), the call frame to have already
// been adjusted, nonPreservedNonReturnGPR holds the exception handler index,
// and all other registers to be available for use. We use JITStackFrame::args
// to save important information across calls.
CCallHelpers jit(globalData);
slowPathFor(jit, globalData, kind == CodeForCall ? operationLinkCall : operationLinkConstruct);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("DFG link %s slow path thunk", kind == CodeForCall ? "call" : "construct"));
}
MacroAssemblerCodeRef clz32ThunkGenerator(VM* vm)
{
SpecializedThunkJIT jit(vm, 1);
MacroAssembler::Jump nonIntArgJump;
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntArgJump);
SpecializedThunkJIT::Label convertedArgumentReentry(&jit);
jit.countLeadingZeros32(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
jit.returnInt32(SpecializedThunkJIT::regT1);
if (jit.supportsFloatingPointTruncate()) {
nonIntArgJump.link(&jit);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(convertedArgumentReentry, &jit);
jit.appendFailure(jit.jump());
} else
jit.appendFailure(nonIntArgJump);
return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "clz32");
}
// We will jump here if the JIT code tries to make a call, but the
// linking helper (C++ code) decides to throw an exception instead.
MacroAssemblerCodeRef throwExceptionFromCallSlowPathGenerator(VM* vm)
{
CCallHelpers jit(vm);
// The call pushed a return address, so we need to pop it back off to re-align the stack,
// even though we won't use it.
jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR);
// The CallFrame register points to the (failed) callee frame, so we need to pop back one frame.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(GPRInfo::callFrameRegister);
jit.setupArgumentsExecState();
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(lookupExceptionHandler)), GPRInfo::nonArgGPR0);
emitPointerValidation(jit, GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
jit.jumpToExceptionHandler();
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("Throw exception from call slow path thunk"));
}
MacroAssemblerCodeRef floorThunkGenerator(VM* vm)
{
SpecializedThunkJIT jit(vm, 1);
MacroAssembler::Jump nonIntJump;
if (!UnaryDoubleOpWrapper(floor) || !jit.supportsFloatingPoint())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
jit.returnInt32(SpecializedThunkJIT::regT0);
nonIntJump.link(&jit);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
#if CPU(ARM64)
SpecializedThunkJIT::JumpList doubleResult;
jit.floorDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT0);
#else
SpecializedThunkJIT::Jump intResult;
SpecializedThunkJIT::JumpList doubleResult;
if (jit.supportsFloatingPointTruncate()) {
jit.loadDouble(&zeroConstant, SpecializedThunkJIT::fpRegT1);
doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
SpecializedThunkJIT::JumpList slowPath;
// Handle the negative doubles in the slow path for now.
slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0));
intResult = jit.jump();
slowPath.link(&jit);
}
jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(floor));
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
if (jit.supportsFloatingPointTruncate())
intResult.link(&jit);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT0);
#endif // CPU(ARM64)
return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "floor");
}
MacroAssemblerCodeRef baselineGetterReturnThunkGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
#if USE(JSVALUE64)
jit.move(GPRInfo::returnValueGPR, GPRInfo::regT0);
#else
jit.setupResults(GPRInfo::regT0, GPRInfo::regT1);
#endif
unsigned numberOfParameters = 0;
numberOfParameters++; // The 'this' argument.
numberOfParameters++; // The true return PC.
unsigned numberOfRegsForCall =
JSStack::CallFrameHeaderSize + numberOfParameters;
unsigned numberOfBytesForCall =
numberOfRegsForCall * sizeof(Register) - sizeof(CallerFrameAndPC);
unsigned alignedNumberOfBytesForCall =
WTF::roundUpToMultipleOf(stackAlignmentBytes(), numberOfBytesForCall);
// The real return address is stored above the arguments. We passed one argument, which is
// 'this'. So argument at index 1 is the return address.
jit.loadPtr(
AssemblyHelpers::Address(
AssemblyHelpers::stackPointerRegister,
(virtualRegisterForArgument(1).offset() - JSStack::CallerFrameAndPCSize) * sizeof(Register)),
GPRInfo::regT2);
jit.addPtr(
AssemblyHelpers::TrustedImm32(alignedNumberOfBytesForCall),
AssemblyHelpers::stackPointerRegister);
jit.jump(GPRInfo::regT2);
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("baseline getter return thunk"));
}
MacroAssemblerCodeRef ceilThunkGenerator(VM* vm)
{
SpecializedThunkJIT jit(vm, 1);
if (!UnaryDoubleOpWrapper(ceil) || !jit.supportsFloatingPoint())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
MacroAssembler::Jump nonIntJump;
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
jit.returnInt32(SpecializedThunkJIT::regT0);
nonIntJump.link(&jit);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
#if CPU(ARM64)
jit.ceilDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
#else
jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(ceil));
#endif // CPU(ARM64)
SpecializedThunkJIT::JumpList doubleResult;
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT0);
return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "ceil");
}
MacroAssemblerCodeRef arityFixupGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
// We enter with fixup count in argumentGPR0
// We have the guarantee that a0, a1, a2, t3, t4 and t5 (or t0 for Windows) are all distinct :-)
#if USE(JSVALUE64)
#if OS(WINDOWS)
const GPRReg extraTemp = JSInterfaceJIT::regT0;
#else
const GPRReg extraTemp = JSInterfaceJIT::regT5;
#endif
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
// Check to see if we have extra slots we can use
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
JSInterfaceJIT::Label fillExtraSlots(jit.label());
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight));
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
noExtraSlot.link(&jit);
jit.neg64(JSInterfaceJIT::argumentGPR0);
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT3, extraTemp);
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
// Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::argumentGPR0, extraTemp);
jit.lshift64(JSInterfaceJIT::TrustedImm32(3), extraTemp);
jit.addPtr(extraTemp, JSInterfaceJIT::callFrameRegister);
jit.addPtr(extraTemp, JSInterfaceJIT::stackPointerRegister);
done.link(&jit);
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
// Check to see if we have extra slots we can use
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Label fillExtraSlots(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, TagOffset));
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
noExtraSlot.link(&jit);
jit.neg32(JSInterfaceJIT::argumentGPR0);
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, PayloadOffset), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, TagOffset), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
// Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::regT5);
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT5);
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::stackPointerRegister);
done.link(&jit);
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
CodeLocationLabel* ArityCheckFailReturnThunks::returnPCsFor(
VM& vm, unsigned numExpectedArgumentsIncludingThis)
{
ASSERT(numExpectedArgumentsIncludingThis >= 1);
numExpectedArgumentsIncludingThis = WTF::roundUpToMultipleOf(
stackAlignmentRegisters(), numExpectedArgumentsIncludingThis);
{
ConcurrentJITLocker locker(m_lock);
if (numExpectedArgumentsIncludingThis < m_nextSize)
return m_returnPCArrays.last().get();
}
ASSERT(!isCompilationThread());
numExpectedArgumentsIncludingThis = std::max(numExpectedArgumentsIncludingThis, m_nextSize * 2);
AssemblyHelpers jit(&vm, 0);
Vector<AssemblyHelpers::Label> labels;
for (unsigned size = m_nextSize; size <= numExpectedArgumentsIncludingThis; size += stackAlignmentRegisters()) {
labels.append(jit.label());
jit.load32(
AssemblyHelpers::Address(
AssemblyHelpers::stackPointerRegister,
(JSStack::ArgumentCount - JSStack::CallerFrameAndPCSize) * sizeof(Register) +
PayloadOffset),
GPRInfo::regT4);
jit.add32(
AssemblyHelpers::TrustedImm32(
JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + size - 1),
GPRInfo::regT4, GPRInfo::regT2);
jit.lshift32(AssemblyHelpers::TrustedImm32(3), GPRInfo::regT2);
jit.addPtr(AssemblyHelpers::stackPointerRegister, GPRInfo::regT2);
jit.loadPtr(GPRInfo::regT2, GPRInfo::regT2);
jit.addPtr(
AssemblyHelpers::TrustedImm32(size * sizeof(Register)),
AssemblyHelpers::stackPointerRegister);
// Thunks like ours want to use the return PC to figure out where things
// were saved. So, we pay it forward.
jit.store32(
GPRInfo::regT4,
AssemblyHelpers::Address(
AssemblyHelpers::stackPointerRegister,
(JSStack::ArgumentCount - JSStack::CallerFrameAndPCSize) * sizeof(Register) +
PayloadOffset));
jit.jump(GPRInfo::regT2);
}
LinkBuffer linkBuffer(vm, &jit, GLOBAL_THUNK_ID);
unsigned returnPCsSize = numExpectedArgumentsIncludingThis / stackAlignmentRegisters() + 1;
std::unique_ptr<CodeLocationLabel[]> returnPCs =
std::make_unique<CodeLocationLabel[]>(returnPCsSize);
for (unsigned size = 0; size <= numExpectedArgumentsIncludingThis; size += stackAlignmentRegisters()) {
unsigned index = size / stackAlignmentRegisters();
RELEASE_ASSERT(index < returnPCsSize);
if (size < m_nextSize)
returnPCs[index] = m_returnPCArrays.last()[index];
else
returnPCs[index] = linkBuffer.locationOf(labels[(size - m_nextSize) / stackAlignmentRegisters()]);
}
CodeLocationLabel* result = returnPCs.get();
{
ConcurrentJITLocker locker(m_lock);
m_returnPCArrays.append(std::move(returnPCs));
m_refs.append(FINALIZE_CODE(linkBuffer, ("Arity check fail return thunks for up to numArgs = %u", numExpectedArgumentsIncludingThis)));
m_nextSize = numExpectedArgumentsIncludingThis + stackAlignmentRegisters();
}
return result;
}
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall)
{
int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
JSInterfaceJIT jit(vm);
if (entryType == EnterViaCall)
jit.emitFunctionPrologue();
jit.emitPutImmediateToCallFrameHeader(0, JSStack::CodeBlock);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
// Calling convention: f(ecx, edx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue.
// call the function
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister);
#elif CPU(X86_64)
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
#if !OS(WINDOWS)
// Calling convention: f(edi, esi, edx, ecx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
#else
// Calling convention: f(ecx, edx, r8, r9, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
// Leave space for the callee parameter home addresses and align the stack.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
#elif CPU(ARM64)
COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT1, prev_callframe_not_trampled_by_T1);
COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT3, prev_callframe_not_trampled_by_T3);
COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0);
COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1);
COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2);
// Load caller frame's scope chain into this callframe so that whatever we call can
// get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(ARM64Registers::x3);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, ARM64Registers::x3);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1);
jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2);
jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction));
#elif CPU(ARM) || CPU(SH4) || CPU(MIPS)
// Load caller frame's scope chain into this callframe so that whatever we call can get to its global data.
jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT2);
jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2);
jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain);
#if CPU(MIPS)
// Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments.
jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
// Calling convention is f(argumentGPR0, argumentGPR1, ...).
// Host function signature is f(ExecState*).
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
#if CPU(MIPS)
// Restore stack space
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
#else
#error "JIT not supported on this platform."
UNUSED_PARAM(executableOffsetToFunction);
breakpoint();
#endif
// Check for an exception
#if USE(JSVALUE64)
jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
#else
JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::AbsoluteAddress(reinterpret_cast<char*>(vm->addressOfException()) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)),
JSInterfaceJIT::TrustedImm32(JSValue::EmptyValueTag));
#endif
jit.emitFunctionEpilogue();
// Return.
jit.ret();
// Handle an exception
exceptionHandler.link(&jit);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister), JSInterfaceJIT::regT0);
jit.push(JSInterfaceJIT::regT0);
#else
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister), JSInterfaceJIT::argumentGPR0);
#endif
jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3);
jit.call(JSInterfaceJIT::regT3);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
jit.jumpToExceptionHandler();
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJump ? "Tail " : "", toCString(kind).data()));
}
MacroAssemblerCodeRef arityFixup(VM* vm)
{
JSInterfaceJIT jit(vm);
// We enter with fixup count, in aligned stack units, in regT0 and the return thunk in
// regT5 on 32-bit and regT7 on 64-bit.
#if USE(JSVALUE64)
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.lshift32(JSInterfaceJIT::TrustedImm32(logStackAlignmentRegisters()), JSInterfaceJIT::regT0);
jit.neg64(JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT6);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame down regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT6, JSInterfaceJIT::regT1);
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT6, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT6);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 - 1 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT1);
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT6, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT6);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1);
jit.lshift64(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT1);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::stackPointerRegister);
// Save the original return PC.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()), GPRInfo::regT1);
jit.storePtr(GPRInfo::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT6, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
// Install the new return PC.
jit.storePtr(GPRInfo::regT7, JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()));
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.lshift32(JSInterfaceJIT::TrustedImm32(logStackAlignmentRegisters()), JSInterfaceJIT::regT0);
jit.neg32(JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame down regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, 4), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 - 1 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT1);
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT1);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT1, JSInterfaceJIT::stackPointerRegister);
// Save the original return PC.
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()), GPRInfo::regT1);
jit.storePtr(GPRInfo::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
// Install the new return PC.
jit.storePtr(GPRInfo::regT5, JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, CallFrame::returnPCOffset()));
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
static MacroAssemblerCodeRef virtualForThunkGenerator(
JSGlobalData* globalData, CodeSpecializationKind kind)
{
// The return address is on the stack, or in the link register. We will hence
// jump to the callee, or save the return address to the call frame while we
// make a C++ function call to the appropriate DFG operation.
CCallHelpers jit(globalData);
CCallHelpers::JumpList slowCase;
// FIXME: we should have a story for eliminating these checks. In many cases,
// the DFG knows that the value is definitely a cell, or definitely a function.
#if USE(JSVALUE64)
slowCase.append(
jit.branchTestPtr(
CCallHelpers::NonZero, GPRInfo::nonArgGPR0, GPRInfo::tagMaskRegister));
#else
slowCase.append(
jit.branch32(
CCallHelpers::NotEqual, GPRInfo::nonArgGPR1,
CCallHelpers::TrustedImm32(JSValue::CellTag)));
#endif
jit.loadPtr(CCallHelpers::Address(GPRInfo::nonArgGPR0, JSCell::structureOffset()), GPRInfo::nonArgGPR2);
slowCase.append(
jit.branchPtr(
CCallHelpers::NotEqual,
CCallHelpers::Address(GPRInfo::nonArgGPR2, Structure::classInfoOffset()),
CCallHelpers::TrustedImmPtr(&JSFunction::s_info)));
// Now we know we have a JSFunction.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::nonArgGPR0, JSFunction::offsetOfExecutable()),
GPRInfo::nonArgGPR2);
slowCase.append(
jit.branch32(
CCallHelpers::LessThan,
CCallHelpers::Address(
GPRInfo::nonArgGPR2, ExecutableBase::offsetOfNumParametersFor(kind)),
CCallHelpers::TrustedImm32(0)));
// Now we know that we have a CodeBlock, and we're committed to making a fast
// call.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::nonArgGPR0, JSFunction::offsetOfScopeChain()),
GPRInfo::nonArgGPR1);
#if USE(JSVALUE64)
jit.storePtr(
GPRInfo::nonArgGPR1,
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::ScopeChain));
#else
jit.storePtr(
GPRInfo::nonArgGPR1,
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::ScopeChain +
OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
jit.store32(
CCallHelpers::TrustedImm32(JSValue::CellTag),
CCallHelpers::Address(
GPRInfo::callFrameRegister,
static_cast<ptrdiff_t>(sizeof(Register)) * RegisterFile::ScopeChain +
OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
#endif
jit.loadPtr(
CCallHelpers::Address(GPRInfo::nonArgGPR2, ExecutableBase::offsetOfJITCodeWithArityCheckFor(kind)),
GPRInfo::regT0);
// Make a tail call. This will return back to DFG code.
emitPointerValidation(jit, GPRInfo::regT0);
jit.jump(GPRInfo::regT0);
slowCase.link(&jit);
// Here we don't know anything, so revert to the full slow path.
slowPathFor(jit, globalData, kind == CodeForCall ? operationVirtualCall : operationVirtualConstruct);
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("DFG virtual %s slow path thunk", kind == CodeForCall ? "call" : "construct"));
}
static MacroAssemblerCodeRef virtualForThunkGenerator(
VM* vm, CodeSpecializationKind kind, RegisterPreservationMode registers)
{
// The callee is in regT0 (for JSVALUE32_64, the tag is in regT1).
// The return address is on the stack, or in the link register. We will hence
// jump to the callee, or save the return address to the call frame while we
// make a C++ function call to the appropriate JIT operation.
CCallHelpers jit(vm);
CCallHelpers::JumpList slowCase;
// FIXME: we should have a story for eliminating these checks. In many cases,
// the DFG knows that the value is definitely a cell, or definitely a function.
#if USE(JSVALUE64)
jit.move(CCallHelpers::TrustedImm64(TagMask), GPRInfo::regT2);
slowCase.append(
jit.branchTest64(
CCallHelpers::NonZero, GPRInfo::regT0, GPRInfo::regT2));
#else
slowCase.append(
jit.branch32(
CCallHelpers::NotEqual, GPRInfo::regT1,
CCallHelpers::TrustedImm32(JSValue::CellTag)));
#endif
AssemblyHelpers::emitLoadStructure(jit, GPRInfo::regT0, GPRInfo::regT2, GPRInfo::regT1);
slowCase.append(
jit.branchPtr(
CCallHelpers::NotEqual,
CCallHelpers::Address(GPRInfo::regT2, Structure::classInfoOffset()),
CCallHelpers::TrustedImmPtr(JSFunction::info())));
// Now we know we have a JSFunction.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfExecutable()),
GPRInfo::regT2);
jit.loadPtr(
CCallHelpers::Address(
GPRInfo::regT2, ExecutableBase::offsetOfJITCodeWithArityCheckFor(kind, registers)),
GPRInfo::regT2);
slowCase.append(jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT2));
// Now we know that we have a CodeBlock, and we're committed to making a fast
// call.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfScopeChain()),
GPRInfo::regT1);
#if USE(JSVALUE64)
jit.emitPutToCallFrameHeaderBeforePrologue(GPRInfo::regT1, JSStack::ScopeChain);
#else
jit.emitPutPayloadToCallFrameHeaderBeforePrologue(GPRInfo::regT1, JSStack::ScopeChain);
jit.emitPutTagToCallFrameHeaderBeforePrologue(CCallHelpers::TrustedImm32(JSValue::CellTag),
JSStack::ScopeChain);
#endif
// Make a tail call. This will return back to JIT code.
emitPointerValidation(jit, GPRInfo::regT2);
jit.jump(GPRInfo::regT2);
slowCase.link(&jit);
// Here we don't know anything, so revert to the full slow path.
slowPathFor(jit, vm, operationVirtualFor(kind, registers));
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("Virtual %s%s slow path thunk", kind == CodeForCall ? "call" : "construct", registers == MustPreserveRegisters ? " that preserves registers" : ""));
}
virtual void perform() {
const char* thread_name = "rbx.jit";
ManagedThread::set_current(ls_, thread_name);
ls_->set_run_state(ManagedThread::eIndependent);
RUBINIUS_THREAD_START(thread_name, ls_->thread_id(), 1);
#ifndef RBX_WINDOWS
sigset_t set;
sigfillset(&set);
pthread_sigmask(SIG_SETMASK, &set, NULL);
#endif
for(;;) { // forever
BackgroundCompileRequest* req = 0;
// Lock, wait, get a request, unlock
{
utilities::thread::Mutex::LockGuard guard(mutex_);
if(pause_) {
state = cPaused;
paused_ = true;
pause_condition_.broadcast();
if(stop_) goto halt;
while(pause_) {
condition_.wait(mutex_);
if(stop_) goto halt;
}
state = cUnknown;
paused_ = false;
}
// If we've been asked to stop, do so now.
if(stop_) goto halt;
while(pending_requests_.empty()) {
state = cIdle;
// unlock and wait...
condition_.wait(mutex_);
if(stop_) goto halt;
}
// now locked again, shift a request
req = pending_requests_.front();
state = cRunning;
}
// This isn't ideal, but it's the safest. Keep the GC from
// running while we're building the IR.
ls_->gc_dependent();
Context ctx(ls_);
jit::Compiler jit(&ctx);
// mutex now unlock, allowing others to push more requests
//
current_req_ = req;
current_compiler_ = &jit;
int spec_id = 0;
Class* cls = req->receiver_class();
if(cls && !cls->nil_p()) {
spec_id = cls->class_id();
}
void* func = 0;
{
timer::Running<1000000> timer(ls_->shared().stats.jit_time_spent);
jit.compile(req);
func = jit.generate_function();
}
// We were unable to compile this function, likely
// because it's got something we don't support.
if(!func) {
if(ls_->config().jit_show_compiling) {
CompiledCode* code = req->method();
llvm::outs() << "[[[ JIT error background compiling "
<< ls_->enclosure_name(code) << "#" << ls_->symbol_debug_str(code->name())
<< (req->is_block() ? " (block)" : " (method)")
<< " ]]]\n";
}
// If someone was waiting on this, wake them up.
if(utilities::thread::Condition* cond = req->waiter()) {
cond->signal();
}
current_req_ = 0;
current_compiler_ = 0;
pending_requests_.pop_front();
delete req;
// We don't depend on the GC here, so let it run independent
// of us.
ls_->gc_independent();
continue;
}
if(show_machine_code_) {
jit.show_machine_code();
}
// If the method has had jit'ing request disabled since we started
// JIT'ing it, discard our work.
if(!req->machine_code()->jit_disabled()) {
jit::RuntimeDataHolder* rd = ctx.runtime_data_holder();
atomic::memory_barrier();
ls_->start_method_update();
if(!req->is_block()) {
if(spec_id) {
req->method()->add_specialized(spec_id, reinterpret_cast<executor>(func), rd);
} else {
req->method()->set_unspecialized(reinterpret_cast<executor>(func), rd);
}
} else {
req->method()->set_unspecialized(reinterpret_cast<executor>(func), rd);
}
req->machine_code()->clear_compiling();
// assert(req->method()->jit_data());
ls_->end_method_update();
rd->run_write_barrier(ls_->write_barrier(), req->method());
ls_->shared().stats.jitted_methods++;
if(ls_->config().jit_show_compiling) {
CompiledCode* code = req->method();
llvm::outs() << "[[[ JIT finished background compiling "
<< ls_->enclosure_name(code) << "#" << ls_->symbol_debug_str(code->name())
<< (req->is_block() ? " (block)" : " (method)")
<< " ]]]\n";
}
}
// If someone was waiting on this, wake them up.
if(utilities::thread::Condition* cond = req->waiter()) {
cond->signal();
}
current_req_ = 0;
current_compiler_ = 0;
pending_requests_.pop_front();
delete req;
// We don't depend on the GC here, so let it run independent
// of us.
ls_->gc_independent();
}
halt:
RUBINIUS_THREAD_STOP(thread_name, ls_->thread_id(), 1);
}
void jit (const char* const file_contents) {
struct vector instruction_stream;
struct stack relocation_table = { .size = 0, .items = { 0 } };
int relocation_site = 0;
int relative_offset = 0;
GUARD(vector_create(&instruction_stream, 100));
char prologue [] = {
0x55, // push rbp
0x48, 0x89, 0xE5, // mov rsp, rbp
// backup callee saved registers
0x41, 0x54, // pushq %r12
0x41, 0x55, // pushq %r13
0x41, 0x56, // pushq %r14
// %rdi = memset
// %rsi = putchar
// %rdx = getgetchar
// backup args to callee saved registers
0x49, 0x89, 0xFC, // movq %rdi, %r12
0x49, 0x89, 0xF5, // movq %rsi, %r13
0x49, 0x89, 0xD6, // movq %rdx, %r14
// %r12 = memset
// %r13 = putchar
// %r14 = getchar
// allocate 30,008 B on stack
0x48, 0x81, 0xEC, 0x38, 0x75, 0x00, 0x00, // subq $30000, %rsp
// address of beginning of tape
0x48, 0x8D, 0x3C, 0x24, // leaq (%rsp), %rdi
// fill with 0's
0xBE, 0x00, 0x00, 0x00, 0x00, // movl $0, %esi
// length 30,000 B
0x48, 0xC7, 0xC2, 0x30, 0x75, 0x00, 0x00, // movq $30000, %rdx
// memset
0x41, 0xFF, 0xD4, // callq *%r12
0x49, 0x89, 0xE4 // movq %rsp, %r12
// %r12 = &tape[0];
};
GUARD(vector_push(&instruction_stream, prologue, sizeof(prologue)));
for (unsigned long i = 0; file_contents[i] != '\0'; ++i) {
switch (file_contents[i]) {
case '>':
// see: http://stackoverflow.com/a/8550253/1027966
{
char opcodes [] = {
0x49, 0xFF, 0xC4 // inc %r12
};
GUARD(vector_push(&instruction_stream, opcodes, sizeof(opcodes)));
}
break;
case '<':
{
char opcodes [] = {
0x49, 0xFF, 0xCC // dec %r12
};
GUARD(vector_push(&instruction_stream, opcodes, sizeof(opcodes)));
}
break;
case '+':
{
char opcodes [] = {
0x41, 0xFE, 0x04, 0x24 // incb (%r12)
};
GUARD(vector_push(&instruction_stream, opcodes, sizeof(opcodes)));
}
break;
case '-':
{
char opcodes [] = {
0x41, 0xFE, 0x0C, 0x24 // decv (%r12)
};
GUARD(vector_push(&instruction_stream, opcodes, sizeof(opcodes)));
}
break;
case '.':
{
char opcodes [] = {
0x41, 0x0F, 0xB6, 0x3C, 0x24, // movzbl (%r12), %edi
0x41, 0xFF, 0xD5 // callq *%r13
};
GUARD(vector_push(&instruction_stream, opcodes, sizeof(opcodes)));
}
break;
case ',':
{
char opcodes [] = {
0x41, 0xFF, 0xD6, // callq *%r14
0x41, 0x88, 0x04, 0x24 // movb %al, (%r12)
};
GUARD(vector_push(&instruction_stream, opcodes, sizeof(opcodes)));
}
break;
case '[':
{
char opcodes [] = {
0x41, 0x80, 0x3C, 0x24, 0x00, // cmpb $0, (%r12)
// Needs to be patched up
0x0F, 0x84, 0x00, 0x00, 0x00, 0x00 // je <32b relative offset, 2's compliment, LE>
};
GUARD(vector_push(&instruction_stream, opcodes, sizeof(opcodes)));
}
GUARD(stack_push(&relocation_table, instruction_stream.size)); // create a label after
break;
case ']':
{
char opcodes [] = {
0x41, 0x80, 0x3C, 0x24, 0x00, // cmpb $0, (%r12)
// Needs to be patched up
0x0F, 0x85, 0x00, 0x00, 0x00, 0x00 // jne <32b relative offset, 2's compliment, LE>
};
GUARD(vector_push(&instruction_stream, opcodes, sizeof(opcodes)));
}
// patches self and matching open bracket
GUARD(stack_pop(&relocation_table, &relocation_site));
relative_offset = instruction_stream.size - relocation_site;
vector_write32LE(&instruction_stream, instruction_stream.size - 4, -relative_offset);
vector_write32LE(&instruction_stream, relocation_site - 4, relative_offset);
break;
}
}
char epilogue [] = {
0x48, 0x81, 0xC4, 0x38, 0x75, 0x00, 0x00, // addq $30008, %rsp
// restore callee saved registers
0x41, 0x5E, // popq %r14
0x41, 0x5D, // popq %r13
0x41, 0x5C, // popq %r12
0x5d, // pop rbp
0xC3 // ret
};
GUARD(vector_push(&instruction_stream, epilogue, sizeof(epilogue)));
/*print_instruction_stream(&instruction_stream);*/
void* mem = mmap(NULL, instruction_stream.size, PROT_WRITE | PROT_EXEC,
MAP_ANON | MAP_PRIVATE, -1, 0);
memcpy(mem, instruction_stream.data, instruction_stream.size);
void (*jitted_func) (fn_memset, fn_putchar, fn_getchar) = mem;
jitted_func(memset, putchar, getchar);
munmap(mem, instruction_stream.size);
vector_destroy(&instruction_stream);
}
int main (int argc, char* argv []) {
if (argc != 2) err("Usage: jit inputfile");
char* file_contents = read_file(argv[1]);
if (file_contents == NULL) err("Couldn't open file");
jit(file_contents);
free(file_contents);
}
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall)
{
int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind);
JSInterfaceJIT jit(vm);
if (entryType == EnterViaCall)
jit.emitFunctionPrologue();
#if USE(JSVALUE64)
else if (entryType == EnterViaJump) {
// We're coming from a specialized thunk that has saved the prior tag registers' contents.
// Restore them now.
#if CPU(ARM64)
jit.popPair(JSInterfaceJIT::tagTypeNumberRegister, JSInterfaceJIT::tagMaskRegister);
#else
jit.pop(JSInterfaceJIT::tagMaskRegister);
jit.pop(JSInterfaceJIT::tagTypeNumberRegister);
#endif
}
#endif
jit.emitPutToCallFrameHeader(0, JSStack::CodeBlock);
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86)
// Calling convention: f(ecx, edx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue.
// call the function
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister);
#elif CPU(X86_64)
#if !OS(WINDOWS)
// Calling convention: f(edi, esi, edx, ecx, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
#else
// Calling convention: f(ecx, edx, r8, r9, ...);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx);
// Leave space for the callee parameter home addresses.
// At this point the stack is aligned to 16 bytes, but if this changes at some point, we need to emit code to align it.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx);
jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9);
jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction));
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
#elif CPU(ARM64)
COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0);
COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1);
COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2);
// Host function signature: f(ExecState*);
jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1);
jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2);
jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction));
#elif CPU(ARM) || CPU(SH4) || CPU(MIPS)
#if CPU(MIPS)
// Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments.
jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
// Calling convention is f(argumentGPR0, argumentGPR1, ...).
// Host function signature is f(ExecState*).
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1);
jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2);
jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction));
#if CPU(MIPS)
// Restore stack space
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#endif
#else
#error "JIT not supported on this platform."
UNUSED_PARAM(executableOffsetToFunction);
abortWithReason(TGNotSupported);
#endif
// Check for an exception
#if USE(JSVALUE64)
jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2);
JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2);
#else
JSInterfaceJIT::Jump exceptionHandler = jit.branch32(
JSInterfaceJIT::NotEqual,
JSInterfaceJIT::AbsoluteAddress(vm->addressOfException()),
JSInterfaceJIT::TrustedImm32(0));
#endif
jit.emitFunctionEpilogue();
// Return.
jit.ret();
// Handle an exception
exceptionHandler.link(&jit);
jit.copyCalleeSavesToVMCalleeSavesBuffer();
jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT0);
jit.push(JSInterfaceJIT::regT0);
#else
#if OS(WINDOWS)
// Allocate space on stack for the 4 parameter registers.
jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0);
#endif
jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3);
jit.call(JSInterfaceJIT::regT3);
#if CPU(X86) && USE(JSVALUE32_64)
jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister);
#elif OS(WINDOWS)
jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister);
#endif
jit.jumpToExceptionHandler();
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJump ? "Tail " : "", toCString(kind).data()));
}
static MacroAssemblerCodeRef arrayIteratorNextThunkGenerator(VM* vm, ArrayIterationKind kind)
{
typedef SpecializedThunkJIT::TrustedImm32 TrustedImm32;
typedef SpecializedThunkJIT::TrustedImmPtr TrustedImmPtr;
typedef SpecializedThunkJIT::Address Address;
typedef SpecializedThunkJIT::BaseIndex BaseIndex;
typedef SpecializedThunkJIT::Jump Jump;
SpecializedThunkJIT jit(vm);
// Make sure we're being called on an array iterator, and load m_iteratedObject, and m_nextIndex into regT0 and regT1 respectively
jit.loadArgumentWithSpecificClass(JSArrayIterator::info(), SpecializedThunkJIT::ThisArgument, SpecializedThunkJIT::regT4, SpecializedThunkJIT::regT1);
// Early exit if we don't have a thunk for this form of iteration
jit.appendFailure(jit.branch32(SpecializedThunkJIT::AboveOrEqual, Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfIterationKind()), TrustedImm32(ArrayIterateKeyValue)));
jit.loadPtr(Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfIteratedObject()), SpecializedThunkJIT::regT0);
jit.load32(Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex()), SpecializedThunkJIT::regT1);
// Pull out the butterfly from iteratedObject
jit.load8(Address(SpecializedThunkJIT::regT0, JSCell::indexingTypeOffset()), SpecializedThunkJIT::regT3);
jit.loadPtr(Address(SpecializedThunkJIT::regT0, JSObject::butterflyOffset()), SpecializedThunkJIT::regT2);
jit.and32(TrustedImm32(IndexingShapeMask), SpecializedThunkJIT::regT3);
Jump notDone = jit.branch32(SpecializedThunkJIT::Below, SpecializedThunkJIT::regT1, Address(SpecializedThunkJIT::regT2, Butterfly::offsetOfPublicLength()));
// Return the termination signal to indicate that we've finished
jit.move(TrustedImmPtr(vm->iterationTerminator.get()), SpecializedThunkJIT::regT0);
jit.returnJSCell(SpecializedThunkJIT::regT0);
notDone.link(&jit);
if (kind == ArrayIterateKey) {
jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex()));
jit.returnInt32(SpecializedThunkJIT::regT1);
return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "array-iterator-next-key");
}
ASSERT(kind == ArrayIterateValue);
// Okay, now we're returning a value so make sure we're inside the vector size
jit.appendFailure(jit.branch32(SpecializedThunkJIT::AboveOrEqual, SpecializedThunkJIT::regT1, Address(SpecializedThunkJIT::regT2, Butterfly::offsetOfVectorLength())));
// So now we perform inline loads for int32, value/undecided, and double storage
Jump undecidedStorage = jit.branch32(SpecializedThunkJIT::Equal, SpecializedThunkJIT::regT3, TrustedImm32(UndecidedShape));
Jump notContiguousStorage = jit.branch32(SpecializedThunkJIT::NotEqual, SpecializedThunkJIT::regT3, TrustedImm32(ContiguousShape));
undecidedStorage.link(&jit);
jit.loadPtr(Address(SpecializedThunkJIT::regT0, JSObject::butterflyOffset()), SpecializedThunkJIT::regT2);
#if USE(JSVALUE64)
jit.load64(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight), SpecializedThunkJIT::regT0);
Jump notHole = jit.branchTest64(SpecializedThunkJIT::NonZero, SpecializedThunkJIT::regT0);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT0);
notHole.link(&jit);
jit.addPtr(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex()));
jit.returnJSValue(SpecializedThunkJIT::regT0);
#else
jit.load32(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight, JSValue::offsetOfTag()), SpecializedThunkJIT::regT3);
Jump notHole = jit.branch32(SpecializedThunkJIT::NotEqual, SpecializedThunkJIT::regT3, TrustedImm32(JSValue::EmptyValueTag));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1);
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT0);
jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex()));
jit.returnJSValue(SpecializedThunkJIT::regT0, JSInterfaceJIT::regT1);
notHole.link(&jit);
jit.load32(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight, JSValue::offsetOfPayload()), SpecializedThunkJIT::regT0);
jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex()));
jit.move(SpecializedThunkJIT::regT3, SpecializedThunkJIT::regT1);
jit.returnJSValue(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
#endif
notContiguousStorage.link(&jit);
Jump notInt32Storage = jit.branch32(SpecializedThunkJIT::NotEqual, SpecializedThunkJIT::regT3, TrustedImm32(Int32Shape));
jit.loadPtr(Address(SpecializedThunkJIT::regT0, JSObject::butterflyOffset()), SpecializedThunkJIT::regT2);
jit.load32(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight, JSValue::offsetOfPayload()), SpecializedThunkJIT::regT0);
jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex()));
jit.returnInt32(SpecializedThunkJIT::regT0);
notInt32Storage.link(&jit);
jit.appendFailure(jit.branch32(SpecializedThunkJIT::NotEqual, SpecializedThunkJIT::regT3, TrustedImm32(DoubleShape)));
jit.loadPtr(Address(SpecializedThunkJIT::regT0, JSObject::butterflyOffset()), SpecializedThunkJIT::regT2);
jit.loadDouble(BaseIndex(SpecializedThunkJIT::regT2, SpecializedThunkJIT::regT1, SpecializedThunkJIT::TimesEight), SpecializedThunkJIT::fpRegT0);
jit.add32(TrustedImm32(1), Address(SpecializedThunkJIT::regT4, JSArrayIterator::offsetOfNextIndex()));
jit.returnDouble(SpecializedThunkJIT::fpRegT0);
return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "array-iterator-next-value");
}
void compileOSRExit(ExecState* exec)
{
if (exec->vm().callFrameForCatch)
RELEASE_ASSERT(exec->vm().callFrameForCatch == exec);
CodeBlock* codeBlock = exec->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->jitType() == JITCode::DFGJIT);
VM* vm = &exec->vm();
// It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
// really be profitable.
DeferGCForAWhile deferGC(vm->heap);
uint32_t exitIndex = vm->osrExitIndex;
OSRExit& exit = codeBlock->jitCode()->dfg()->osrExit[exitIndex];
if (vm->callFrameForCatch)
ASSERT(exit.m_kind == GenericUnwind);
if (exit.isExceptionHandler())
ASSERT(!!vm->exception());
prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
// Compute the value recoveries.
Operands<ValueRecovery> operands;
codeBlock->jitCode()->dfg()->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->jitCode()->dfg()->minifiedDFG, exit.m_streamIndex, operands);
SpeculationRecovery* recovery = 0;
if (exit.m_recoveryIndex != UINT_MAX)
recovery = &codeBlock->jitCode()->dfg()->speculationRecovery[exit.m_recoveryIndex];
{
CCallHelpers jit(vm, codeBlock);
OSRExitCompiler exitCompiler(jit);
if (exit.m_kind == GenericUnwind) {
// We are acting as a defacto op_catch because we arrive here from genericUnwind().
// So, we must restore our call frame and stack pointer.
jit.restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer();
jit.loadPtr(vm->addressOfCallFrameForCatch(), GPRInfo::callFrameRegister);
jit.addPtr(CCallHelpers::TrustedImm32(codeBlock->stackPointerOffset() * sizeof(Register)),
GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
}
jit.jitAssertHasValidCallFrame();
if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
Profiler::Database& database = *vm->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
exitCompiler.compileExit(exit, operands, recovery);
LinkBuffer patchBuffer(*vm, jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldDumpDisassembly() || Options::verboseOSR(),
patchBuffer,
("DFG OSR exit #%u (%s, %s) from %s, with operands = %s",
exitIndex, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(operands)).data()));
}
MacroAssembler::repatchJump(exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
vm->osrExitJumpDestination = exit.m_code.code().executableAddress();
}
MacroAssemblerCodeRef generateRegisterPreservationWrapper(VM& vm, ExecutableBase* executable, MacroAssemblerCodePtr target)
{
#if ENABLE(FTL_JIT)
// We shouldn't ever be generating wrappers for native functions.
RegisterSet toSave = registersToPreserve();
ptrdiff_t offset = registerPreservationOffset();
AssemblyHelpers jit(&vm, 0);
jit.preserveReturnAddressAfterCall(GPRInfo::regT1);
jit.load32(
AssemblyHelpers::Address(
AssemblyHelpers::stackPointerRegister,
(JSStack::ArgumentCount - JSStack::CallerFrameAndPCSize) * sizeof(Register) + PayloadOffset),
GPRInfo::regT2);
// Place the stack pointer where we want it to be.
jit.subPtr(AssemblyHelpers::TrustedImm32(offset), AssemblyHelpers::stackPointerRegister);
// Compute the number of things we will be copying.
jit.add32(
AssemblyHelpers::TrustedImm32(
JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize),
GPRInfo::regT2);
ASSERT(!toSave.get(GPRInfo::regT4));
jit.move(AssemblyHelpers::stackPointerRegister, GPRInfo::regT4);
AssemblyHelpers::Label loop = jit.label();
jit.sub32(AssemblyHelpers::TrustedImm32(1), GPRInfo::regT2);
jit.load64(AssemblyHelpers::Address(GPRInfo::regT4, offset), GPRInfo::regT0);
jit.store64(GPRInfo::regT0, GPRInfo::regT4);
jit.addPtr(AssemblyHelpers::TrustedImm32(sizeof(Register)), GPRInfo::regT4);
jit.branchTest32(AssemblyHelpers::NonZero, GPRInfo::regT2).linkTo(loop, &jit);
// At this point regT4 + offset points to where we save things.
ptrdiff_t currentOffset = 0;
jit.storePtr(GPRInfo::regT1, AssemblyHelpers::Address(GPRInfo::regT4, currentOffset));
for (GPRReg gpr = AssemblyHelpers::firstRegister(); gpr <= AssemblyHelpers::lastRegister(); gpr = static_cast<GPRReg>(gpr + 1)) {
if (!toSave.get(gpr))
continue;
currentOffset += sizeof(Register);
jit.store64(gpr, AssemblyHelpers::Address(GPRInfo::regT4, currentOffset));
}
// Assume that there aren't any saved FP registers.
// Restore the tag registers.
jit.move(AssemblyHelpers::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
jit.add64(AssemblyHelpers::TrustedImm32(TagMask - TagTypeNumber), GPRInfo::tagTypeNumberRegister, GPRInfo::tagMaskRegister);
jit.move(
AssemblyHelpers::TrustedImmPtr(
vm.getCTIStub(registerRestorationThunkGenerator).code().executableAddress()),
GPRInfo::nonArgGPR0);
jit.restoreReturnAddressBeforeReturn(GPRInfo::nonArgGPR0);
AssemblyHelpers::Jump jump = jit.jump();
LinkBuffer linkBuffer(vm, &jit, GLOBAL_THUNK_ID);
linkBuffer.link(jump, CodeLocationLabel(target));
if (Options::verboseFTLToJSThunk())
dataLog("Need a thunk for calls from FTL to non-FTL version of ", *executable, "\n");
return FINALIZE_DFG_CODE(linkBuffer, ("Register preservation wrapper for %s/%s, %p", toCString(executable->hashFor(CodeForCall)).data(), toCString(executable->hashFor(CodeForConstruct)).data(), target.executableAddress()));
#else // ENABLE(FTL_JIT)
UNUSED_PARAM(vm);
UNUSED_PARAM(executable);
UNUSED_PARAM(target);
// We don't support non-FTL builds for two reasons:
// - It just so happens that currently only the FTL bottoms out in this code.
// - The code above uses 64-bit instructions. It doesn't necessarily have to; it would be
// easy to change it so that it doesn't. But obviously making that change would be a
// prerequisite to removing this #if.
UNREACHABLE_FOR_PLATFORM();
return MacroAssemblerCodeRef();
#endif // ENABLE(FTL_JIT)
}
MacroAssemblerCodeRef arityFixup(VM* vm)
{
JSInterfaceJIT jit(vm);
// We enter with fixup count in regT0
#if USE(JSVALUE64)
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.neg64(JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame down regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), JSInterfaceJIT::regT1);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register to account for missing args
jit.lshift64(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT0);
jit.addPtr(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister);
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.neg32(JSInterfaceJIT::regT0);
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * 8), JSInterfaceJIT::regT2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::regT2);
// Move current frame down regT0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(JSInterfaceJIT::regT3, JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, 4), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(copyLoop, &jit);
// Fill in regT0 missing arg slots with undefined
jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::regT2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT1);
jit.store32(JSInterfaceJIT::regT1, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::regT0, JSInterfaceJIT::TimesEight, 4));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::regT2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register to account for missing args
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT0);
jit.addPtr(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister);
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
void compileOSRExit(ExecState* exec)
{
SamplingRegion samplingRegion("DFG OSR Exit Compilation");
CodeBlock* codeBlock = exec->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->jitType() == JITCode::DFGJIT);
VM* vm = &exec->vm();
// It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
// really be profitable.
DeferGCForAWhile deferGC(vm->heap);
uint32_t exitIndex = vm->osrExitIndex;
OSRExit& exit = codeBlock->jitCode()->dfg()->osrExit[exitIndex];
prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
// Compute the value recoveries.
Operands<ValueRecovery> operands;
codeBlock->jitCode()->dfg()->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->jitCode()->dfg()->minifiedDFG, exit.m_streamIndex, operands);
// There may be an override, for forward speculations.
if (!!exit.m_valueRecoveryOverride) {
operands.setOperand(
exit.m_valueRecoveryOverride->operand, exit.m_valueRecoveryOverride->recovery);
}
SpeculationRecovery* recovery = 0;
if (exit.m_recoveryIndex != UINT_MAX)
recovery = &codeBlock->jitCode()->dfg()->speculationRecovery[exit.m_recoveryIndex];
{
CCallHelpers jit(vm, codeBlock);
OSRExitCompiler exitCompiler(jit);
jit.jitAssertHasValidCallFrame();
if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
Profiler::Database& database = *vm->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
exitCompiler.compileExit(exit, operands, recovery);
LinkBuffer patchBuffer(*vm, jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldShowDisassembly() || Options::verboseOSR(),
patchBuffer,
("DFG OSR exit #%u (%s, %s) from %s, with operands = %s",
exitIndex, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(operands)).data()));
}
{
RepatchBuffer repatchBuffer(codeBlock);
repatchBuffer.relink(exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
}
vm->osrExitJumpDestination = exit.m_code.code().executableAddress();
}
// FIXME: We should distinguish between a megamorphic virtual call vs. a slow
// path virtual call so that we can enable fast tail calls for megamorphic
// virtual calls by using the shuffler.
// https://bugs.webkit.org/show_bug.cgi?id=148831
MacroAssemblerCodeRef virtualThunkFor(VM* vm, CallLinkInfo& callLinkInfo)
{
// The callee is in regT0 (for JSVALUE32_64, the tag is in regT1).
// The return address is on the stack, or in the link register. We will hence
// jump to the callee, or save the return address to the call frame while we
// make a C++ function call to the appropriate JIT operation.
CCallHelpers jit(vm);
CCallHelpers::JumpList slowCase;
// This is a slow path execution, and regT2 contains the CallLinkInfo. Count the
// slow path execution for the profiler.
jit.add32(
CCallHelpers::TrustedImm32(1),
CCallHelpers::Address(GPRInfo::regT2, CallLinkInfo::offsetOfSlowPathCount()));
// FIXME: we should have a story for eliminating these checks. In many cases,
// the DFG knows that the value is definitely a cell, or definitely a function.
#if USE(JSVALUE64)
jit.move(CCallHelpers::TrustedImm64(TagMask), GPRInfo::regT4);
slowCase.append(
jit.branchTest64(
CCallHelpers::NonZero, GPRInfo::regT0, GPRInfo::regT4));
#else
slowCase.append(
jit.branch32(
CCallHelpers::NotEqual, GPRInfo::regT1,
CCallHelpers::TrustedImm32(JSValue::CellTag)));
#endif
AssemblyHelpers::emitLoadStructure(jit, GPRInfo::regT0, GPRInfo::regT4, GPRInfo::regT1);
slowCase.append(
jit.branchPtr(
CCallHelpers::NotEqual,
CCallHelpers::Address(GPRInfo::regT4, Structure::classInfoOffset()),
CCallHelpers::TrustedImmPtr(JSFunction::info())));
// Now we know we have a JSFunction.
jit.loadPtr(
CCallHelpers::Address(GPRInfo::regT0, JSFunction::offsetOfExecutable()),
GPRInfo::regT4);
jit.loadPtr(
CCallHelpers::Address(
GPRInfo::regT4, ExecutableBase::offsetOfJITCodeWithArityCheckFor(
callLinkInfo.specializationKind())),
GPRInfo::regT4);
slowCase.append(jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::regT4));
// Now we know that we have a CodeBlock, and we're committed to making a fast
// call.
// Make a tail call. This will return back to JIT code.
emitPointerValidation(jit, GPRInfo::regT4);
if (callLinkInfo.isTailCall()) {
jit.preserveReturnAddressAfterCall(GPRInfo::regT0);
jit.prepareForTailCallSlow(GPRInfo::regT4);
}
jit.jump(GPRInfo::regT4);
slowCase.link(&jit);
// Here we don't know anything, so revert to the full slow path.
slowPathFor(jit, vm, operationVirtualCall);
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(
patchBuffer,
("Virtual %s slow path thunk",
callLinkInfo.callMode() == CallMode::Regular ? "call" : callLinkInfo.callMode() == CallMode::Tail ? "tail call" : "construct"));
}
void LLVMState::run(STATE) {
GCTokenImpl gct;
JITCompileRequest* compile_request = nil<JITCompileRequest>();
OnStack<1> os(state, compile_request);
metrics().init(metrics::eJITMetrics);
state->gc_dependent(gct, 0);
bool show_machine_code_ = jit_dump_code() & cMachineCode;
while(!thread_exit_) {
current_compiler_ = 0;
{
GCIndependent guard(state, 0);
{
utilities::thread::Mutex::LockGuard lg(compile_lock_);
while(compile_list_.get()->empty_p()) {
compile_cond_.wait(compile_lock_);
if(thread_exit_) break;
}
}
}
if(thread_exit_) break;
{
utilities::thread::Mutex::LockGuard guard(request_lock_);
compile_request = try_as<JITCompileRequest>(compile_list_.get()->shift(state));
if(!compile_request || compile_request->nil_p()) continue;
}
utilities::thread::Condition* cond = compile_request->waiter();
// Don't proceed until requester has reached the wait_cond
if(cond) wait_mutex.lock();
Context ctx(this);
jit::Compiler jit(&ctx);
current_compiler_ = &jit;
uint32_t class_id = 0;
uint32_t serial_id = 0;
void* func = 0;
try {
if(compile_request->receiver_class() &&
!compile_request->receiver_class()->nil_p()) {
// Apparently already compiled, probably some race
if(compile_request->method()->find_specialized(
compile_request->receiver_class())) {
if(config().jit_show_compiling) {
CompiledCode* code = compile_request->method();
llvm::outs() << "[[[ JIT already compiled "
<< enclosure_name(code) << "#" << symbol_debug_str(code->name())
<< (compile_request->is_block() ? " (block)" : " (method)")
<< " ]]]\n";
}
// If someone was waiting on this, wake them up.
if(cond) {
wait_mutex.unlock();
cond->signal();
}
current_compiler_ = 0;
continue;
}
class_id = compile_request->receiver_class()->class_id();
serial_id = compile_request->receiver_class()->serial_id();
}
{
timer::StopWatch<timer::microseconds> timer(
metrics().m.jit_metrics.time_last_us,
metrics().m.jit_metrics.time_total_us);
jit.compile(compile_request);
bool indy = !config().jit_sync;
func = jit.generate_function(indy);
}
// We were unable to compile this function, likely
// because it's got something we don't support.
if(!func) {
if(config().jit_show_compiling) {
CompiledCode* code = compile_request->method();
llvm::outs() << "[[[ JIT error background compiling "
<< enclosure_name(code) << "#" << symbol_debug_str(code->name())
<< (compile_request->is_block() ? " (block)" : " (method)")
<< " ]]]\n";
}
// If someone was waiting on this, wake them up.
if(cond) {
wait_mutex.unlock();
cond->signal();
}
current_compiler_ = 0;
continue;
}
} catch(LLVMState::CompileError& e) {
utilities::logger::warn("JIT: compile error: %s", e.error());
metrics().m.jit_metrics.methods_failed++;
// If someone was waiting on this, wake them up.
if(cond) {
wait_mutex.unlock();
cond->signal();
}
current_compiler_ = 0;
continue;
}
if(show_machine_code_) {
jit.show_machine_code();
}
// If the method has had jit'ing request disabled since we started
// JIT'ing it, discard our work.
if(!compile_request->machine_code()->jit_disabled()) {
jit::RuntimeDataHolder* rd = ctx.runtime_data_holder();
atomic::memory_barrier();
start_method_update();
if(!compile_request->is_block()) {
if(class_id) {
compile_request->method()->add_specialized(state,
class_id, serial_id, reinterpret_cast<executor>(func), rd);
} else {
compile_request->method()->set_unspecialized(reinterpret_cast<executor>(func), rd);
}
} else {
compile_request->method()->set_unspecialized(reinterpret_cast<executor>(func), rd);
}
compile_request->machine_code()->clear_compiling();
end_method_update();
rd->run_write_barrier(shared().om, compile_request->method());
if(config().jit_show_compiling) {
CompiledCode* code = compile_request->method();
llvm::outs() << "[[[ JIT finished background compiling "
<< enclosure_name(code) << "#" << symbol_debug_str(code->name())
<< (compile_request->is_block() ? " (block)" : " (method)")
<< " ]]]\n";
}
}
// If someone was waiting on this, wake them up.
if(cond) {
wait_mutex.unlock();
cond->signal();
}
current_compiler_ = 0;
metrics().m.jit_metrics.methods_compiled++;
}
}
void compileOSRExit(ExecState* exec)
{
SamplingRegion samplingRegion("DFG OSR Exit Compilation");
CodeBlock* codeBlock = exec->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->getJITType() == JITCode::DFGJIT);
JSGlobalData* globalData = &exec->globalData();
uint32_t exitIndex = globalData->osrExitIndex;
OSRExit& exit = codeBlock->osrExit(exitIndex);
// Make sure all code on our inline stack is JIT compiled. This is necessary since
// we may opt to inline a code block even before it had ever been compiled by the
// JIT, but our OSR exit infrastructure currently only works if the target of the
// OSR exit is JIT code. This could be changed since there is nothing particularly
// hard about doing an OSR exit into the interpreter, but for now this seems to make
// sense in that if we're OSR exiting from inlined code of a DFG code block, then
// probably it's a good sign that the thing we're exiting into is hot. Even more
// interestingly, since the code was inlined, it may never otherwise get JIT
// compiled since the act of inlining it may ensure that it otherwise never runs.
for (CodeOrigin codeOrigin = exit.m_codeOrigin; codeOrigin.inlineCallFrame; codeOrigin = codeOrigin.inlineCallFrame->caller) {
static_cast<FunctionExecutable*>(codeOrigin.inlineCallFrame->executable.get())
->baselineCodeBlockFor(codeOrigin.inlineCallFrame->isCall ? CodeForCall : CodeForConstruct)
->jitCompile(exec);
}
// Compute the value recoveries.
Operands<ValueRecovery> operands;
codeBlock->variableEventStream().reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->minifiedDFG(), exit.m_streamIndex, operands);
// There may be an override, for forward speculations.
if (!!exit.m_valueRecoveryOverride) {
operands.setOperand(
exit.m_valueRecoveryOverride->operand, exit.m_valueRecoveryOverride->recovery);
}
SpeculationRecovery* recovery = 0;
if (exit.m_recoveryIndex)
recovery = &codeBlock->speculationRecovery(exit.m_recoveryIndex - 1);
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLog(
"Generating OSR exit #", exitIndex, " (seq#", exit.m_streamIndex,
", bc#", exit.m_codeOrigin.bytecodeIndex, ", @", exit.m_nodeIndex, ", ",
exit.m_kind, ") for ", *codeBlock, ".\n");
#endif
{
CCallHelpers jit(globalData, codeBlock);
OSRExitCompiler exitCompiler(jit);
jit.jitAssertHasValidCallFrame();
if (globalData->m_perBytecodeProfiler && codeBlock->compilation()) {
Profiler::Database& database = *globalData->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->compilation();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind,
exit.m_watchpointIndex != std::numeric_limits<unsigned>::max());
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
exitCompiler.compileExit(exit, operands, recovery);
LinkBuffer patchBuffer(*globalData, &jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldShowDisassembly(),
patchBuffer,
("DFG OSR exit #%u (bc#%u, @%u, %s) from %s",
exitIndex, exit.m_codeOrigin.bytecodeIndex, exit.m_nodeIndex,
exitKindToString(exit.m_kind), toCString(*codeBlock).data()));
}
{
RepatchBuffer repatchBuffer(codeBlock);
repatchBuffer.relink(exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
}
globalData->osrExitJumpDestination = exit.m_code.code().executableAddress();
}
