MacroAssemblerCodePtr powThunkGenerator(JSGlobalData* globalData, ExecutablePool* pool)
{
#if USE(JSVALUE64) || USE(JSVALUE32_64)
SpecializedThunkJIT jit(2, globalData, pool);
if (!jit.supportsFloatingPoint())
return globalData->jitStubs->ctiNativeCall();
jit.loadDouble(&oneConstant, SpecializedThunkJIT::fpRegT1);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
MacroAssembler::Jump nonIntExponent;
jit.loadInt32Argument(1, SpecializedThunkJIT::regT0, nonIntExponent);
jit.appendFailure(jit.branch32(MacroAssembler::LessThan, SpecializedThunkJIT::regT0, MacroAssembler::Imm32(0)));
MacroAssembler::Jump exponentIsZero = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0);
MacroAssembler::Label startLoop(jit.label());
MacroAssembler::Jump exponentIsEven = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0, MacroAssembler::Imm32(1));
jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
exponentIsEven.link(&jit);
jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
jit.rshift32(MacroAssembler::Imm32(1), SpecializedThunkJIT::regT0);
jit.branchTest32(MacroAssembler::NonZero, SpecializedThunkJIT::regT0).linkTo(startLoop, &jit);
exponentIsZero.link(&jit);
{
SpecializedThunkJIT::JumpList doubleResult;
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT1);
}
if (jit.supportsFloatingPointSqrt()) {
nonIntExponent.link(&jit);
jit.loadDouble(&negativeHalfConstant, SpecializedThunkJIT::fpRegT3);
jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::regT0);
jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleLessThanOrEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::fpRegT3));
jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
jit.divDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
SpecializedThunkJIT::JumpList doubleResult;
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT1);
} else
jit.appendFailure(nonIntExponent);
return jit.finalize(globalData->jitStubs->ctiNativeCall());
#else
UNUSED_PARAM(pool);
return globalData->jitStubs->ctiNativeCall();
#endif
}
MacroAssemblerCodeRef powThunkGenerator(JSGlobalData* globalData)
{
SpecializedThunkJIT jit(2);
if (!jit.supportsFloatingPoint())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(globalData->jitStubs->ctiNativeCall());
jit.loadDouble(&oneConstant, SpecializedThunkJIT::fpRegT1);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
MacroAssembler::Jump nonIntExponent;
jit.loadInt32Argument(1, SpecializedThunkJIT::regT0, nonIntExponent);
jit.appendFailure(jit.branch32(MacroAssembler::LessThan, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(0)));
MacroAssembler::Jump exponentIsZero = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0);
MacroAssembler::Label startLoop(jit.label());
MacroAssembler::Jump exponentIsEven = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1));
jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
exponentIsEven.link(&jit);
jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
jit.rshift32(MacroAssembler::TrustedImm32(1), SpecializedThunkJIT::regT0);
jit.branchTest32(MacroAssembler::NonZero, SpecializedThunkJIT::regT0).linkTo(startLoop, &jit);
exponentIsZero.link(&jit);
{
SpecializedThunkJIT::JumpList doubleResult;
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT1);
}
if (jit.supportsFloatingPointSqrt()) {
nonIntExponent.link(&jit);
jit.loadDouble(&negativeHalfConstant, SpecializedThunkJIT::fpRegT3);
jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::regT0);
jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleLessThanOrEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::fpRegT3));
jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
jit.divDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
SpecializedThunkJIT::JumpList doubleResult;
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT1);
} else
jit.appendFailure(nonIntExponent);
return jit.finalize(*globalData, globalData->jitStubs->ctiNativeCall(), "pow");
}
MacroAssemblerCodeRef roundThunkGenerator(JSGlobalData* globalData)
{
SpecializedThunkJIT jit(1);
if (!UnaryDoubleOpWrapper(jsRound) || !jit.supportsFloatingPoint())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(globalData->jitStubs->ctiNativeCall());
MacroAssembler::Jump nonIntJump;
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
jit.returnInt32(SpecializedThunkJIT::regT0);
nonIntJump.link(&jit);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
SpecializedThunkJIT::Jump intResult;
SpecializedThunkJIT::JumpList doubleResult;
if (jit.supportsFloatingPointTruncate()) {
jit.loadDouble(&zeroConstant, SpecializedThunkJIT::fpRegT1);
doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
SpecializedThunkJIT::JumpList slowPath;
// Handle the negative doubles in the slow path for now.
slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
jit.loadDouble(&halfConstant, SpecializedThunkJIT::fpRegT1);
jit.addDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0));
intResult = jit.jump();
slowPath.link(&jit);
}
jit.callDoubleToDouble(UnaryDoubleOpWrapper(jsRound));
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
if (jit.supportsFloatingPointTruncate())
intResult.link(&jit);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT0);
return jit.finalize(*globalData, globalData->jitStubs->ctiNativeCall(), "round");
}
void AssemblyHelpers::purifyNaN(FPRReg fpr)
{
MacroAssembler::Jump notNaN = branchDouble(DoubleEqual, fpr, fpr);
static const double NaN = PNaN;
loadDouble(TrustedImmPtr(&NaN), fpr);
notNaN.link(this);
}
MacroAssemblerCodeRef ceilThunkGenerator(JSGlobalData* globalData)
{
SpecializedThunkJIT jit(1);
if (!UnaryDoubleOpWrapper(ceil) || !jit.supportsFloatingPoint())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(globalData->jitStubs->ctiNativeCall());
MacroAssembler::Jump nonIntJump;
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
jit.returnInt32(SpecializedThunkJIT::regT0);
nonIntJump.link(&jit);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
jit.callDoubleToDouble(UnaryDoubleOpWrapper(ceil));
SpecializedThunkJIT::JumpList doubleResult;
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT0);
return jit.finalize(*globalData, globalData->jitStubs->ctiNativeCall(), "ceil");
}
MacroAssemblerCodeRef absThunkGenerator(JSGlobalData* globalData)
{
SpecializedThunkJIT jit(1);
if (!jit.supportsFloatingPointAbs())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(globalData->jitStubs->ctiNativeCall());
MacroAssembler::Jump nonIntJump;
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
jit.rshift32(SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(31), SpecializedThunkJIT::regT1);
jit.add32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
jit.xor32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0);
jit.appendFailure(jit.branch32(MacroAssembler::Equal, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1 << 31)));
jit.returnInt32(SpecializedThunkJIT::regT0);
nonIntJump.link(&jit);
// Shame about the double int conversion here.
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
jit.absDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1);
jit.returnDouble(SpecializedThunkJIT::fpRegT1);
return jit.finalize(*globalData, globalData->jitStubs->ctiNativeCall(), "abs");
}
MacroAssemblerCodeRef osrEntryThunkGenerator(VM* vm)
{
AssemblyHelpers jit(nullptr);
// We get passed the address of a scratch buffer. The first 8-byte slot of the buffer
// is the frame size. The second 8-byte slot is the pointer to where we are supposed to
// jump. The remaining bytes are the new call frame header followed by the locals.
ptrdiff_t offsetOfFrameSize = 0; // This is the DFG frame count.
ptrdiff_t offsetOfTargetPC = offsetOfFrameSize + sizeof(EncodedJSValue);
ptrdiff_t offsetOfPayload = offsetOfTargetPC + sizeof(EncodedJSValue);
ptrdiff_t offsetOfLocals = offsetOfPayload + sizeof(Register) * CallFrame::headerSizeInRegisters;
jit.move(GPRInfo::returnValueGPR2, GPRInfo::regT0);
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfFrameSize), GPRInfo::regT1); // Load the frame size.
jit.move(GPRInfo::regT1, GPRInfo::regT2);
jit.lshiftPtr(MacroAssembler::Imm32(3), GPRInfo::regT2);
jit.move(GPRInfo::callFrameRegister, MacroAssembler::stackPointerRegister);
jit.subPtr(GPRInfo::regT2, MacroAssembler::stackPointerRegister);
MacroAssembler::Label loop = jit.label();
jit.subPtr(MacroAssembler::TrustedImm32(1), GPRInfo::regT1);
jit.move(GPRInfo::regT1, GPRInfo::regT4);
jit.negPtr(GPRInfo::regT4);
jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals), GPRInfo::regT2);
jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals + sizeof(int32_t)), GPRInfo::regT3);
jit.store32(GPRInfo::regT2, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register))));
jit.store32(GPRInfo::regT3, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register)) + static_cast<intptr_t>(sizeof(int32_t))));
jit.branchPtr(MacroAssembler::NotEqual, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(-static_cast<intptr_t>(CallFrame::headerSizeInRegisters)))).linkTo(loop, &jit);
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfTargetPC), GPRInfo::regT1);
MacroAssembler::Jump ok = jit.branchPtr(MacroAssembler::Above, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(static_cast<intptr_t>(1000))));
jit.abortWithReason(DFGUnreasonableOSREntryJumpDestination);
ok.link(&jit);
jit.restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(*vm);
jit.emitMaterializeTagCheckRegisters();
jit.jump(GPRInfo::regT1);
LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("DFG OSR entry thunk"));
}
MacroAssemblerCodeRef clz32ThunkGenerator(VM* vm)
{
SpecializedThunkJIT jit(vm, 1);
MacroAssembler::Jump nonIntArgJump;
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntArgJump);
SpecializedThunkJIT::Label convertedArgumentReentry(&jit);
jit.countLeadingZeros32(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1);
jit.returnInt32(SpecializedThunkJIT::regT1);
if (jit.supportsFloatingPointTruncate()) {
nonIntArgJump.link(&jit);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(convertedArgumentReentry, &jit);
jit.appendFailure(jit.jump());
} else
jit.appendFailure(nonIntArgJump);
return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "clz32");
}
MacroAssemblerCodeRef floorThunkGenerator(VM* vm)
{
SpecializedThunkJIT jit(vm, 1);
MacroAssembler::Jump nonIntJump;
if (!UnaryDoubleOpWrapper(floor) || !jit.supportsFloatingPoint())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
jit.returnInt32(SpecializedThunkJIT::regT0);
nonIntJump.link(&jit);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
#if CPU(ARM64)
SpecializedThunkJIT::JumpList doubleResult;
jit.floorDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT0);
#else
SpecializedThunkJIT::Jump intResult;
SpecializedThunkJIT::JumpList doubleResult;
if (jit.supportsFloatingPointTruncate()) {
jit.loadDouble(&zeroConstant, SpecializedThunkJIT::fpRegT1);
doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
SpecializedThunkJIT::JumpList slowPath;
// Handle the negative doubles in the slow path for now.
slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1));
slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0));
intResult = jit.jump();
slowPath.link(&jit);
}
jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(floor));
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
if (jit.supportsFloatingPointTruncate())
intResult.link(&jit);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT0);
#endif // CPU(ARM64)
return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "floor");
}
MacroAssemblerCodeRef ceilThunkGenerator(VM* vm)
{
SpecializedThunkJIT jit(vm, 1);
if (!UnaryDoubleOpWrapper(ceil) || !jit.supportsFloatingPoint())
return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm));
MacroAssembler::Jump nonIntJump;
jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump);
jit.returnInt32(SpecializedThunkJIT::regT0);
nonIntJump.link(&jit);
jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0);
#if CPU(ARM64)
jit.ceilDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0);
#else
jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(ceil));
#endif // CPU(ARM64)
SpecializedThunkJIT::JumpList doubleResult;
jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1);
jit.returnInt32(SpecializedThunkJIT::regT0);
doubleResult.link(&jit);
jit.returnDouble(SpecializedThunkJIT::fpRegT0);
return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "ceil");
}
MacroAssemblerCodeRef jitWriteThunkGenerator(void* writableAddr, void* stubBase, size_t stubSize)
{
using namespace ARM64Registers;
using TrustedImm32 = MacroAssembler::TrustedImm32;
MacroAssembler jit;
jit.move(MacroAssembler::TrustedImmPtr(writableAddr), x7);
jit.addPtr(x7, x0);
jit.move(x0, x3);
MacroAssembler::Jump smallCopy = jit.branch64(MacroAssembler::Below, x2, MacroAssembler::TrustedImm64(64));
jit.add64(TrustedImm32(32), x3);
jit.and64(TrustedImm32(-32), x3);
jit.loadPair64(x1, x12, x13);
jit.loadPair64(x1, TrustedImm32(16), x14, x15);
jit.sub64(x3, x0, x5);
jit.addPtr(x5, x1);
jit.loadPair64(x1, x8, x9);
jit.loadPair64(x1, TrustedImm32(16), x10, x11);
jit.add64(TrustedImm32(32), x1);
jit.sub64(x5, x2);
jit.storePair64(x12, x13, x0);
jit.storePair64(x14, x15, x0, TrustedImm32(16));
MacroAssembler::Jump cleanup = jit.branchSub64(MacroAssembler::BelowOrEqual, TrustedImm32(64), x2);
MacroAssembler::Label copyLoop = jit.label();
jit.storePair64WithNonTemporalAccess(x8, x9, x3);
jit.storePair64WithNonTemporalAccess(x10, x11, x3, TrustedImm32(16));
jit.add64(TrustedImm32(32), x3);
jit.loadPair64WithNonTemporalAccess(x1, x8, x9);
jit.loadPair64WithNonTemporalAccess(x1, TrustedImm32(16), x10, x11);
jit.add64(TrustedImm32(32), x1);
jit.branchSub64(MacroAssembler::Above, TrustedImm32(32), x2).linkTo(copyLoop, &jit);
cleanup.link(&jit);
jit.add64(x2, x1);
jit.loadPair64(x1, x12, x13);
jit.loadPair64(x1, TrustedImm32(16), x14, x15);
jit.storePair64(x8, x9, x3);
jit.storePair64(x10, x11, x3, TrustedImm32(16));
jit.addPtr(x2, x3);
jit.storePair64(x12, x13, x3, TrustedImm32(32));
jit.storePair64(x14, x15, x3, TrustedImm32(48));
jit.ret();
MacroAssembler::Label local0 = jit.label();
jit.load64(x1, PostIndex(8), x6);
jit.store64(x6, x3, PostIndex(8));
smallCopy.link(&jit);
jit.branchSub64(MacroAssembler::AboveOrEqual, TrustedImm32(8), x2).linkTo(local0, &jit);
MacroAssembler::Jump local2 = jit.branchAdd64(MacroAssembler::Equal, TrustedImm32(8), x2);
MacroAssembler::Label local1 = jit.label();
jit.load8(x1, PostIndex(1), x6);
jit.store8(x6, x3, PostIndex(1));
jit.branchSub64(MacroAssembler::NotEqual, TrustedImm32(1), x2).linkTo(local1, &jit);
local2.link(&jit);
jit.ret();
LinkBuffer linkBuffer(jit, stubBase, stubSize);
return FINALIZE_CODE(linkBuffer, ("Bulletproof JIT write thunk"));
}
static void compileStub(
unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
{
StackMaps::Record* record = nullptr;
for (unsigned i = jitCode->stackmaps.records.size(); i--;) {
record = &jitCode->stackmaps.records[i];
if (record->patchpointID == exit.m_stackmapID)
break;
}
RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID);
// This code requires framePointerRegister is the same as callFrameRegister
static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");
CCallHelpers jit(vm, codeBlock);
// We need scratch space to save all registers, to build up the JS stack, to deal with unwind
// fixup, pointers to all of the objects we materialize, and the elements inside those objects
// that we materialize.
// Figure out how much space we need for those object allocations.
unsigned numMaterializations = 0;
size_t maxMaterializationNumArguments = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
numMaterializations++;
maxMaterializationNumArguments = std::max(
maxMaterializationNumArguments,
materialization->properties().size());
}
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(
sizeof(EncodedJSValue) * (
exit.m_values.size() + numMaterializations + maxMaterializationNumArguments) +
requiredScratchMemorySizeInBytes() +
codeBlock->calleeSaveRegisters()->size() * sizeof(uint64_t));
EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
EncodedJSValue* materializationPointers = scratch + exit.m_values.size();
EncodedJSValue* materializationArguments = materializationPointers + numMaterializations;
char* registerScratch = bitwise_cast<char*>(materializationArguments + maxMaterializationNumArguments);
uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*> materializationToPointer;
unsigned materializationCount = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
materializationToPointer.add(
materialization, materializationPointers + materializationCount++);
}
// Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave().
// We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use
// that slot for saveAllRegisters().
saveAllRegisters(jit, registerScratch);
// Bring the stack back into a sane form and assert that it's sane.
jit.popToRestore(GPRInfo::regT0);
jit.checkStackPointerAlignment();
if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
Profiler::Database& database = *vm->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
// The remaining code assumes that SP/FP are in the same state that they were in the FTL's
// call frame.
// Get the call frame and tag thingies.
// Restore the exiting function's callFrame value into a regT4
jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
// Do some value profiling.
if (exit.m_profileDataFormat != DataFormatNone) {
record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
reboxAccordingToFormat(
exit.m_profileDataFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1);
jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
}
}
if (!!exit.m_valueProfile)
jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0));
}
// Materialize all objects. Don't materialize an object until all
// of the objects it needs have been materialized. We break cycles
// by populating objects late - we only consider an object as
// needing another object if the later is needed for the
// allocation of the former.
HashSet<ExitTimeObjectMaterialization*> toMaterialize;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations)
toMaterialize.add(materialization);
while (!toMaterialize.isEmpty()) {
unsigned previousToMaterializeSize = toMaterialize.size();
Vector<ExitTimeObjectMaterialization*> worklist;
worklist.appendRange(toMaterialize.begin(), toMaterialize.end());
for (ExitTimeObjectMaterialization* materialization : worklist) {
// Check if we can do anything about this right now.
bool allGood = true;
for (ExitPropertyValue value : materialization->properties()) {
if (!value.value().isObjectMaterialization())
continue;
if (!value.location().neededForMaterialization())
continue;
if (toMaterialize.contains(value.value().objectMaterialization())) {
// Gotta skip this one, since it needs a
// materialization that hasn't been materialized.
allGood = false;
break;
}
}
if (!allGood)
continue;
// All systems go for materializing the object. First we
// recover the values of all of its fields and then we
// call a function to actually allocate the beast.
// We only recover the fields that are needed for the allocation.
for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
const ExitPropertyValue& property = materialization->properties()[propertyIndex];
const ExitValue& value = property.value();
if (!property.location().neededForMaterialization())
continue;
compileRecovery(
jit, value, record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
}
// This call assumes that we don't pass arguments on the stack.
jit.setupArgumentsWithExecState(
CCallHelpers::TrustedImmPtr(materialization),
CCallHelpers::TrustedImmPtr(materializationArguments));
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationMaterializeObjectInOSR)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
jit.storePtr(GPRInfo::returnValueGPR, materializationToPointer.get(materialization));
// Let everyone know that we're done.
toMaterialize.remove(materialization);
}
// We expect progress! This ensures that we crash rather than looping infinitely if there
// is something broken about this fixpoint. Or, this could happen if we ever violate the
// "materializations form a DAG" rule.
RELEASE_ASSERT(toMaterialize.size() < previousToMaterializeSize);
}
// Now that all the objects have been allocated, we populate them
// with the correct values. This time we can recover all the
// fields, including those that are only needed for the allocation.
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
const ExitValue& value = materialization->properties()[propertyIndex].value();
compileRecovery(
jit, value, record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
}
// This call assumes that we don't pass arguments on the stack
jit.setupArgumentsWithExecState(
CCallHelpers::TrustedImmPtr(materialization),
CCallHelpers::TrustedImmPtr(materializationToPointer.get(materialization)),
CCallHelpers::TrustedImmPtr(materializationArguments));
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationPopulateObjectInOSR)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
}
// Save all state from wherever the exit data tells us it was, into the appropriate place in
// the scratch buffer. This also does the reboxing.
for (unsigned index = exit.m_values.size(); index--;) {
compileRecovery(
jit, exit.m_values[index], record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.store64(GPRInfo::regT0, scratch + index);
}
// Henceforth we make it look like the exiting function was called through a register
// preservation wrapper. This implies that FP must be nudged down by a certain amount. Then
// we restore the various things according to either exit.m_values or by copying from the
// old frame, and finally we save the various callee-save registers into where the
// restoration thunk would restore them from.
// Before we start messing with the frame, we need to set aside any registers that the
// FTL code was preserving.
for (unsigned i = codeBlock->calleeSaveRegisters()->size(); i--;) {
RegisterAtOffset entry = codeBlock->calleeSaveRegisters()->at(i);
jit.load64(
MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()),
GPRInfo::regT0);
jit.store64(GPRInfo::regT0, unwindScratch + i);
}
jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2);
// Let's say that the FTL function had failed its arity check. In that case, the stack will
// contain some extra stuff.
//
// We compute the padded stack space:
//
// paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1)
//
// The stack will have regT2 + CallFrameHeaderSize stuff.
// We want to make the stack look like this, from higher addresses down:
//
// - argument padding
// - actual arguments
// - call frame header
// This code assumes that we're dealing with FunctionCode.
RELEASE_ASSERT(codeBlock->codeType() == FunctionCode);
jit.add32(
MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2,
GPRInfo::regT3);
MacroAssembler::Jump arityIntact = jit.branch32(
MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0));
jit.neg32(GPRInfo::regT3);
jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3);
jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3);
jit.add32(GPRInfo::regT3, GPRInfo::regT2);
arityIntact.link(&jit);
CodeBlock* baselineCodeBlock = jit.baselineCodeBlockFor(exit.m_codeOrigin);
// First set up SP so that our data doesn't get clobbered by signals.
unsigned conservativeStackDelta =
(exit.m_values.numberOfLocals() + baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters()) * sizeof(Register) +
maxFrameExtentForSlowPathCall;
conservativeStackDelta = WTF::roundUpToMultipleOf(
stackAlignmentBytes(), conservativeStackDelta);
jit.addPtr(
MacroAssembler::TrustedImm32(-conservativeStackDelta),
MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
jit.checkStackPointerAlignment();
RegisterSet allFTLCalleeSaves = RegisterSet::ftlCalleeSaveRegisters();
RegisterAtOffsetList* baselineCalleeSaves = baselineCodeBlock->calleeSaveRegisters();
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
if (!allFTLCalleeSaves.get(reg))
continue;
unsigned unwindIndex = codeBlock->calleeSaveRegisters()->indexOf(reg);
RegisterAtOffset* baselineRegisterOffset = baselineCalleeSaves->find(reg);
if (reg.isGPR()) {
GPRReg regToLoad = baselineRegisterOffset ? GPRInfo::regT0 : reg.gpr();
if (unwindIndex == UINT_MAX) {
// The FTL compilation didn't preserve this register. This means that it also
// didn't use the register. So its value at the beginning of OSR exit should be
// preserved by the thunk. Luckily, we saved all registers into the register
// scratch buffer, so we can restore them from there.
jit.load64(registerScratch + offsetOfReg(reg), regToLoad);
} else {
// The FTL compilation preserved the register. Its new value is therefore
// irrelevant, but we can get the value that was preserved by using the unwind
// data. We've already copied all unwind-able preserved registers into the unwind
// scratch buffer, so we can get it from there.
jit.load64(unwindScratch + unwindIndex, regToLoad);
}
if (baselineRegisterOffset)
jit.store64(regToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset()));
} else {
FPRReg fpRegToLoad = baselineRegisterOffset ? FPRInfo::fpRegT0 : reg.fpr();
if (unwindIndex == UINT_MAX)
jit.loadDouble(MacroAssembler::TrustedImmPtr(registerScratch + offsetOfReg(reg)), fpRegToLoad);
else
jit.loadDouble(MacroAssembler::TrustedImmPtr(unwindScratch + unwindIndex), fpRegToLoad);
if (baselineRegisterOffset)
jit.storeDouble(fpRegToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset()));
}
}
size_t baselineVirtualRegistersForCalleeSaves = baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters();
// Now get state out of the scratch buffer and place it back into the stack. The values are
// already reboxed so we just move them.
for (unsigned index = exit.m_values.size(); index--;) {
VirtualRegister reg = exit.m_values.virtualRegisterForIndex(index);
if (reg.isLocal() && reg.toLocal() < static_cast<int>(baselineVirtualRegistersForCalleeSaves))
continue;
jit.load64(scratch + index, GPRInfo::regT0);
jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(reg));
}
handleExitCounts(jit, exit);
reifyInlinedCallFrames(jit, exit);
adjustAndJumpToTarget(jit, exit, false);
LinkBuffer patchBuffer(*vm, jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(),
patchBuffer,
("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s",
exitID, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(exit.m_values)).data(),
toCString(*record).data()));
}