MacroAssemblerCodePtr powThunkGenerator(JSGlobalData* globalData, ExecutablePool* pool) { #if USE(JSVALUE64) || USE(JSVALUE32_64) SpecializedThunkJIT jit(2, globalData, pool); if (!jit.supportsFloatingPoint()) return globalData->jitStubs->ctiNativeCall(); jit.loadDouble(&oneConstant, SpecializedThunkJIT::fpRegT1); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); MacroAssembler::Jump nonIntExponent; jit.loadInt32Argument(1, SpecializedThunkJIT::regT0, nonIntExponent); jit.appendFailure(jit.branch32(MacroAssembler::LessThan, SpecializedThunkJIT::regT0, MacroAssembler::Imm32(0))); MacroAssembler::Jump exponentIsZero = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0); MacroAssembler::Label startLoop(jit.label()); MacroAssembler::Jump exponentIsEven = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0, MacroAssembler::Imm32(1)); jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); exponentIsEven.link(&jit); jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); jit.rshift32(MacroAssembler::Imm32(1), SpecializedThunkJIT::regT0); jit.branchTest32(MacroAssembler::NonZero, SpecializedThunkJIT::regT0).linkTo(startLoop, &jit); exponentIsZero.link(&jit); { SpecializedThunkJIT::JumpList doubleResult; jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT1); } if (jit.supportsFloatingPointSqrt()) { nonIntExponent.link(&jit); jit.loadDouble(&negativeHalfConstant, SpecializedThunkJIT::fpRegT3); jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::regT0); jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleLessThanOrEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::fpRegT3)); jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); jit.divDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); SpecializedThunkJIT::JumpList doubleResult; jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT1); } else jit.appendFailure(nonIntExponent); return jit.finalize(globalData->jitStubs->ctiNativeCall()); #else UNUSED_PARAM(pool); return globalData->jitStubs->ctiNativeCall(); #endif }
MacroAssemblerCodeRef powThunkGenerator(JSGlobalData* globalData) { SpecializedThunkJIT jit(2); if (!jit.supportsFloatingPoint()) return MacroAssemblerCodeRef::createSelfManagedCodeRef(globalData->jitStubs->ctiNativeCall()); jit.loadDouble(&oneConstant, SpecializedThunkJIT::fpRegT1); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); MacroAssembler::Jump nonIntExponent; jit.loadInt32Argument(1, SpecializedThunkJIT::regT0, nonIntExponent); jit.appendFailure(jit.branch32(MacroAssembler::LessThan, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(0))); MacroAssembler::Jump exponentIsZero = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0); MacroAssembler::Label startLoop(jit.label()); MacroAssembler::Jump exponentIsEven = jit.branchTest32(MacroAssembler::Zero, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1)); jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); exponentIsEven.link(&jit); jit.mulDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); jit.rshift32(MacroAssembler::TrustedImm32(1), SpecializedThunkJIT::regT0); jit.branchTest32(MacroAssembler::NonZero, SpecializedThunkJIT::regT0).linkTo(startLoop, &jit); exponentIsZero.link(&jit); { SpecializedThunkJIT::JumpList doubleResult; jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT1); } if (jit.supportsFloatingPointSqrt()) { nonIntExponent.link(&jit); jit.loadDouble(&negativeHalfConstant, SpecializedThunkJIT::fpRegT3); jit.loadDoubleArgument(1, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::regT0); jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleLessThanOrEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); jit.appendFailure(jit.branchDouble(MacroAssembler::DoubleNotEqualOrUnordered, SpecializedThunkJIT::fpRegT2, SpecializedThunkJIT::fpRegT3)); jit.sqrtDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); jit.divDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); SpecializedThunkJIT::JumpList doubleResult; jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT0); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT1); } else jit.appendFailure(nonIntExponent); return jit.finalize(*globalData, globalData->jitStubs->ctiNativeCall(), "pow"); }
MacroAssemblerCodeRef roundThunkGenerator(JSGlobalData* globalData) { SpecializedThunkJIT jit(1); if (!UnaryDoubleOpWrapper(jsRound) || !jit.supportsFloatingPoint()) return MacroAssemblerCodeRef::createSelfManagedCodeRef(globalData->jitStubs->ctiNativeCall()); MacroAssembler::Jump nonIntJump; jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump); jit.returnInt32(SpecializedThunkJIT::regT0); nonIntJump.link(&jit); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); SpecializedThunkJIT::Jump intResult; SpecializedThunkJIT::JumpList doubleResult; if (jit.supportsFloatingPointTruncate()) { jit.loadDouble(&zeroConstant, SpecializedThunkJIT::fpRegT1); doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); SpecializedThunkJIT::JumpList slowPath; // Handle the negative doubles in the slow path for now. slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); jit.loadDouble(&halfConstant, SpecializedThunkJIT::fpRegT1); jit.addDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT1, SpecializedThunkJIT::regT0)); intResult = jit.jump(); slowPath.link(&jit); } jit.callDoubleToDouble(UnaryDoubleOpWrapper(jsRound)); jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1); if (jit.supportsFloatingPointTruncate()) intResult.link(&jit); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT0); return jit.finalize(*globalData, globalData->jitStubs->ctiNativeCall(), "round"); }
void AssemblyHelpers::purifyNaN(FPRReg fpr) { MacroAssembler::Jump notNaN = branchDouble(DoubleEqual, fpr, fpr); static const double NaN = PNaN; loadDouble(TrustedImmPtr(&NaN), fpr); notNaN.link(this); }
MacroAssemblerCodeRef ceilThunkGenerator(JSGlobalData* globalData) { SpecializedThunkJIT jit(1); if (!UnaryDoubleOpWrapper(ceil) || !jit.supportsFloatingPoint()) return MacroAssemblerCodeRef::createSelfManagedCodeRef(globalData->jitStubs->ctiNativeCall()); MacroAssembler::Jump nonIntJump; jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump); jit.returnInt32(SpecializedThunkJIT::regT0); nonIntJump.link(&jit); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); jit.callDoubleToDouble(UnaryDoubleOpWrapper(ceil)); SpecializedThunkJIT::JumpList doubleResult; jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT0); return jit.finalize(*globalData, globalData->jitStubs->ctiNativeCall(), "ceil"); }
MacroAssemblerCodeRef absThunkGenerator(JSGlobalData* globalData) { SpecializedThunkJIT jit(1); if (!jit.supportsFloatingPointAbs()) return MacroAssemblerCodeRef::createSelfManagedCodeRef(globalData->jitStubs->ctiNativeCall()); MacroAssembler::Jump nonIntJump; jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump); jit.rshift32(SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(31), SpecializedThunkJIT::regT1); jit.add32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0); jit.xor32(SpecializedThunkJIT::regT1, SpecializedThunkJIT::regT0); jit.appendFailure(jit.branch32(MacroAssembler::Equal, SpecializedThunkJIT::regT0, MacroAssembler::TrustedImm32(1 << 31))); jit.returnInt32(SpecializedThunkJIT::regT0); nonIntJump.link(&jit); // Shame about the double int conversion here. jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); jit.absDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1); jit.returnDouble(SpecializedThunkJIT::fpRegT1); return jit.finalize(*globalData, globalData->jitStubs->ctiNativeCall(), "abs"); }
MacroAssemblerCodeRef osrEntryThunkGenerator(VM* vm) { AssemblyHelpers jit(nullptr); // We get passed the address of a scratch buffer. The first 8-byte slot of the buffer // is the frame size. The second 8-byte slot is the pointer to where we are supposed to // jump. The remaining bytes are the new call frame header followed by the locals. ptrdiff_t offsetOfFrameSize = 0; // This is the DFG frame count. ptrdiff_t offsetOfTargetPC = offsetOfFrameSize + sizeof(EncodedJSValue); ptrdiff_t offsetOfPayload = offsetOfTargetPC + sizeof(EncodedJSValue); ptrdiff_t offsetOfLocals = offsetOfPayload + sizeof(Register) * CallFrame::headerSizeInRegisters; jit.move(GPRInfo::returnValueGPR2, GPRInfo::regT0); jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfFrameSize), GPRInfo::regT1); // Load the frame size. jit.move(GPRInfo::regT1, GPRInfo::regT2); jit.lshiftPtr(MacroAssembler::Imm32(3), GPRInfo::regT2); jit.move(GPRInfo::callFrameRegister, MacroAssembler::stackPointerRegister); jit.subPtr(GPRInfo::regT2, MacroAssembler::stackPointerRegister); MacroAssembler::Label loop = jit.label(); jit.subPtr(MacroAssembler::TrustedImm32(1), GPRInfo::regT1); jit.move(GPRInfo::regT1, GPRInfo::regT4); jit.negPtr(GPRInfo::regT4); jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals), GPRInfo::regT2); jit.load32(MacroAssembler::BaseIndex(GPRInfo::regT0, GPRInfo::regT1, MacroAssembler::TimesEight, offsetOfLocals + sizeof(int32_t)), GPRInfo::regT3); jit.store32(GPRInfo::regT2, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register)))); jit.store32(GPRInfo::regT3, MacroAssembler::BaseIndex(GPRInfo::callFrameRegister, GPRInfo::regT4, MacroAssembler::TimesEight, -static_cast<intptr_t>(sizeof(Register)) + static_cast<intptr_t>(sizeof(int32_t)))); jit.branchPtr(MacroAssembler::NotEqual, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(-static_cast<intptr_t>(CallFrame::headerSizeInRegisters)))).linkTo(loop, &jit); jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, offsetOfTargetPC), GPRInfo::regT1); MacroAssembler::Jump ok = jit.branchPtr(MacroAssembler::Above, GPRInfo::regT1, MacroAssembler::TrustedImmPtr(bitwise_cast<void*>(static_cast<intptr_t>(1000)))); jit.abortWithReason(DFGUnreasonableOSREntryJumpDestination); ok.link(&jit); jit.restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer(*vm); jit.emitMaterializeTagCheckRegisters(); jit.jump(GPRInfo::regT1); LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID); return FINALIZE_CODE(patchBuffer, ("DFG OSR entry thunk")); }
MacroAssemblerCodeRef clz32ThunkGenerator(VM* vm) { SpecializedThunkJIT jit(vm, 1); MacroAssembler::Jump nonIntArgJump; jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntArgJump); SpecializedThunkJIT::Label convertedArgumentReentry(&jit); jit.countLeadingZeros32(SpecializedThunkJIT::regT0, SpecializedThunkJIT::regT1); jit.returnInt32(SpecializedThunkJIT::regT1); if (jit.supportsFloatingPointTruncate()) { nonIntArgJump.link(&jit); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, SpecializedThunkJIT::BranchIfTruncateSuccessful).linkTo(convertedArgumentReentry, &jit); jit.appendFailure(jit.jump()); } else jit.appendFailure(nonIntArgJump); return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "clz32"); }
MacroAssemblerCodeRef floorThunkGenerator(VM* vm) { SpecializedThunkJIT jit(vm, 1); MacroAssembler::Jump nonIntJump; if (!UnaryDoubleOpWrapper(floor) || !jit.supportsFloatingPoint()) return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm)); jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump); jit.returnInt32(SpecializedThunkJIT::regT0); nonIntJump.link(&jit); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); #if CPU(ARM64) SpecializedThunkJIT::JumpList doubleResult; jit.floorDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT0); #else SpecializedThunkJIT::Jump intResult; SpecializedThunkJIT::JumpList doubleResult; if (jit.supportsFloatingPointTruncate()) { jit.loadDouble(&zeroConstant, SpecializedThunkJIT::fpRegT1); doubleResult.append(jit.branchDouble(MacroAssembler::DoubleEqual, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); SpecializedThunkJIT::JumpList slowPath; // Handle the negative doubles in the slow path for now. slowPath.append(jit.branchDouble(MacroAssembler::DoubleLessThanOrUnordered, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT1)); slowPath.append(jit.branchTruncateDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0)); intResult = jit.jump(); slowPath.link(&jit); } jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(floor)); jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1); if (jit.supportsFloatingPointTruncate()) intResult.link(&jit); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT0); #endif // CPU(ARM64) return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "floor"); }
MacroAssemblerCodeRef ceilThunkGenerator(VM* vm) { SpecializedThunkJIT jit(vm, 1); if (!UnaryDoubleOpWrapper(ceil) || !jit.supportsFloatingPoint()) return MacroAssemblerCodeRef::createSelfManagedCodeRef(vm->jitStubs->ctiNativeCall(vm)); MacroAssembler::Jump nonIntJump; jit.loadInt32Argument(0, SpecializedThunkJIT::regT0, nonIntJump); jit.returnInt32(SpecializedThunkJIT::regT0); nonIntJump.link(&jit); jit.loadDoubleArgument(0, SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0); #if CPU(ARM64) jit.ceilDouble(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::fpRegT0); #else jit.callDoubleToDoublePreservingReturn(UnaryDoubleOpWrapper(ceil)); #endif // CPU(ARM64) SpecializedThunkJIT::JumpList doubleResult; jit.branchConvertDoubleToInt32(SpecializedThunkJIT::fpRegT0, SpecializedThunkJIT::regT0, doubleResult, SpecializedThunkJIT::fpRegT1); jit.returnInt32(SpecializedThunkJIT::regT0); doubleResult.link(&jit); jit.returnDouble(SpecializedThunkJIT::fpRegT0); return jit.finalize(vm->jitStubs->ctiNativeTailCall(vm), "ceil"); }
MacroAssemblerCodeRef jitWriteThunkGenerator(void* writableAddr, void* stubBase, size_t stubSize) { using namespace ARM64Registers; using TrustedImm32 = MacroAssembler::TrustedImm32; MacroAssembler jit; jit.move(MacroAssembler::TrustedImmPtr(writableAddr), x7); jit.addPtr(x7, x0); jit.move(x0, x3); MacroAssembler::Jump smallCopy = jit.branch64(MacroAssembler::Below, x2, MacroAssembler::TrustedImm64(64)); jit.add64(TrustedImm32(32), x3); jit.and64(TrustedImm32(-32), x3); jit.loadPair64(x1, x12, x13); jit.loadPair64(x1, TrustedImm32(16), x14, x15); jit.sub64(x3, x0, x5); jit.addPtr(x5, x1); jit.loadPair64(x1, x8, x9); jit.loadPair64(x1, TrustedImm32(16), x10, x11); jit.add64(TrustedImm32(32), x1); jit.sub64(x5, x2); jit.storePair64(x12, x13, x0); jit.storePair64(x14, x15, x0, TrustedImm32(16)); MacroAssembler::Jump cleanup = jit.branchSub64(MacroAssembler::BelowOrEqual, TrustedImm32(64), x2); MacroAssembler::Label copyLoop = jit.label(); jit.storePair64WithNonTemporalAccess(x8, x9, x3); jit.storePair64WithNonTemporalAccess(x10, x11, x3, TrustedImm32(16)); jit.add64(TrustedImm32(32), x3); jit.loadPair64WithNonTemporalAccess(x1, x8, x9); jit.loadPair64WithNonTemporalAccess(x1, TrustedImm32(16), x10, x11); jit.add64(TrustedImm32(32), x1); jit.branchSub64(MacroAssembler::Above, TrustedImm32(32), x2).linkTo(copyLoop, &jit); cleanup.link(&jit); jit.add64(x2, x1); jit.loadPair64(x1, x12, x13); jit.loadPair64(x1, TrustedImm32(16), x14, x15); jit.storePair64(x8, x9, x3); jit.storePair64(x10, x11, x3, TrustedImm32(16)); jit.addPtr(x2, x3); jit.storePair64(x12, x13, x3, TrustedImm32(32)); jit.storePair64(x14, x15, x3, TrustedImm32(48)); jit.ret(); MacroAssembler::Label local0 = jit.label(); jit.load64(x1, PostIndex(8), x6); jit.store64(x6, x3, PostIndex(8)); smallCopy.link(&jit); jit.branchSub64(MacroAssembler::AboveOrEqual, TrustedImm32(8), x2).linkTo(local0, &jit); MacroAssembler::Jump local2 = jit.branchAdd64(MacroAssembler::Equal, TrustedImm32(8), x2); MacroAssembler::Label local1 = jit.label(); jit.load8(x1, PostIndex(1), x6); jit.store8(x6, x3, PostIndex(1)); jit.branchSub64(MacroAssembler::NotEqual, TrustedImm32(1), x2).linkTo(local1, &jit); local2.link(&jit); jit.ret(); LinkBuffer linkBuffer(jit, stubBase, stubSize); return FINALIZE_CODE(linkBuffer, ("Bulletproof JIT write thunk")); }
static void compileStub( unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock) { StackMaps::Record* record = nullptr; for (unsigned i = jitCode->stackmaps.records.size(); i--;) { record = &jitCode->stackmaps.records[i]; if (record->patchpointID == exit.m_stackmapID) break; } RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID); // This code requires framePointerRegister is the same as callFrameRegister static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same"); CCallHelpers jit(vm, codeBlock); // We need scratch space to save all registers, to build up the JS stack, to deal with unwind // fixup, pointers to all of the objects we materialize, and the elements inside those objects // that we materialize. // Figure out how much space we need for those object allocations. unsigned numMaterializations = 0; size_t maxMaterializationNumArguments = 0; for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) { numMaterializations++; maxMaterializationNumArguments = std::max( maxMaterializationNumArguments, materialization->properties().size()); } ScratchBuffer* scratchBuffer = vm->scratchBufferForSize( sizeof(EncodedJSValue) * ( exit.m_values.size() + numMaterializations + maxMaterializationNumArguments) + requiredScratchMemorySizeInBytes() + codeBlock->calleeSaveRegisters()->size() * sizeof(uint64_t)); EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0; EncodedJSValue* materializationPointers = scratch + exit.m_values.size(); EncodedJSValue* materializationArguments = materializationPointers + numMaterializations; char* registerScratch = bitwise_cast<char*>(materializationArguments + maxMaterializationNumArguments); uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes()); HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*> materializationToPointer; unsigned materializationCount = 0; for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) { materializationToPointer.add( materialization, materializationPointers + materializationCount++); } // Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave(). // We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use // that slot for saveAllRegisters(). saveAllRegisters(jit, registerScratch); // Bring the stack back into a sane form and assert that it's sane. jit.popToRestore(GPRInfo::regT0); jit.checkStackPointerAlignment(); if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) { Profiler::Database& database = *vm->m_perBytecodeProfiler; Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get(); Profiler::OSRExit* profilerExit = compilation->addOSRExit( exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin), exit.m_kind, exit.m_kind == UncountableInvalidation); jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress())); } // The remaining code assumes that SP/FP are in the same state that they were in the FTL's // call frame. // Get the call frame and tag thingies. // Restore the exiting function's callFrame value into a regT4 jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister); jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister); // Do some value profiling. if (exit.m_profileDataFormat != DataFormatNone) { record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0); reboxAccordingToFormat( exit.m_profileDataFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2); if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) { CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile; if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) { jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1); jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID()); jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1); jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2); jit.lshift32(GPRInfo::regT1, GPRInfo::regT2); jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes())); } } if (!!exit.m_valueProfile) jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0)); } // Materialize all objects. Don't materialize an object until all // of the objects it needs have been materialized. We break cycles // by populating objects late - we only consider an object as // needing another object if the later is needed for the // allocation of the former. HashSet<ExitTimeObjectMaterialization*> toMaterialize; for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) toMaterialize.add(materialization); while (!toMaterialize.isEmpty()) { unsigned previousToMaterializeSize = toMaterialize.size(); Vector<ExitTimeObjectMaterialization*> worklist; worklist.appendRange(toMaterialize.begin(), toMaterialize.end()); for (ExitTimeObjectMaterialization* materialization : worklist) { // Check if we can do anything about this right now. bool allGood = true; for (ExitPropertyValue value : materialization->properties()) { if (!value.value().isObjectMaterialization()) continue; if (!value.location().neededForMaterialization()) continue; if (toMaterialize.contains(value.value().objectMaterialization())) { // Gotta skip this one, since it needs a // materialization that hasn't been materialized. allGood = false; break; } } if (!allGood) continue; // All systems go for materializing the object. First we // recover the values of all of its fields and then we // call a function to actually allocate the beast. // We only recover the fields that are needed for the allocation. for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) { const ExitPropertyValue& property = materialization->properties()[propertyIndex]; const ExitValue& value = property.value(); if (!property.location().neededForMaterialization()) continue; compileRecovery( jit, value, record, jitCode->stackmaps, registerScratch, materializationToPointer); jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex); } // This call assumes that we don't pass arguments on the stack. jit.setupArgumentsWithExecState( CCallHelpers::TrustedImmPtr(materialization), CCallHelpers::TrustedImmPtr(materializationArguments)); jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationMaterializeObjectInOSR)), GPRInfo::nonArgGPR0); jit.call(GPRInfo::nonArgGPR0); jit.storePtr(GPRInfo::returnValueGPR, materializationToPointer.get(materialization)); // Let everyone know that we're done. toMaterialize.remove(materialization); } // We expect progress! This ensures that we crash rather than looping infinitely if there // is something broken about this fixpoint. Or, this could happen if we ever violate the // "materializations form a DAG" rule. RELEASE_ASSERT(toMaterialize.size() < previousToMaterializeSize); } // Now that all the objects have been allocated, we populate them // with the correct values. This time we can recover all the // fields, including those that are only needed for the allocation. for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) { for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) { const ExitValue& value = materialization->properties()[propertyIndex].value(); compileRecovery( jit, value, record, jitCode->stackmaps, registerScratch, materializationToPointer); jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex); } // This call assumes that we don't pass arguments on the stack jit.setupArgumentsWithExecState( CCallHelpers::TrustedImmPtr(materialization), CCallHelpers::TrustedImmPtr(materializationToPointer.get(materialization)), CCallHelpers::TrustedImmPtr(materializationArguments)); jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationPopulateObjectInOSR)), GPRInfo::nonArgGPR0); jit.call(GPRInfo::nonArgGPR0); } // Save all state from wherever the exit data tells us it was, into the appropriate place in // the scratch buffer. This also does the reboxing. for (unsigned index = exit.m_values.size(); index--;) { compileRecovery( jit, exit.m_values[index], record, jitCode->stackmaps, registerScratch, materializationToPointer); jit.store64(GPRInfo::regT0, scratch + index); } // Henceforth we make it look like the exiting function was called through a register // preservation wrapper. This implies that FP must be nudged down by a certain amount. Then // we restore the various things according to either exit.m_values or by copying from the // old frame, and finally we save the various callee-save registers into where the // restoration thunk would restore them from. // Before we start messing with the frame, we need to set aside any registers that the // FTL code was preserving. for (unsigned i = codeBlock->calleeSaveRegisters()->size(); i--;) { RegisterAtOffset entry = codeBlock->calleeSaveRegisters()->at(i); jit.load64( MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()), GPRInfo::regT0); jit.store64(GPRInfo::regT0, unwindScratch + i); } jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2); // Let's say that the FTL function had failed its arity check. In that case, the stack will // contain some extra stuff. // // We compute the padded stack space: // // paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1) // // The stack will have regT2 + CallFrameHeaderSize stuff. // We want to make the stack look like this, from higher addresses down: // // - argument padding // - actual arguments // - call frame header // This code assumes that we're dealing with FunctionCode. RELEASE_ASSERT(codeBlock->codeType() == FunctionCode); jit.add32( MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2, GPRInfo::regT3); MacroAssembler::Jump arityIntact = jit.branch32( MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0)); jit.neg32(GPRInfo::regT3); jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3); jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3); jit.add32(GPRInfo::regT3, GPRInfo::regT2); arityIntact.link(&jit); CodeBlock* baselineCodeBlock = jit.baselineCodeBlockFor(exit.m_codeOrigin); // First set up SP so that our data doesn't get clobbered by signals. unsigned conservativeStackDelta = (exit.m_values.numberOfLocals() + baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters()) * sizeof(Register) + maxFrameExtentForSlowPathCall; conservativeStackDelta = WTF::roundUpToMultipleOf( stackAlignmentBytes(), conservativeStackDelta); jit.addPtr( MacroAssembler::TrustedImm32(-conservativeStackDelta), MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister); jit.checkStackPointerAlignment(); RegisterSet allFTLCalleeSaves = RegisterSet::ftlCalleeSaveRegisters(); RegisterAtOffsetList* baselineCalleeSaves = baselineCodeBlock->calleeSaveRegisters(); for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) { if (!allFTLCalleeSaves.get(reg)) continue; unsigned unwindIndex = codeBlock->calleeSaveRegisters()->indexOf(reg); RegisterAtOffset* baselineRegisterOffset = baselineCalleeSaves->find(reg); if (reg.isGPR()) { GPRReg regToLoad = baselineRegisterOffset ? GPRInfo::regT0 : reg.gpr(); if (unwindIndex == UINT_MAX) { // The FTL compilation didn't preserve this register. This means that it also // didn't use the register. So its value at the beginning of OSR exit should be // preserved by the thunk. Luckily, we saved all registers into the register // scratch buffer, so we can restore them from there. jit.load64(registerScratch + offsetOfReg(reg), regToLoad); } else { // The FTL compilation preserved the register. Its new value is therefore // irrelevant, but we can get the value that was preserved by using the unwind // data. We've already copied all unwind-able preserved registers into the unwind // scratch buffer, so we can get it from there. jit.load64(unwindScratch + unwindIndex, regToLoad); } if (baselineRegisterOffset) jit.store64(regToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset())); } else { FPRReg fpRegToLoad = baselineRegisterOffset ? FPRInfo::fpRegT0 : reg.fpr(); if (unwindIndex == UINT_MAX) jit.loadDouble(MacroAssembler::TrustedImmPtr(registerScratch + offsetOfReg(reg)), fpRegToLoad); else jit.loadDouble(MacroAssembler::TrustedImmPtr(unwindScratch + unwindIndex), fpRegToLoad); if (baselineRegisterOffset) jit.storeDouble(fpRegToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset())); } } size_t baselineVirtualRegistersForCalleeSaves = baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters(); // Now get state out of the scratch buffer and place it back into the stack. The values are // already reboxed so we just move them. for (unsigned index = exit.m_values.size(); index--;) { VirtualRegister reg = exit.m_values.virtualRegisterForIndex(index); if (reg.isLocal() && reg.toLocal() < static_cast<int>(baselineVirtualRegistersForCalleeSaves)) continue; jit.load64(scratch + index, GPRInfo::regT0); jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(reg)); } handleExitCounts(jit, exit); reifyInlinedCallFrames(jit, exit); adjustAndJumpToTarget(jit, exit, false); LinkBuffer patchBuffer(*vm, jit, codeBlock); exit.m_code = FINALIZE_CODE_IF( shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(), patchBuffer, ("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s", exitID, toCString(exit.m_codeOrigin).data(), exitKindToString(exit.m_kind), toCString(*codeBlock).data(), toCString(ignoringContext<DumpContext>(exit.m_values)).data(), toCString(*record).data())); }