MacroAssemblerCodeRef osrExitGenerationThunkGenerator(VM* vm)
{
MacroAssembler jit;
// This needs to happen before we use the scratch buffer because this function also uses the scratch buffer.
adjustFrameAndStackInOSRExitCompilerThunk<DFG::JITCode>(jit, vm, JITCode::DFGJIT);
size_t scratchSize = sizeof(EncodedJSValue) * (GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters);
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(scratchSize);
EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.store64(GPRInfo::toRegister(i), buffer + i);
#else
jit.store32(GPRInfo::toRegister(i), buffer + i);
#endif
}
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.storeDouble(FPRInfo::toRegister(i), MacroAssembler::Address(GPRInfo::regT0));
}
// Tell GC mark phase how much of the scratch buffer is active during call.
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->addressOfActiveLength()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(scratchSize), MacroAssembler::Address(GPRInfo::regT0));
// Set up one argument.
#if CPU(X86)
jit.poke(GPRInfo::callFrameRegister, 0);
#else
jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
#endif
MacroAssembler::Call functionCall = jit.call();
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->addressOfActiveLength()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(0), MacroAssembler::Address(GPRInfo::regT0));
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.loadDouble(MacroAssembler::Address(GPRInfo::regT0), FPRInfo::toRegister(i));
}
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.load64(buffer + i, GPRInfo::toRegister(i));
#else
jit.load32(buffer + i, GPRInfo::toRegister(i));
#endif
}
jit.jump(MacroAssembler::AbsoluteAddress(&vm->osrExitJumpDestination));
LinkBuffer patchBuffer(jit, GLOBAL_THUNK_ID);
patchBuffer.link(functionCall, OSRExit::compileOSRExit);
return FINALIZE_CODE(patchBuffer, ("DFG OSR exit generation thunk"));
}
MacroAssemblerCodeRef osrExitGenerationThunkGenerator(JSGlobalData* globalData)
{
MacroAssembler jit;
size_t scratchSize = sizeof(EncodedJSValue) * (GPRInfo::numberOfRegisters + FPRInfo::numberOfRegisters);
ScratchBuffer* scratchBuffer = globalData->scratchBufferForSize(scratchSize);
EncodedJSValue* buffer = static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer());
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.store64(GPRInfo::toRegister(i), buffer + i);
#else
jit.store32(GPRInfo::toRegister(i), buffer + i);
#endif
}
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.storeDouble(FPRInfo::toRegister(i), GPRInfo::regT0);
}
// Tell GC mark phase how much of the scratch buffer is active during call.
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->activeLengthPtr()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(scratchSize), GPRInfo::regT0);
// Set up one argument.
#if CPU(X86)
jit.poke(GPRInfo::callFrameRegister, 0);
#else
jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
#endif
MacroAssembler::Call functionCall = jit.call();
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->activeLengthPtr()), GPRInfo::regT0);
jit.storePtr(MacroAssembler::TrustedImmPtr(0), GPRInfo::regT0);
for (unsigned i = 0; i < FPRInfo::numberOfRegisters; ++i) {
jit.move(MacroAssembler::TrustedImmPtr(buffer + GPRInfo::numberOfRegisters + i), GPRInfo::regT0);
jit.loadDouble(GPRInfo::regT0, FPRInfo::toRegister(i));
}
for (unsigned i = 0; i < GPRInfo::numberOfRegisters; ++i) {
#if USE(JSVALUE64)
jit.load64(buffer + i, GPRInfo::toRegister(i));
#else
jit.load32(buffer + i, GPRInfo::toRegister(i));
#endif
}
jit.jump(MacroAssembler::AbsoluteAddress(&globalData->osrExitJumpDestination));
LinkBuffer patchBuffer(*globalData, &jit, GLOBAL_THUNK_ID);
patchBuffer.link(functionCall, compileOSRExit);
return FINALIZE_CODE(patchBuffer, ("DFG OSR exit generation thunk"));
}
void VM::gatherConservativeRoots(ConservativeRoots& conservativeRoots)
{
for (size_t i = 0; i < scratchBuffers.size(); i++) {
ScratchBuffer* scratchBuffer = scratchBuffers[i];
if (scratchBuffer->activeLength()) {
void* bufferStart = scratchBuffer->dataBuffer();
conservativeRoots.add(bufferStart, static_cast<void*>(static_cast<char*>(bufferStart) + scratchBuffer->activeLength()));
}
}
}
void OSRExitCompiler::compileExit(const OSRExit& exit, const Operands<ValueRecovery>& operands, SpeculationRecovery* recovery)
{
// 1) Pro-forma stuff.
if (Options::printEachOSRExit()) {
SpeculationFailureDebugInfo* debugInfo = new SpeculationFailureDebugInfo;
debugInfo->codeBlock = m_jit.codeBlock();
debugInfo->kind = exit.m_kind;
debugInfo->bytecodeOffset = exit.m_codeOrigin.bytecodeIndex;
m_jit.debugCall(debugOperationPrintSpeculationFailure, debugInfo);
}
// Need to ensure that the stack pointer accounts for the worst-case stack usage at exit.
m_jit.addPtr(
CCallHelpers::TrustedImm32(
-m_jit.codeBlock()->jitCode()->dfgCommon()->requiredRegisterCountForExit * sizeof(Register)),
CCallHelpers::framePointerRegister, CCallHelpers::stackPointerRegister);
// 2) Perform speculation recovery. This only comes into play when an operation
// starts mutating state before verifying the speculation it has already made.
if (recovery) {
switch (recovery->type()) {
case SpeculativeAdd:
m_jit.sub32(recovery->src(), recovery->dest());
break;
case BooleanSpeculationCheck:
break;
default:
break;
}
}
// 3) Refine some value profile, if appropriate.
if (!!exit.m_jsValueSource) {
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
// If the instruction that this originated from has an array profile, then
// refine it. If it doesn't, then do nothing. The latter could happen for
// hoisted checks, or checks emitted for operations that didn't have array
// profiling - either ops that aren't array accesses at all, or weren't
// known to be array acceses in the bytecode. The latter case is a FIXME
// while the former case is an outcome of a CheckStructure not knowing why
// it was emitted (could be either due to an inline cache of a property
// property access, or due to an array profile).
// Note: We are free to assume that the jsValueSource is already known to
// be a cell since both BadCache and BadIndexingType exits occur after
// the cell check would have already happened.
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
if (ArrayProfile* arrayProfile = m_jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
GPRReg usedRegister1;
GPRReg usedRegister2;
if (exit.m_jsValueSource.isAddress()) {
usedRegister1 = exit.m_jsValueSource.base();
usedRegister2 = InvalidGPRReg;
} else {
usedRegister1 = exit.m_jsValueSource.payloadGPR();
if (exit.m_jsValueSource.hasKnownTag())
usedRegister2 = InvalidGPRReg;
else
usedRegister2 = exit.m_jsValueSource.tagGPR();
}
GPRReg scratch1;
GPRReg scratch2;
scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2);
scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister1, usedRegister2, scratch1);
#if CPU(ARM64)
m_jit.pushToSave(scratch1);
m_jit.pushToSave(scratch2);
#else
m_jit.push(scratch1);
m_jit.push(scratch2);
#endif
GPRReg value;
if (exit.m_jsValueSource.isAddress()) {
value = scratch1;
m_jit.loadPtr(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), value);
} else
value = exit.m_jsValueSource.payloadGPR();
m_jit.loadPtr(AssemblyHelpers::Address(value, JSCell::structureIDOffset()), scratch1);
m_jit.storePtr(scratch1, arrayProfile->addressOfLastSeenStructureID());
m_jit.load8(AssemblyHelpers::Address(scratch1, Structure::indexingTypeOffset()), scratch1);
m_jit.move(AssemblyHelpers::TrustedImm32(1), scratch2);
m_jit.lshift32(scratch1, scratch2);
m_jit.or32(scratch2, AssemblyHelpers::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
#if CPU(ARM64)
m_jit.popToRestore(scratch2);
m_jit.popToRestore(scratch1);
#else
m_jit.pop(scratch2);
m_jit.pop(scratch1);
#endif
}
}
if (!!exit.m_valueProfile) {
EncodedJSValue* bucket = exit.m_valueProfile.getSpecFailBucket(0);
if (exit.m_jsValueSource.isAddress()) {
// Save a register so we can use it.
GPRReg scratch = AssemblyHelpers::selectScratchGPR(exit.m_jsValueSource.base());
#if CPU(ARM64)
m_jit.pushToSave(scratch);
#else
m_jit.push(scratch);
#endif
m_jit.load32(exit.m_jsValueSource.asAddress(OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), scratch);
m_jit.store32(scratch, &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.tag);
m_jit.load32(exit.m_jsValueSource.asAddress(OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)), scratch);
m_jit.store32(scratch, &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.payload);
#if CPU(ARM64)
m_jit.popToRestore(scratch);
#else
m_jit.pop(scratch);
#endif
} else if (exit.m_jsValueSource.hasKnownTag()) {
m_jit.store32(AssemblyHelpers::TrustedImm32(exit.m_jsValueSource.tag()), &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.tag);
m_jit.store32(exit.m_jsValueSource.payloadGPR(), &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.payload);
} else {
m_jit.store32(exit.m_jsValueSource.tagGPR(), &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.tag);
m_jit.store32(exit.m_jsValueSource.payloadGPR(), &bitwise_cast<EncodedValueDescriptor*>(bucket)->asBits.payload);
}
}
}
// Do a simplified OSR exit. See DFGOSRExitCompiler64.cpp's comment regarding how and wny we
// do this simple approach.
// 4) Save all state from GPRs into the scratch buffer.
ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(sizeof(EncodedJSValue) * operands.size());
EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case UnboxedInt32InGPR:
case UnboxedBooleanInGPR:
case UnboxedCellInGPR:
m_jit.store32(
recovery.gpr(),
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload);
break;
case InPair:
m_jit.store32(
recovery.tagGPR(),
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag);
m_jit.store32(
recovery.payloadGPR(),
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload);
break;
default:
break;
}
}
// Now all GPRs are free to reuse.
// 5) Save all state from FPRs into the scratch buffer.
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case InFPR:
m_jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0);
m_jit.storeDouble(recovery.fpr(), MacroAssembler::Address(GPRInfo::regT0));
break;
default:
break;
}
}
// Now all FPRs are free to reuse.
// 6) Save all state from the stack into the scratch buffer. For simplicity we
// do this even for state that's already in the right place on the stack.
// It makes things simpler later.
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case DisplacedInJSStack:
case Int32DisplacedInJSStack:
case DoubleDisplacedInJSStack:
case CellDisplacedInJSStack:
case BooleanDisplacedInJSStack:
m_jit.load32(
AssemblyHelpers::tagFor(recovery.virtualRegister()),
GPRInfo::regT0);
m_jit.load32(
AssemblyHelpers::payloadFor(recovery.virtualRegister()),
GPRInfo::regT1);
m_jit.store32(
GPRInfo::regT0,
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag);
m_jit.store32(
GPRInfo::regT1,
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload);
break;
default:
break;
}
}
// 7) Do all data format conversions and store the results into the stack.
bool haveArguments = false;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
int operand = operands.operandForIndex(index);
switch (recovery.technique()) {
case InPair:
case DisplacedInJSStack:
m_jit.load32(
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.tag,
GPRInfo::regT0);
m_jit.load32(
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
GPRInfo::regT1);
m_jit.store32(
GPRInfo::regT0,
AssemblyHelpers::tagFor(operand));
m_jit.store32(
GPRInfo::regT1,
AssemblyHelpers::payloadFor(operand));
break;
case InFPR:
case DoubleDisplacedInJSStack:
m_jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0);
m_jit.loadDouble(GPRInfo::regT0, FPRInfo::fpRegT0);
m_jit.purifyNaN(FPRInfo::fpRegT0);
m_jit.storeDouble(FPRInfo::fpRegT0, AssemblyHelpers::addressFor(operand));
break;
case UnboxedInt32InGPR:
case Int32DisplacedInJSStack:
m_jit.load32(
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
GPRInfo::regT0);
m_jit.store32(
AssemblyHelpers::TrustedImm32(JSValue::Int32Tag),
AssemblyHelpers::tagFor(operand));
m_jit.store32(
GPRInfo::regT0,
AssemblyHelpers::payloadFor(operand));
break;
case UnboxedCellInGPR:
case CellDisplacedInJSStack:
m_jit.load32(
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
GPRInfo::regT0);
m_jit.store32(
AssemblyHelpers::TrustedImm32(JSValue::CellTag),
AssemblyHelpers::tagFor(operand));
m_jit.store32(
GPRInfo::regT0,
AssemblyHelpers::payloadFor(operand));
break;
case UnboxedBooleanInGPR:
case BooleanDisplacedInJSStack:
m_jit.load32(
&bitwise_cast<EncodedValueDescriptor*>(scratch + index)->asBits.payload,
GPRInfo::regT0);
m_jit.store32(
AssemblyHelpers::TrustedImm32(JSValue::BooleanTag),
AssemblyHelpers::tagFor(operand));
m_jit.store32(
GPRInfo::regT0,
AssemblyHelpers::payloadFor(operand));
break;
case Constant:
m_jit.store32(
AssemblyHelpers::TrustedImm32(recovery.constant().tag()),
AssemblyHelpers::tagFor(operand));
m_jit.store32(
AssemblyHelpers::TrustedImm32(recovery.constant().payload()),
AssemblyHelpers::payloadFor(operand));
break;
case ArgumentsThatWereNotCreated:
haveArguments = true;
m_jit.store32(
AssemblyHelpers::TrustedImm32(JSValue().tag()),
AssemblyHelpers::tagFor(operand));
m_jit.store32(
AssemblyHelpers::TrustedImm32(JSValue().payload()),
AssemblyHelpers::payloadFor(operand));
break;
default:
break;
}
}
// 8) Adjust the old JIT's execute counter. Since we are exiting OSR, we know
// that all new calls into this code will go to the new JIT, so the execute
// counter only affects call frames that performed OSR exit and call frames
// that were still executing the old JIT at the time of another call frame's
// OSR exit. We want to ensure that the following is true:
//
// (a) Code the performs an OSR exit gets a chance to reenter optimized
// code eventually, since optimized code is faster. But we don't
// want to do such reentery too aggressively (see (c) below).
//
// (b) If there is code on the call stack that is still running the old
// JIT's code and has never OSR'd, then it should get a chance to
// perform OSR entry despite the fact that we've exited.
//
// (c) Code the performs an OSR exit should not immediately retry OSR
// entry, since both forms of OSR are expensive. OSR entry is
// particularly expensive.
//
// (d) Frequent OSR failures, even those that do not result in the code
// running in a hot loop, result in recompilation getting triggered.
//
// To ensure (c), we'd like to set the execute counter to
// counterValueForOptimizeAfterWarmUp(). This seems like it would endanger
// (a) and (b), since then every OSR exit would delay the opportunity for
// every call frame to perform OSR entry. Essentially, if OSR exit happens
// frequently and the function has few loops, then the counter will never
// become non-negative and OSR entry will never be triggered. OSR entry
// will only happen if a loop gets hot in the old JIT, which does a pretty
// good job of ensuring (a) and (b). But that doesn't take care of (d),
// since each speculation failure would reset the execute counter.
// So we check here if the number of speculation failures is significantly
// larger than the number of successes (we want 90% success rate), and if
// there have been a large enough number of failures. If so, we set the
// counter to 0; otherwise we set the counter to
// counterValueForOptimizeAfterWarmUp().
handleExitCounts(m_jit, exit);
// 9) Reify inlined call frames.
reifyInlinedCallFrames(m_jit, exit);
// 10) Create arguments if necessary and place them into the appropriate aliased
// registers.
if (haveArguments) {
ArgumentsRecoveryGenerator argumentsRecovery;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
if (recovery.technique() != ArgumentsThatWereNotCreated)
continue;
argumentsRecovery.generateFor(
operands.operandForIndex(index), exit.m_codeOrigin, m_jit);
}
}
// 12) And finish.
adjustAndJumpToTarget(m_jit, exit);
}
void OSRExitCompiler::compileExit(const OSRExit& exit, const Operands<ValueRecovery>& operands, SpeculationRecovery* recovery)
{
// 1) Pro-forma stuff.
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLogF("OSR exit for (");
for (CodeOrigin codeOrigin = exit.m_codeOrigin; ; codeOrigin = codeOrigin.inlineCallFrame->caller) {
dataLogF("bc#%u", codeOrigin.bytecodeIndex);
if (!codeOrigin.inlineCallFrame)
break;
dataLogF(" -> %p ", codeOrigin.inlineCallFrame->executable.get());
}
dataLogF(") ");
dataLog(operands);
#endif
if (Options::printEachOSRExit()) {
SpeculationFailureDebugInfo* debugInfo = new SpeculationFailureDebugInfo;
debugInfo->codeBlock = m_jit.codeBlock();
m_jit.debugCall(debugOperationPrintSpeculationFailure, debugInfo);
}
#if DFG_ENABLE(JIT_BREAK_ON_SPECULATION_FAILURE)
m_jit.breakpoint();
#endif
#if DFG_ENABLE(SUCCESS_STATS)
static SamplingCounter counter("SpeculationFailure");
m_jit.emitCount(counter);
#endif
// 2) Perform speculation recovery. This only comes into play when an operation
// starts mutating state before verifying the speculation it has already made.
if (recovery) {
switch (recovery->type()) {
case SpeculativeAdd:
m_jit.sub32(recovery->src(), recovery->dest());
m_jit.or64(GPRInfo::tagTypeNumberRegister, recovery->dest());
break;
case BooleanSpeculationCheck:
m_jit.xor64(AssemblyHelpers::TrustedImm32(static_cast<int32_t>(ValueFalse)), recovery->dest());
break;
default:
break;
}
}
// 3) Refine some array and/or value profile, if appropriate.
if (!!exit.m_jsValueSource) {
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
// If the instruction that this originated from has an array profile, then
// refine it. If it doesn't, then do nothing. The latter could happen for
// hoisted checks, or checks emitted for operations that didn't have array
// profiling - either ops that aren't array accesses at all, or weren't
// known to be array acceses in the bytecode. The latter case is a FIXME
// while the former case is an outcome of a CheckStructure not knowing why
// it was emitted (could be either due to an inline cache of a property
// property access, or due to an array profile).
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
if (ArrayProfile* arrayProfile = m_jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
GPRReg usedRegister;
if (exit.m_jsValueSource.isAddress())
usedRegister = exit.m_jsValueSource.base();
else
usedRegister = exit.m_jsValueSource.gpr();
GPRReg scratch1;
GPRReg scratch2;
scratch1 = AssemblyHelpers::selectScratchGPR(usedRegister);
scratch2 = AssemblyHelpers::selectScratchGPR(usedRegister, scratch1);
#if CPU(ARM64)
m_jit.pushToSave(scratch1);
m_jit.pushToSave(scratch2);
#else
m_jit.push(scratch1);
m_jit.push(scratch2);
#endif
GPRReg value;
if (exit.m_jsValueSource.isAddress()) {
value = scratch1;
m_jit.loadPtr(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), value);
} else
value = exit.m_jsValueSource.gpr();
m_jit.loadPtr(AssemblyHelpers::Address(value, JSCell::structureOffset()), scratch1);
m_jit.storePtr(scratch1, arrayProfile->addressOfLastSeenStructure());
m_jit.load8(AssemblyHelpers::Address(scratch1, Structure::indexingTypeOffset()), scratch1);
m_jit.move(AssemblyHelpers::TrustedImm32(1), scratch2);
m_jit.lshift32(scratch1, scratch2);
m_jit.or32(scratch2, AssemblyHelpers::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
#if CPU(ARM64)
m_jit.popToRestore(scratch2);
m_jit.popToRestore(scratch1);
#else
m_jit.pop(scratch2);
m_jit.pop(scratch1);
#endif
}
}
if (!!exit.m_valueProfile) {
EncodedJSValue* bucket = exit.m_valueProfile.getSpecFailBucket(0);
if (exit.m_jsValueSource.isAddress()) {
// We can't be sure that we have a spare register. So use the tagTypeNumberRegister,
// since we know how to restore it.
m_jit.load64(AssemblyHelpers::Address(exit.m_jsValueSource.asAddress()), GPRInfo::tagTypeNumberRegister);
m_jit.store64(GPRInfo::tagTypeNumberRegister, bucket);
m_jit.move(AssemblyHelpers::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
} else
m_jit.store64(exit.m_jsValueSource.gpr(), bucket);
}
}
// What follows is an intentionally simple OSR exit implementation that generates
// fairly poor code but is very easy to hack. In particular, it dumps all state that
// needs conversion into a scratch buffer so that in step 6, where we actually do the
// conversions, we know that all temp registers are free to use and the variable is
// definitely in a well-known spot in the scratch buffer regardless of whether it had
// originally been in a register or spilled. This allows us to decouple "where was
// the variable" from "how was it represented". Consider that the
// Int32DisplacedInJSStack recovery: it tells us that the value is in a
// particular place and that that place holds an unboxed int32. We have two different
// places that a value could be (displaced, register) and a bunch of different
// ways of representing a value. The number of recoveries is two * a bunch. The code
// below means that we have to have two + a bunch cases rather than two * a bunch.
// Once we have loaded the value from wherever it was, the reboxing is the same
// regardless of its location. Likewise, before we do the reboxing, the way we get to
// the value (i.e. where we load it from) is the same regardless of its type. Because
// the code below always dumps everything into a scratch buffer first, the two
// questions become orthogonal, which simplifies adding new types and adding new
// locations.
//
// This raises the question: does using such a suboptimal implementation of OSR exit,
// where we always emit code to dump all state into a scratch buffer only to then
// dump it right back into the stack, hurt us in any way? The asnwer is that OSR exits
// are rare. Our tiering strategy ensures this. This is because if an OSR exit is
// taken more than ~100 times, we jettison the DFG code block along with all of its
// exits. It is impossible for an OSR exit - i.e. the code we compile below - to
// execute frequently enough for the codegen to matter that much. It probably matters
// enough that we don't want to turn this into some super-slow function call, but so
// long as we're generating straight-line code, that code can be pretty bad. Also
// because we tend to exit only along one OSR exit from any DFG code block - that's an
// empirical result that we're extremely confident about - the code size of this
// doesn't matter much. Hence any attempt to optimize the codegen here is just purely
// harmful to the system: it probably won't reduce either net memory usage or net
// execution time. It will only prevent us from cleanly decoupling "where was the
// variable" from "how was it represented", which will make it more difficult to add
// features in the future and it will make it harder to reason about bugs.
// 4) Save all state from GPRs into the scratch buffer.
ScratchBuffer* scratchBuffer = m_jit.vm()->scratchBufferForSize(sizeof(EncodedJSValue) * operands.size());
EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case InGPR:
case UnboxedInt32InGPR:
case UInt32InGPR:
case UnboxedInt52InGPR:
case UnboxedStrictInt52InGPR:
case UnboxedCellInGPR:
m_jit.store64(recovery.gpr(), scratch + index);
break;
default:
break;
}
}
// And voila, all GPRs are free to reuse.
// 5) Save all state from FPRs into the scratch buffer.
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case InFPR:
m_jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0);
m_jit.storeDouble(recovery.fpr(), GPRInfo::regT0);
break;
default:
break;
}
}
// Now, all FPRs are also free.
// 6) Save all state from the stack into the scratch buffer. For simplicity we
// do this even for state that's already in the right place on the stack.
// It makes things simpler later.
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
switch (recovery.technique()) {
case DisplacedInJSStack:
case CellDisplacedInJSStack:
case BooleanDisplacedInJSStack:
case Int32DisplacedInJSStack:
case DoubleDisplacedInJSStack:
case Int52DisplacedInJSStack:
case StrictInt52DisplacedInJSStack:
m_jit.load64(AssemblyHelpers::addressFor(recovery.virtualRegister()), GPRInfo::regT0);
m_jit.store64(GPRInfo::regT0, scratch + index);
break;
default:
break;
}
}
// 7) Do all data format conversions and store the results into the stack.
bool haveArguments = false;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
int operand = operands.operandForIndex(index);
switch (recovery.technique()) {
case InGPR:
case UnboxedCellInGPR:
case DisplacedInJSStack:
case CellDisplacedInJSStack:
case BooleanDisplacedInJSStack:
m_jit.load64(scratch + index, GPRInfo::regT0);
m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
break;
case UnboxedInt32InGPR:
case Int32DisplacedInJSStack:
m_jit.load64(scratch + index, GPRInfo::regT0);
m_jit.zeroExtend32ToPtr(GPRInfo::regT0, GPRInfo::regT0);
m_jit.or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
break;
case UnboxedInt52InGPR:
case Int52DisplacedInJSStack:
m_jit.load64(scratch + index, GPRInfo::regT0);
m_jit.rshift64(
AssemblyHelpers::TrustedImm32(JSValue::int52ShiftAmount), GPRInfo::regT0);
m_jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0);
m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
break;
case UnboxedStrictInt52InGPR:
case StrictInt52DisplacedInJSStack:
m_jit.load64(scratch + index, GPRInfo::regT0);
m_jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0);
m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
break;
case UInt32InGPR:
m_jit.load64(scratch + index, GPRInfo::regT0);
m_jit.zeroExtend32ToPtr(GPRInfo::regT0, GPRInfo::regT0);
m_jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0);
m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
break;
case InFPR:
case DoubleDisplacedInJSStack:
m_jit.move(AssemblyHelpers::TrustedImmPtr(scratch + index), GPRInfo::regT0);
m_jit.loadDouble(GPRInfo::regT0, FPRInfo::fpRegT0);
m_jit.boxDouble(FPRInfo::fpRegT0, GPRInfo::regT0);
m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
break;
case Constant:
m_jit.store64(
AssemblyHelpers::TrustedImm64(JSValue::encode(recovery.constant())),
AssemblyHelpers::addressFor(operand));
break;
case ArgumentsThatWereNotCreated:
haveArguments = true;
// We can't restore this yet but we can make sure that the stack appears
// sane.
m_jit.store64(
AssemblyHelpers::TrustedImm64(JSValue::encode(JSValue())),
AssemblyHelpers::addressFor(operand));
break;
default:
break;
}
}
// 8) Adjust the old JIT's execute counter. Since we are exiting OSR, we know
// that all new calls into this code will go to the new JIT, so the execute
// counter only affects call frames that performed OSR exit and call frames
// that were still executing the old JIT at the time of another call frame's
// OSR exit. We want to ensure that the following is true:
//
// (a) Code the performs an OSR exit gets a chance to reenter optimized
// code eventually, since optimized code is faster. But we don't
// want to do such reentery too aggressively (see (c) below).
//
// (b) If there is code on the call stack that is still running the old
// JIT's code and has never OSR'd, then it should get a chance to
// perform OSR entry despite the fact that we've exited.
//
// (c) Code the performs an OSR exit should not immediately retry OSR
// entry, since both forms of OSR are expensive. OSR entry is
// particularly expensive.
//
// (d) Frequent OSR failures, even those that do not result in the code
// running in a hot loop, result in recompilation getting triggered.
//
// To ensure (c), we'd like to set the execute counter to
// counterValueForOptimizeAfterWarmUp(). This seems like it would endanger
// (a) and (b), since then every OSR exit would delay the opportunity for
// every call frame to perform OSR entry. Essentially, if OSR exit happens
// frequently and the function has few loops, then the counter will never
// become non-negative and OSR entry will never be triggered. OSR entry
// will only happen if a loop gets hot in the old JIT, which does a pretty
// good job of ensuring (a) and (b). But that doesn't take care of (d),
// since each speculation failure would reset the execute counter.
// So we check here if the number of speculation failures is significantly
// larger than the number of successes (we want 90% success rate), and if
// there have been a large enough number of failures. If so, we set the
// counter to 0; otherwise we set the counter to
// counterValueForOptimizeAfterWarmUp().
handleExitCounts(m_jit, exit);
// 9) Reify inlined call frames.
reifyInlinedCallFrames(m_jit, exit);
// 10) Create arguments if necessary and place them into the appropriate aliased
// registers.
if (haveArguments) {
HashSet<InlineCallFrame*, DefaultHash<InlineCallFrame*>::Hash,
NullableHashTraits<InlineCallFrame*> > didCreateArgumentsObject;
for (size_t index = 0; index < operands.size(); ++index) {
const ValueRecovery& recovery = operands[index];
if (recovery.technique() != ArgumentsThatWereNotCreated)
continue;
int operand = operands.operandForIndex(index);
// Find the right inline call frame.
InlineCallFrame* inlineCallFrame = 0;
for (InlineCallFrame* current = exit.m_codeOrigin.inlineCallFrame;
current;
current = current->caller.inlineCallFrame) {
if (current->stackOffset >= operand) {
inlineCallFrame = current;
break;
}
}
if (!m_jit.baselineCodeBlockFor(inlineCallFrame)->usesArguments())
continue;
VirtualRegister argumentsRegister = m_jit.baselineArgumentsRegisterFor(inlineCallFrame);
if (didCreateArgumentsObject.add(inlineCallFrame).isNewEntry) {
// We know this call frame optimized out an arguments object that
// the baseline JIT would have created. Do that creation now.
if (inlineCallFrame) {
m_jit.addPtr(AssemblyHelpers::TrustedImm32(inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister, GPRInfo::regT0);
m_jit.setupArguments(GPRInfo::regT0);
} else
m_jit.setupArgumentsExecState();
m_jit.move(
AssemblyHelpers::TrustedImmPtr(
bitwise_cast<void*>(operationCreateArguments)),
GPRInfo::nonArgGPR0);
m_jit.call(GPRInfo::nonArgGPR0);
m_jit.store64(GPRInfo::returnValueGPR, AssemblyHelpers::addressFor(argumentsRegister));
m_jit.store64(
GPRInfo::returnValueGPR,
AssemblyHelpers::addressFor(unmodifiedArgumentsRegister(argumentsRegister)));
m_jit.move(GPRInfo::returnValueGPR, GPRInfo::regT0); // no-op move on almost all platforms.
}
m_jit.load64(AssemblyHelpers::addressFor(argumentsRegister), GPRInfo::regT0);
m_jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
}
}
// 11) Load the result of the last bytecode operation into regT0.
if (exit.m_lastSetOperand.isValid())
m_jit.load64(AssemblyHelpers::addressFor(exit.m_lastSetOperand), GPRInfo::cachedResultRegister);
// 12) And finish.
adjustAndJumpToTarget(m_jit, exit);
}
static void compileStubWithOSRExitStackmap(
unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
{
StackMaps::Record* record;
for (unsigned i = jitCode->stackmaps.records.size(); i--;) {
record = &jitCode->stackmaps.records[i];
if (record->patchpointID == exit.m_stackmapID)
break;
}
RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID);
CCallHelpers jit(vm, codeBlock);
// We need scratch space to save all registers and to build up the JSStack.
// Use a scratch buffer to transfer all values.
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(sizeof(EncodedJSValue) * exit.m_values.size() + requiredScratchMemorySizeInBytes());
EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
char* registerScratch = bitwise_cast<char*>(scratch + exit.m_values.size());
// Make sure that saveAllRegisters() has a place on top of the stack to spill things. That
// function expects to be able to use top of stack for scratch memory.
jit.push(GPRInfo::regT0);
saveAllRegisters(jit, registerScratch);
// Bring the stack back into a sane form.
jit.pop(GPRInfo::regT0);
jit.pop(GPRInfo::regT0);
// The remaining code assumes that SP/FP are in the same state that they were in the FTL's
// call frame.
// Get the call frame and tag thingies.
record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::callFrameRegister);
jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
// Do some value profiling.
if (exit.m_profileValueFormat != InvalidValueFormat) {
record->locations[1].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
reboxAccordingToFormat(
exit.m_profileValueFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
jit.loadPtr(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureOffset()), GPRInfo::regT1);
jit.storePtr(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructure());
jit.load8(MacroAssembler::Address(GPRInfo::regT1, Structure::indexingTypeOffset()), GPRInfo::regT1);
jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
}
}
if (!!exit.m_valueProfile)
jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0));
}
// Save all state from wherever the exit data tells us it was, into the appropriate place in
// the scratch buffer. This doesn't rebox any values yet.
for (unsigned index = exit.m_values.size(); index--;) {
ExitValue value = exit.m_values[index];
switch (value.kind()) {
case ExitValueDead:
jit.move(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), GPRInfo::regT0);
break;
case ExitValueConstant:
jit.move(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), GPRInfo::regT0);
break;
case ExitValueArgument:
record->locations[value.exitArgument().argument()].restoreInto(
jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
break;
case ExitValueInJSStack:
case ExitValueInJSStackAsInt32:
case ExitValueInJSStackAsInt52:
case ExitValueInJSStackAsDouble:
jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
jit.store64(GPRInfo::regT0, scratch + index);
}
// Now get state out of the scratch buffer and place it back into the stack. This part does
// all reboxing.
for (unsigned index = exit.m_values.size(); index--;) {
int operand = exit.m_values.operandForIndex(index);
ExitValue value = exit.m_values[index];
jit.load64(scratch + index, GPRInfo::regT0);
reboxAccordingToFormat(
value.valueFormat(), jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(operand));
}
handleExitCounts(jit, exit);
reifyInlinedCallFrames(jit, exit);
jit.move(MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
jit.pop(MacroAssembler::framePointerRegister);
jit.pop(GPRInfo::nonArgGPR0); // ignore the result.
if (exit.m_lastSetOperand.isValid()) {
jit.load64(
AssemblyHelpers::addressFor(exit.m_lastSetOperand), GPRInfo::cachedResultRegister);
}
adjustAndJumpToTarget(jit, exit);
LinkBuffer patchBuffer(*vm, &jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldShowDisassembly(),
patchBuffer,
("FTL OSR exit #%u (bc#%u, %s) from %s, with operands = %s, and record = %s",
exitID, exit.m_codeOrigin.bytecodeIndex,
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(exit.m_values)).data(),
toCString(*record).data()));
}
static void compileStubWithoutOSRExitStackmap(
unsigned exitID, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
{
CCallHelpers jit(vm, codeBlock);
// Make ourselves look like a real C function.
jit.push(MacroAssembler::framePointerRegister);
jit.move(MacroAssembler::stackPointerRegister, MacroAssembler::framePointerRegister);
// This is actually fairly easy, even though it is horribly gross. We know that
// LLVM would have passes us all of the state via arguments. We know how to get
// the arguments. And, we know how to pop stack back to the JIT stack frame, sort
// of: we know that it's two frames beneath us. This is terrible and I feel
// ashamed of it, but it will work for now.
CArgumentGetter arguments(jit, 2);
// First recover our call frame and tag thingies.
arguments.loadNextPtr(GPRInfo::callFrameRegister);
jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
// Do some value profiling.
if (exit.m_profileValueFormat != InvalidValueFormat) {
arguments.loadNextAndBox(exit.m_profileValueFormat, GPRInfo::nonArgGPR0);
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
jit.loadPtr(MacroAssembler::Address(GPRInfo::nonArgGPR0, JSCell::structureOffset()), GPRInfo::nonArgGPR1);
jit.storePtr(GPRInfo::nonArgGPR1, arrayProfile->addressOfLastSeenStructure());
jit.load8(MacroAssembler::Address(GPRInfo::nonArgGPR1, Structure::indexingTypeOffset()), GPRInfo::nonArgGPR1);
jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::nonArgGPR2);
jit.lshift32(GPRInfo::nonArgGPR1, GPRInfo::nonArgGPR2);
jit.or32(GPRInfo::nonArgGPR2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
}
}
if (!!exit.m_valueProfile)
jit.store64(GPRInfo::nonArgGPR0, exit.m_valueProfile.getSpecFailBucket(0));
}
// Use a scratch buffer to transfer all values.
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(sizeof(EncodedJSValue) * exit.m_values.size());
EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
// Start by dumping all argument exit values to the stack.
Vector<ExitArgumentForOperand, 16> sortedArguments;
for (unsigned i = exit.m_values.size(); i--;) {
ExitValue value = exit.m_values[i];
int operand = exit.m_values.operandForIndex(i);
if (!value.isArgument())
continue;
sortedArguments.append(ExitArgumentForOperand(value.exitArgument(), VirtualRegister(operand)));
}
std::sort(sortedArguments.begin(), sortedArguments.end(), lesserArgumentIndex);
for (unsigned i = 0; i < sortedArguments.size(); ++i) {
ExitArgumentForOperand argument = sortedArguments[i];
arguments.loadNextAndBox(argument.exitArgument().format(), GPRInfo::nonArgGPR0);
jit.store64(
GPRInfo::nonArgGPR0, scratch + exit.m_values.indexForOperand(argument.operand()));
}
// All temp registers are free at this point.
// Move anything on the stack into the appropriate place in the scratch buffer.
for (unsigned i = exit.m_values.size(); i--;) {
ExitValue value = exit.m_values[i];
switch (value.kind()) {
case ExitValueInJSStack:
jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
break;
case ExitValueInJSStackAsInt32:
jit.load32(
AssemblyHelpers::addressFor(value.virtualRegister()).withOffset(
OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)),
GPRInfo::regT0);
jit.or64(GPRInfo::tagTypeNumberRegister, GPRInfo::regT0);
break;
case ExitValueInJSStackAsInt52:
jit.load64(AssemblyHelpers::addressFor(value.virtualRegister()), GPRInfo::regT0);
jit.rshift64(
AssemblyHelpers::TrustedImm32(JSValue::int52ShiftAmount), GPRInfo::regT0);
jit.boxInt52(GPRInfo::regT0, GPRInfo::regT0, GPRInfo::regT1, FPRInfo::fpRegT0);
break;
case ExitValueInJSStackAsDouble:
jit.loadDouble(AssemblyHelpers::addressFor(value.virtualRegister()), FPRInfo::fpRegT0);
jit.boxDouble(FPRInfo::fpRegT0, GPRInfo::regT0);
break;
case ExitValueDead:
case ExitValueConstant:
case ExitValueArgument:
// Don't do anything for these.
continue;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
jit.store64(GPRInfo::regT0, scratch + i);
}
// Move everything from the scratch buffer to the stack; this also reifies constants.
for (unsigned i = exit.m_values.size(); i--;) {
ExitValue value = exit.m_values[i];
int operand = exit.m_values.operandForIndex(i);
MacroAssembler::Address address = AssemblyHelpers::addressFor(operand);
switch (value.kind()) {
case ExitValueDead:
jit.store64(MacroAssembler::TrustedImm64(JSValue::encode(jsUndefined())), address);
break;
case ExitValueConstant:
jit.store64(MacroAssembler::TrustedImm64(JSValue::encode(value.constant())), address);
break;
case ExitValueInJSStack:
case ExitValueInJSStackAsInt32:
case ExitValueInJSStackAsInt52:
case ExitValueInJSStackAsDouble:
case ExitValueArgument:
jit.load64(scratch + i, GPRInfo::regT0);
jit.store64(GPRInfo::regT0, address);
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
}
handleExitCounts(jit, exit);
reifyInlinedCallFrames(jit, exit);
jit.pop(MacroAssembler::framePointerRegister);
jit.move(MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
jit.pop(MacroAssembler::framePointerRegister);
jit.pop(GPRInfo::nonArgGPR0); // ignore the result.
if (exit.m_lastSetOperand.isValid()) {
jit.load64(
AssemblyHelpers::addressFor(exit.m_lastSetOperand), GPRInfo::cachedResultRegister);
}
adjustAndJumpToTarget(jit, exit);
LinkBuffer patchBuffer(*vm, &jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldShowDisassembly(),
patchBuffer,
("FTL OSR exit #%u (bc#%u, %s) from %s, with operands = %s",
exitID, exit.m_codeOrigin.bytecodeIndex,
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(exit.m_values)).data()));
}
MacroAssemblerCodeRef osrExitGenerationThunkGenerator(VM* vm)
{
AssemblyHelpers jit(vm, 0);
// Note that the "return address" will be the OSR exit ID.
ptrdiff_t stackMisalignment = MacroAssembler::pushToSaveByteOffset();
// Pretend that we're a C call frame.
jit.pushToSave(MacroAssembler::framePointerRegister);
jit.move(MacroAssembler::stackPointerRegister, MacroAssembler::framePointerRegister);
stackMisalignment += MacroAssembler::pushToSaveByteOffset();
// Now create ourselves enough stack space to give saveAllRegisters() a scratch slot.
unsigned numberOfRequiredPops = 0;
do {
jit.pushToSave(GPRInfo::regT0);
stackMisalignment += MacroAssembler::pushToSaveByteOffset();
numberOfRequiredPops++;
} while (stackMisalignment % stackAlignmentBytes());
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(requiredScratchMemorySizeInBytes());
char* buffer = static_cast<char*>(scratchBuffer->dataBuffer());
saveAllRegisters(jit, buffer);
// Tell GC mark phase how much of the scratch buffer is active during call.
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->activeLengthPtr()), GPRInfo::nonArgGPR1);
jit.storePtr(MacroAssembler::TrustedImmPtr(requiredScratchMemorySizeInBytes()), GPRInfo::nonArgGPR1);
jit.loadPtr(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
jit.peek(
GPRInfo::argumentGPR1,
(stackMisalignment - MacroAssembler::pushToSaveByteOffset()) / sizeof(void*));
MacroAssembler::Call functionCall = jit.call();
// At this point we want to make a tail call to what was returned to us in the
// returnValueGPR. But at the same time as we do this, we must restore all registers.
// The way we will accomplish this is by arranging to have the tail call target in the
// return address "slot" (be it a register or the stack).
jit.move(GPRInfo::returnValueGPR, GPRInfo::regT0);
// Make sure we tell the GC that we're not using the scratch buffer anymore.
jit.move(MacroAssembler::TrustedImmPtr(scratchBuffer->activeLengthPtr()), GPRInfo::regT1);
jit.storePtr(MacroAssembler::TrustedImmPtr(0), GPRInfo::regT1);
// Prepare for tail call.
while (numberOfRequiredPops--)
jit.popToRestore(GPRInfo::regT1);
jit.popToRestore(MacroAssembler::framePointerRegister);
// At this point we're sitting on the return address - so if we did a jump right now, the
// tail-callee would be happy. Instead we'll stash the callee in the return address and then
// restore all registers.
jit.restoreReturnAddressBeforeReturn(GPRInfo::regT0);
restoreAllRegisters(jit, buffer);
jit.ret();
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
patchBuffer.link(functionCall, compileFTLOSRExit);
return FINALIZE_CODE(patchBuffer, ("FTL OSR exit generation thunk"));
}
static void compileStub(
unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
{
StackMaps::Record* record = nullptr;
for (unsigned i = jitCode->stackmaps.records.size(); i--;) {
record = &jitCode->stackmaps.records[i];
if (record->patchpointID == exit.m_stackmapID)
break;
}
RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID);
// This code requires framePointerRegister is the same as callFrameRegister
static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");
CCallHelpers jit(vm, codeBlock);
// We need scratch space to save all registers, to build up the JS stack, to deal with unwind
// fixup, pointers to all of the objects we materialize, and the elements inside those objects
// that we materialize.
// Figure out how much space we need for those object allocations.
unsigned numMaterializations = 0;
size_t maxMaterializationNumArguments = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
numMaterializations++;
maxMaterializationNumArguments = std::max(
maxMaterializationNumArguments,
materialization->properties().size());
}
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(
sizeof(EncodedJSValue) * (
exit.m_values.size() + numMaterializations + maxMaterializationNumArguments) +
requiredScratchMemorySizeInBytes() +
codeBlock->calleeSaveRegisters()->size() * sizeof(uint64_t));
EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
EncodedJSValue* materializationPointers = scratch + exit.m_values.size();
EncodedJSValue* materializationArguments = materializationPointers + numMaterializations;
char* registerScratch = bitwise_cast<char*>(materializationArguments + maxMaterializationNumArguments);
uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*> materializationToPointer;
unsigned materializationCount = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
materializationToPointer.add(
materialization, materializationPointers + materializationCount++);
}
// Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave().
// We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use
// that slot for saveAllRegisters().
saveAllRegisters(jit, registerScratch);
// Bring the stack back into a sane form and assert that it's sane.
jit.popToRestore(GPRInfo::regT0);
jit.checkStackPointerAlignment();
if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
Profiler::Database& database = *vm->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
// The remaining code assumes that SP/FP are in the same state that they were in the FTL's
// call frame.
// Get the call frame and tag thingies.
// Restore the exiting function's callFrame value into a regT4
jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
// Do some value profiling.
if (exit.m_profileDataFormat != DataFormatNone) {
record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
reboxAccordingToFormat(
exit.m_profileDataFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1);
jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
}
}
if (!!exit.m_valueProfile)
jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0));
}
// Materialize all objects. Don't materialize an object until all
// of the objects it needs have been materialized. We break cycles
// by populating objects late - we only consider an object as
// needing another object if the later is needed for the
// allocation of the former.
HashSet<ExitTimeObjectMaterialization*> toMaterialize;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations)
toMaterialize.add(materialization);
while (!toMaterialize.isEmpty()) {
unsigned previousToMaterializeSize = toMaterialize.size();
Vector<ExitTimeObjectMaterialization*> worklist;
worklist.appendRange(toMaterialize.begin(), toMaterialize.end());
for (ExitTimeObjectMaterialization* materialization : worklist) {
// Check if we can do anything about this right now.
bool allGood = true;
for (ExitPropertyValue value : materialization->properties()) {
if (!value.value().isObjectMaterialization())
continue;
if (!value.location().neededForMaterialization())
continue;
if (toMaterialize.contains(value.value().objectMaterialization())) {
// Gotta skip this one, since it needs a
// materialization that hasn't been materialized.
allGood = false;
break;
}
}
if (!allGood)
continue;
// All systems go for materializing the object. First we
// recover the values of all of its fields and then we
// call a function to actually allocate the beast.
// We only recover the fields that are needed for the allocation.
for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
const ExitPropertyValue& property = materialization->properties()[propertyIndex];
const ExitValue& value = property.value();
if (!property.location().neededForMaterialization())
continue;
compileRecovery(
jit, value, record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
}
// This call assumes that we don't pass arguments on the stack.
jit.setupArgumentsWithExecState(
CCallHelpers::TrustedImmPtr(materialization),
CCallHelpers::TrustedImmPtr(materializationArguments));
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationMaterializeObjectInOSR)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
jit.storePtr(GPRInfo::returnValueGPR, materializationToPointer.get(materialization));
// Let everyone know that we're done.
toMaterialize.remove(materialization);
}
// We expect progress! This ensures that we crash rather than looping infinitely if there
// is something broken about this fixpoint. Or, this could happen if we ever violate the
// "materializations form a DAG" rule.
RELEASE_ASSERT(toMaterialize.size() < previousToMaterializeSize);
}
// Now that all the objects have been allocated, we populate them
// with the correct values. This time we can recover all the
// fields, including those that are only needed for the allocation.
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
const ExitValue& value = materialization->properties()[propertyIndex].value();
compileRecovery(
jit, value, record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
}
// This call assumes that we don't pass arguments on the stack
jit.setupArgumentsWithExecState(
CCallHelpers::TrustedImmPtr(materialization),
CCallHelpers::TrustedImmPtr(materializationToPointer.get(materialization)),
CCallHelpers::TrustedImmPtr(materializationArguments));
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationPopulateObjectInOSR)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
}
// Save all state from wherever the exit data tells us it was, into the appropriate place in
// the scratch buffer. This also does the reboxing.
for (unsigned index = exit.m_values.size(); index--;) {
compileRecovery(
jit, exit.m_values[index], record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.store64(GPRInfo::regT0, scratch + index);
}
// Henceforth we make it look like the exiting function was called through a register
// preservation wrapper. This implies that FP must be nudged down by a certain amount. Then
// we restore the various things according to either exit.m_values or by copying from the
// old frame, and finally we save the various callee-save registers into where the
// restoration thunk would restore them from.
// Before we start messing with the frame, we need to set aside any registers that the
// FTL code was preserving.
for (unsigned i = codeBlock->calleeSaveRegisters()->size(); i--;) {
RegisterAtOffset entry = codeBlock->calleeSaveRegisters()->at(i);
jit.load64(
MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()),
GPRInfo::regT0);
jit.store64(GPRInfo::regT0, unwindScratch + i);
}
jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2);
// Let's say that the FTL function had failed its arity check. In that case, the stack will
// contain some extra stuff.
//
// We compute the padded stack space:
//
// paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1)
//
// The stack will have regT2 + CallFrameHeaderSize stuff.
// We want to make the stack look like this, from higher addresses down:
//
// - argument padding
// - actual arguments
// - call frame header
// This code assumes that we're dealing with FunctionCode.
RELEASE_ASSERT(codeBlock->codeType() == FunctionCode);
jit.add32(
MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2,
GPRInfo::regT3);
MacroAssembler::Jump arityIntact = jit.branch32(
MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0));
jit.neg32(GPRInfo::regT3);
jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3);
jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3);
jit.add32(GPRInfo::regT3, GPRInfo::regT2);
arityIntact.link(&jit);
CodeBlock* baselineCodeBlock = jit.baselineCodeBlockFor(exit.m_codeOrigin);
// First set up SP so that our data doesn't get clobbered by signals.
unsigned conservativeStackDelta =
(exit.m_values.numberOfLocals() + baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters()) * sizeof(Register) +
maxFrameExtentForSlowPathCall;
conservativeStackDelta = WTF::roundUpToMultipleOf(
stackAlignmentBytes(), conservativeStackDelta);
jit.addPtr(
MacroAssembler::TrustedImm32(-conservativeStackDelta),
MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
jit.checkStackPointerAlignment();
RegisterSet allFTLCalleeSaves = RegisterSet::ftlCalleeSaveRegisters();
RegisterAtOffsetList* baselineCalleeSaves = baselineCodeBlock->calleeSaveRegisters();
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
if (!allFTLCalleeSaves.get(reg))
continue;
unsigned unwindIndex = codeBlock->calleeSaveRegisters()->indexOf(reg);
RegisterAtOffset* baselineRegisterOffset = baselineCalleeSaves->find(reg);
if (reg.isGPR()) {
GPRReg regToLoad = baselineRegisterOffset ? GPRInfo::regT0 : reg.gpr();
if (unwindIndex == UINT_MAX) {
// The FTL compilation didn't preserve this register. This means that it also
// didn't use the register. So its value at the beginning of OSR exit should be
// preserved by the thunk. Luckily, we saved all registers into the register
// scratch buffer, so we can restore them from there.
jit.load64(registerScratch + offsetOfReg(reg), regToLoad);
} else {
// The FTL compilation preserved the register. Its new value is therefore
// irrelevant, but we can get the value that was preserved by using the unwind
// data. We've already copied all unwind-able preserved registers into the unwind
// scratch buffer, so we can get it from there.
jit.load64(unwindScratch + unwindIndex, regToLoad);
}
if (baselineRegisterOffset)
jit.store64(regToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset()));
} else {
FPRReg fpRegToLoad = baselineRegisterOffset ? FPRInfo::fpRegT0 : reg.fpr();
if (unwindIndex == UINT_MAX)
jit.loadDouble(MacroAssembler::TrustedImmPtr(registerScratch + offsetOfReg(reg)), fpRegToLoad);
else
jit.loadDouble(MacroAssembler::TrustedImmPtr(unwindScratch + unwindIndex), fpRegToLoad);
if (baselineRegisterOffset)
jit.storeDouble(fpRegToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset()));
}
}
size_t baselineVirtualRegistersForCalleeSaves = baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters();
// Now get state out of the scratch buffer and place it back into the stack. The values are
// already reboxed so we just move them.
for (unsigned index = exit.m_values.size(); index--;) {
VirtualRegister reg = exit.m_values.virtualRegisterForIndex(index);
if (reg.isLocal() && reg.toLocal() < static_cast<int>(baselineVirtualRegistersForCalleeSaves))
continue;
jit.load64(scratch + index, GPRInfo::regT0);
jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(reg));
}
handleExitCounts(jit, exit);
reifyInlinedCallFrames(jit, exit);
adjustAndJumpToTarget(jit, exit, false);
LinkBuffer patchBuffer(*vm, jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(),
patchBuffer,
("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s",
exitID, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(exit.m_values)).data(),
toCString(*record).data()));
}