void CallFrameShuffleData::setupCalleeSaveRegisters(CodeBlock* codeBlock)
{
RegisterSet calleeSaveRegisters { RegisterSet::vmCalleeSaveRegisters() };
RegisterAtOffsetList* registerSaveLocations = codeBlock->calleeSaveRegisters();
for (size_t i = 0; i < registerSaveLocations->size(); ++i) {
RegisterAtOffset entry { registerSaveLocations->at(i) };
if (!calleeSaveRegisters.get(entry.reg()))
continue;
VirtualRegister saveSlot { entry.offsetAsIndex() };
registers[entry.reg()]
= ValueRecovery::displacedInJSStack(saveSlot, DataFormatJS);
}
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
if (!calleeSaveRegisters.get(reg))
continue;
if (registers[reg])
continue;
registers[reg] = ValueRecovery::inRegister(reg, DataFormatJS);
}
}
SUPPRESS_ASAN
void* prepareOSREntry(ExecState* exec, CodeBlock* codeBlock, unsigned bytecodeIndex)
{
ASSERT(JITCode::isOptimizingJIT(codeBlock->jitType()));
ASSERT(codeBlock->alternative());
ASSERT(codeBlock->alternative()->jitType() == JITCode::BaselineJIT);
ASSERT(!codeBlock->jitCodeMap());
if (!Options::useOSREntryToDFG())
return 0;
if (Options::verboseOSR()) {
dataLog(
"DFG OSR in ", *codeBlock->alternative(), " -> ", *codeBlock,
" from bc#", bytecodeIndex, "\n");
}
VM* vm = &exec->vm();
sanitizeStackForVM(vm);
if (bytecodeIndex)
codeBlock->ownerScriptExecutable()->setDidTryToEnterInLoop(true);
if (codeBlock->jitType() != JITCode::DFGJIT) {
RELEASE_ASSERT(codeBlock->jitType() == JITCode::FTLJIT);
// When will this happen? We could have:
//
// - An exit from the FTL JIT into the baseline JIT followed by an attempt
// to reenter. We're fine with allowing this to fail. If it happens
// enough we'll just reoptimize. It basically means that the OSR exit cost
// us dearly and so reoptimizing is the right thing to do.
//
// - We have recursive code with hot loops. Consider that foo has a hot loop
// that calls itself. We have two foo's on the stack, lets call them foo1
// and foo2, with foo1 having called foo2 from foo's hot loop. foo2 gets
// optimized all the way into the FTL. Then it returns into foo1, and then
// foo1 wants to get optimized. It might reach this conclusion from its
// hot loop and attempt to OSR enter. And we'll tell it that it can't. It
// might be worth addressing this case, but I just think this case will
// be super rare. For now, if it does happen, it'll cause some compilation
// thrashing.
if (Options::verboseOSR())
dataLog(" OSR failed because the target code block is not DFG.\n");
return 0;
}
JITCode* jitCode = codeBlock->jitCode()->dfg();
OSREntryData* entry = jitCode->osrEntryDataForBytecodeIndex(bytecodeIndex);
if (!entry) {
if (Options::verboseOSR())
dataLogF(" OSR failed because the entrypoint was optimized out.\n");
return 0;
}
ASSERT(entry->m_bytecodeIndex == bytecodeIndex);
// The code below checks if it is safe to perform OSR entry. It may find
// that it is unsafe to do so, for any number of reasons, which are documented
// below. If the code decides not to OSR then it returns 0, and it's the caller's
// responsibility to patch up the state in such a way as to ensure that it's
// both safe and efficient to continue executing baseline code for now. This
// should almost certainly include calling either codeBlock->optimizeAfterWarmUp()
// or codeBlock->dontOptimizeAnytimeSoon().
// 1) Verify predictions. If the predictions are inconsistent with the actual
// values, then OSR entry is not possible at this time. It's tempting to
// assume that we could somehow avoid this case. We can certainly avoid it
// for first-time loop OSR - that is, OSR into a CodeBlock that we have just
// compiled. Then we are almost guaranteed that all of the predictions will
// check out. It would be pretty easy to make that a hard guarantee. But
// then there would still be the case where two call frames with the same
// baseline CodeBlock are on the stack at the same time. The top one
// triggers compilation and OSR. In that case, we may no longer have
// accurate value profiles for the one deeper in the stack. Hence, when we
// pop into the CodeBlock that is deeper on the stack, we might OSR and
// realize that the predictions are wrong. Probably, in most cases, this is
// just an anomaly in the sense that the older CodeBlock simply went off
// into a less-likely path. So, the wisest course of action is to simply not
// OSR at this time.
for (size_t argument = 0; argument < entry->m_expectedValues.numberOfArguments(); ++argument) {
if (argument >= exec->argumentCountIncludingThis()) {
if (Options::verboseOSR()) {
dataLogF(" OSR failed because argument %zu was not passed, expected ", argument);
entry->m_expectedValues.argument(argument).dump(WTF::dataFile());
dataLogF(".\n");
}
return 0;
}
JSValue value;
if (!argument)
value = exec->thisValue();
else
value = exec->argument(argument - 1);
if (!entry->m_expectedValues.argument(argument).validate(value)) {
if (Options::verboseOSR()) {
dataLog(
" OSR failed because argument ", argument, " is ", value,
", expected ", entry->m_expectedValues.argument(argument), ".\n");
}
return 0;
}
}
for (size_t local = 0; local < entry->m_expectedValues.numberOfLocals(); ++local) {
int localOffset = virtualRegisterForLocal(local).offset();
if (entry->m_localsForcedDouble.get(local)) {
if (!exec->registers()[localOffset].asanUnsafeJSValue().isNumber()) {
if (Options::verboseOSR()) {
dataLog(
" OSR failed because variable ", localOffset, " is ",
exec->registers()[localOffset].asanUnsafeJSValue(), ", expected number.\n");
}
return 0;
}
continue;
}
if (entry->m_localsForcedAnyInt.get(local)) {
if (!exec->registers()[localOffset].asanUnsafeJSValue().isAnyInt()) {
if (Options::verboseOSR()) {
dataLog(
" OSR failed because variable ", localOffset, " is ",
exec->registers()[localOffset].asanUnsafeJSValue(), ", expected ",
"machine int.\n");
}
return 0;
}
continue;
}
if (!entry->m_expectedValues.local(local).validate(exec->registers()[localOffset].asanUnsafeJSValue())) {
if (Options::verboseOSR()) {
dataLog(
" OSR failed because variable ", localOffset, " is ",
exec->registers()[localOffset].asanUnsafeJSValue(), ", expected ",
entry->m_expectedValues.local(local), ".\n");
}
return 0;
}
}
// 2) Check the stack height. The DFG JIT may require a taller stack than the
// baseline JIT, in some cases. If we can't grow the stack, then don't do
// OSR right now. That's the only option we have unless we want basic block
// boundaries to start throwing RangeErrors. Although that would be possible,
// it seems silly: you'd be diverting the program to error handling when it
// would have otherwise just kept running albeit less quickly.
unsigned frameSizeForCheck = jitCode->common.requiredRegisterCountForExecutionAndExit();
if (!vm->interpreter->stack().ensureCapacityFor(&exec->registers()[virtualRegisterForLocal(frameSizeForCheck - 1).offset()])) {
if (Options::verboseOSR())
dataLogF(" OSR failed because stack growth failed.\n");
return 0;
}
if (Options::verboseOSR())
dataLogF(" OSR should succeed.\n");
// At this point we're committed to entering. We will do some work to set things up,
// but we also rely on our caller recognizing that when we return a non-null pointer,
// that means that we're already past the point of no return and we must succeed at
// entering.
// 3) Set up the data in the scratch buffer and perform data format conversions.
unsigned frameSize = jitCode->common.frameRegisterCount;
unsigned baselineFrameSize = entry->m_expectedValues.numberOfLocals();
unsigned maxFrameSize = std::max(frameSize, baselineFrameSize);
Register* scratch = bitwise_cast<Register*>(vm->scratchBufferForSize(sizeof(Register) * (2 + JSStack::CallFrameHeaderSize + maxFrameSize))->dataBuffer());
*bitwise_cast<size_t*>(scratch + 0) = frameSize;
void* targetPC = codeBlock->jitCode()->executableAddressAtOffset(entry->m_machineCodeOffset);
if (Options::verboseOSR())
dataLogF(" OSR using target PC %p.\n", targetPC);
RELEASE_ASSERT(targetPC);
*bitwise_cast<void**>(scratch + 1) = targetPC;
Register* pivot = scratch + 2 + JSStack::CallFrameHeaderSize;
for (int index = -JSStack::CallFrameHeaderSize; index < static_cast<int>(baselineFrameSize); ++index) {
VirtualRegister reg(-1 - index);
if (reg.isLocal()) {
if (entry->m_localsForcedDouble.get(reg.toLocal())) {
*bitwise_cast<double*>(pivot + index) = exec->registers()[reg.offset()].asanUnsafeJSValue().asNumber();
continue;
}
if (entry->m_localsForcedAnyInt.get(reg.toLocal())) {
*bitwise_cast<int64_t*>(pivot + index) = exec->registers()[reg.offset()].asanUnsafeJSValue().asAnyInt() << JSValue::int52ShiftAmount;
continue;
}
}
pivot[index] = exec->registers()[reg.offset()].asanUnsafeJSValue();
}
// 4) Reshuffle those registers that need reshuffling.
Vector<JSValue> temporaryLocals(entry->m_reshufflings.size());
for (unsigned i = entry->m_reshufflings.size(); i--;)
temporaryLocals[i] = pivot[VirtualRegister(entry->m_reshufflings[i].fromOffset).toLocal()].asanUnsafeJSValue();
for (unsigned i = entry->m_reshufflings.size(); i--;)
pivot[VirtualRegister(entry->m_reshufflings[i].toOffset).toLocal()] = temporaryLocals[i];
// 5) Clear those parts of the call frame that the DFG ain't using. This helps GC on
// some programs by eliminating some stale pointer pathologies.
for (unsigned i = frameSize; i--;) {
if (entry->m_machineStackUsed.get(i))
continue;
pivot[i] = JSValue();
}
// 6) Copy our callee saves to buffer.
#if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0
RegisterAtOffsetList* registerSaveLocations = codeBlock->calleeSaveRegisters();
RegisterAtOffsetList* allCalleeSaves = vm->getAllCalleeSaveRegisterOffsets();
RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs());
unsigned registerCount = registerSaveLocations->size();
VMEntryRecord* record = vmEntryRecord(vm->topVMEntryFrame);
for (unsigned i = 0; i < registerCount; i++) {
RegisterAtOffset currentEntry = registerSaveLocations->at(i);
if (dontSaveRegisters.get(currentEntry.reg()))
continue;
RegisterAtOffset* calleeSavesEntry = allCalleeSaves->find(currentEntry.reg());
*(bitwise_cast<intptr_t*>(pivot - 1) - currentEntry.offsetAsIndex()) = record->calleeSaveRegistersBuffer[calleeSavesEntry->offsetAsIndex()];
}
#endif
// 7) Fix the call frame to have the right code block.
*bitwise_cast<CodeBlock**>(pivot - 1 - JSStack::CodeBlock) = codeBlock;
if (Options::verboseOSR())
dataLogF(" OSR returning data buffer %p.\n", scratch);
return scratch;
}
static void compileStub(
unsigned exitID, JITCode* jitCode, OSRExit& exit, VM* vm, CodeBlock* codeBlock)
{
StackMaps::Record* record = nullptr;
for (unsigned i = jitCode->stackmaps.records.size(); i--;) {
record = &jitCode->stackmaps.records[i];
if (record->patchpointID == exit.m_stackmapID)
break;
}
RELEASE_ASSERT(record->patchpointID == exit.m_stackmapID);
// This code requires framePointerRegister is the same as callFrameRegister
static_assert(MacroAssembler::framePointerRegister == GPRInfo::callFrameRegister, "MacroAssembler::framePointerRegister and GPRInfo::callFrameRegister must be the same");
CCallHelpers jit(vm, codeBlock);
// We need scratch space to save all registers, to build up the JS stack, to deal with unwind
// fixup, pointers to all of the objects we materialize, and the elements inside those objects
// that we materialize.
// Figure out how much space we need for those object allocations.
unsigned numMaterializations = 0;
size_t maxMaterializationNumArguments = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
numMaterializations++;
maxMaterializationNumArguments = std::max(
maxMaterializationNumArguments,
materialization->properties().size());
}
ScratchBuffer* scratchBuffer = vm->scratchBufferForSize(
sizeof(EncodedJSValue) * (
exit.m_values.size() + numMaterializations + maxMaterializationNumArguments) +
requiredScratchMemorySizeInBytes() +
codeBlock->calleeSaveRegisters()->size() * sizeof(uint64_t));
EncodedJSValue* scratch = scratchBuffer ? static_cast<EncodedJSValue*>(scratchBuffer->dataBuffer()) : 0;
EncodedJSValue* materializationPointers = scratch + exit.m_values.size();
EncodedJSValue* materializationArguments = materializationPointers + numMaterializations;
char* registerScratch = bitwise_cast<char*>(materializationArguments + maxMaterializationNumArguments);
uint64_t* unwindScratch = bitwise_cast<uint64_t*>(registerScratch + requiredScratchMemorySizeInBytes());
HashMap<ExitTimeObjectMaterialization*, EncodedJSValue*> materializationToPointer;
unsigned materializationCount = 0;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
materializationToPointer.add(
materialization, materializationPointers + materializationCount++);
}
// Note that we come in here, the stack used to be as LLVM left it except that someone called pushToSave().
// We don't care about the value they saved. But, we do appreciate the fact that they did it, because we use
// that slot for saveAllRegisters().
saveAllRegisters(jit, registerScratch);
// Bring the stack back into a sane form and assert that it's sane.
jit.popToRestore(GPRInfo::regT0);
jit.checkStackPointerAlignment();
if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
Profiler::Database& database = *vm->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitID, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
// The remaining code assumes that SP/FP are in the same state that they were in the FTL's
// call frame.
// Get the call frame and tag thingies.
// Restore the exiting function's callFrame value into a regT4
jit.move(MacroAssembler::TrustedImm64(TagTypeNumber), GPRInfo::tagTypeNumberRegister);
jit.move(MacroAssembler::TrustedImm64(TagMask), GPRInfo::tagMaskRegister);
// Do some value profiling.
if (exit.m_profileDataFormat != DataFormatNone) {
record->locations[0].restoreInto(jit, jitCode->stackmaps, registerScratch, GPRInfo::regT0);
reboxAccordingToFormat(
exit.m_profileDataFormat, jit, GPRInfo::regT0, GPRInfo::regT1, GPRInfo::regT2);
if (exit.m_kind == BadCache || exit.m_kind == BadIndexingType) {
CodeOrigin codeOrigin = exit.m_codeOriginForExitProfile;
if (ArrayProfile* arrayProfile = jit.baselineCodeBlockFor(codeOrigin)->getArrayProfile(codeOrigin.bytecodeIndex)) {
jit.load32(MacroAssembler::Address(GPRInfo::regT0, JSCell::structureIDOffset()), GPRInfo::regT1);
jit.store32(GPRInfo::regT1, arrayProfile->addressOfLastSeenStructureID());
jit.load8(MacroAssembler::Address(GPRInfo::regT0, JSCell::indexingTypeOffset()), GPRInfo::regT1);
jit.move(MacroAssembler::TrustedImm32(1), GPRInfo::regT2);
jit.lshift32(GPRInfo::regT1, GPRInfo::regT2);
jit.or32(GPRInfo::regT2, MacroAssembler::AbsoluteAddress(arrayProfile->addressOfArrayModes()));
}
}
if (!!exit.m_valueProfile)
jit.store64(GPRInfo::regT0, exit.m_valueProfile.getSpecFailBucket(0));
}
// Materialize all objects. Don't materialize an object until all
// of the objects it needs have been materialized. We break cycles
// by populating objects late - we only consider an object as
// needing another object if the later is needed for the
// allocation of the former.
HashSet<ExitTimeObjectMaterialization*> toMaterialize;
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations)
toMaterialize.add(materialization);
while (!toMaterialize.isEmpty()) {
unsigned previousToMaterializeSize = toMaterialize.size();
Vector<ExitTimeObjectMaterialization*> worklist;
worklist.appendRange(toMaterialize.begin(), toMaterialize.end());
for (ExitTimeObjectMaterialization* materialization : worklist) {
// Check if we can do anything about this right now.
bool allGood = true;
for (ExitPropertyValue value : materialization->properties()) {
if (!value.value().isObjectMaterialization())
continue;
if (!value.location().neededForMaterialization())
continue;
if (toMaterialize.contains(value.value().objectMaterialization())) {
// Gotta skip this one, since it needs a
// materialization that hasn't been materialized.
allGood = false;
break;
}
}
if (!allGood)
continue;
// All systems go for materializing the object. First we
// recover the values of all of its fields and then we
// call a function to actually allocate the beast.
// We only recover the fields that are needed for the allocation.
for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
const ExitPropertyValue& property = materialization->properties()[propertyIndex];
const ExitValue& value = property.value();
if (!property.location().neededForMaterialization())
continue;
compileRecovery(
jit, value, record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
}
// This call assumes that we don't pass arguments on the stack.
jit.setupArgumentsWithExecState(
CCallHelpers::TrustedImmPtr(materialization),
CCallHelpers::TrustedImmPtr(materializationArguments));
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationMaterializeObjectInOSR)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
jit.storePtr(GPRInfo::returnValueGPR, materializationToPointer.get(materialization));
// Let everyone know that we're done.
toMaterialize.remove(materialization);
}
// We expect progress! This ensures that we crash rather than looping infinitely if there
// is something broken about this fixpoint. Or, this could happen if we ever violate the
// "materializations form a DAG" rule.
RELEASE_ASSERT(toMaterialize.size() < previousToMaterializeSize);
}
// Now that all the objects have been allocated, we populate them
// with the correct values. This time we can recover all the
// fields, including those that are only needed for the allocation.
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) {
for (unsigned propertyIndex = materialization->properties().size(); propertyIndex--;) {
const ExitValue& value = materialization->properties()[propertyIndex].value();
compileRecovery(
jit, value, record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.storePtr(GPRInfo::regT0, materializationArguments + propertyIndex);
}
// This call assumes that we don't pass arguments on the stack
jit.setupArgumentsWithExecState(
CCallHelpers::TrustedImmPtr(materialization),
CCallHelpers::TrustedImmPtr(materializationToPointer.get(materialization)),
CCallHelpers::TrustedImmPtr(materializationArguments));
jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(operationPopulateObjectInOSR)), GPRInfo::nonArgGPR0);
jit.call(GPRInfo::nonArgGPR0);
}
// Save all state from wherever the exit data tells us it was, into the appropriate place in
// the scratch buffer. This also does the reboxing.
for (unsigned index = exit.m_values.size(); index--;) {
compileRecovery(
jit, exit.m_values[index], record, jitCode->stackmaps, registerScratch,
materializationToPointer);
jit.store64(GPRInfo::regT0, scratch + index);
}
// Henceforth we make it look like the exiting function was called through a register
// preservation wrapper. This implies that FP must be nudged down by a certain amount. Then
// we restore the various things according to either exit.m_values or by copying from the
// old frame, and finally we save the various callee-save registers into where the
// restoration thunk would restore them from.
// Before we start messing with the frame, we need to set aside any registers that the
// FTL code was preserving.
for (unsigned i = codeBlock->calleeSaveRegisters()->size(); i--;) {
RegisterAtOffset entry = codeBlock->calleeSaveRegisters()->at(i);
jit.load64(
MacroAssembler::Address(MacroAssembler::framePointerRegister, entry.offset()),
GPRInfo::regT0);
jit.store64(GPRInfo::regT0, unwindScratch + i);
}
jit.load32(CCallHelpers::payloadFor(JSStack::ArgumentCount), GPRInfo::regT2);
// Let's say that the FTL function had failed its arity check. In that case, the stack will
// contain some extra stuff.
//
// We compute the padded stack space:
//
// paddedStackSpace = roundUp(codeBlock->numParameters - regT2 + 1)
//
// The stack will have regT2 + CallFrameHeaderSize stuff.
// We want to make the stack look like this, from higher addresses down:
//
// - argument padding
// - actual arguments
// - call frame header
// This code assumes that we're dealing with FunctionCode.
RELEASE_ASSERT(codeBlock->codeType() == FunctionCode);
jit.add32(
MacroAssembler::TrustedImm32(-codeBlock->numParameters()), GPRInfo::regT2,
GPRInfo::regT3);
MacroAssembler::Jump arityIntact = jit.branch32(
MacroAssembler::GreaterThanOrEqual, GPRInfo::regT3, MacroAssembler::TrustedImm32(0));
jit.neg32(GPRInfo::regT3);
jit.add32(MacroAssembler::TrustedImm32(1 + stackAlignmentRegisters() - 1), GPRInfo::regT3);
jit.and32(MacroAssembler::TrustedImm32(-stackAlignmentRegisters()), GPRInfo::regT3);
jit.add32(GPRInfo::regT3, GPRInfo::regT2);
arityIntact.link(&jit);
CodeBlock* baselineCodeBlock = jit.baselineCodeBlockFor(exit.m_codeOrigin);
// First set up SP so that our data doesn't get clobbered by signals.
unsigned conservativeStackDelta =
(exit.m_values.numberOfLocals() + baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters()) * sizeof(Register) +
maxFrameExtentForSlowPathCall;
conservativeStackDelta = WTF::roundUpToMultipleOf(
stackAlignmentBytes(), conservativeStackDelta);
jit.addPtr(
MacroAssembler::TrustedImm32(-conservativeStackDelta),
MacroAssembler::framePointerRegister, MacroAssembler::stackPointerRegister);
jit.checkStackPointerAlignment();
RegisterSet allFTLCalleeSaves = RegisterSet::ftlCalleeSaveRegisters();
RegisterAtOffsetList* baselineCalleeSaves = baselineCodeBlock->calleeSaveRegisters();
for (Reg reg = Reg::first(); reg <= Reg::last(); reg = reg.next()) {
if (!allFTLCalleeSaves.get(reg))
continue;
unsigned unwindIndex = codeBlock->calleeSaveRegisters()->indexOf(reg);
RegisterAtOffset* baselineRegisterOffset = baselineCalleeSaves->find(reg);
if (reg.isGPR()) {
GPRReg regToLoad = baselineRegisterOffset ? GPRInfo::regT0 : reg.gpr();
if (unwindIndex == UINT_MAX) {
// The FTL compilation didn't preserve this register. This means that it also
// didn't use the register. So its value at the beginning of OSR exit should be
// preserved by the thunk. Luckily, we saved all registers into the register
// scratch buffer, so we can restore them from there.
jit.load64(registerScratch + offsetOfReg(reg), regToLoad);
} else {
// The FTL compilation preserved the register. Its new value is therefore
// irrelevant, but we can get the value that was preserved by using the unwind
// data. We've already copied all unwind-able preserved registers into the unwind
// scratch buffer, so we can get it from there.
jit.load64(unwindScratch + unwindIndex, regToLoad);
}
if (baselineRegisterOffset)
jit.store64(regToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset()));
} else {
FPRReg fpRegToLoad = baselineRegisterOffset ? FPRInfo::fpRegT0 : reg.fpr();
if (unwindIndex == UINT_MAX)
jit.loadDouble(MacroAssembler::TrustedImmPtr(registerScratch + offsetOfReg(reg)), fpRegToLoad);
else
jit.loadDouble(MacroAssembler::TrustedImmPtr(unwindScratch + unwindIndex), fpRegToLoad);
if (baselineRegisterOffset)
jit.storeDouble(fpRegToLoad, MacroAssembler::Address(MacroAssembler::framePointerRegister, baselineRegisterOffset->offset()));
}
}
size_t baselineVirtualRegistersForCalleeSaves = baselineCodeBlock->calleeSaveSpaceAsVirtualRegisters();
// Now get state out of the scratch buffer and place it back into the stack. The values are
// already reboxed so we just move them.
for (unsigned index = exit.m_values.size(); index--;) {
VirtualRegister reg = exit.m_values.virtualRegisterForIndex(index);
if (reg.isLocal() && reg.toLocal() < static_cast<int>(baselineVirtualRegistersForCalleeSaves))
continue;
jit.load64(scratch + index, GPRInfo::regT0);
jit.store64(GPRInfo::regT0, AssemblyHelpers::addressFor(reg));
}
handleExitCounts(jit, exit);
reifyInlinedCallFrames(jit, exit);
adjustAndJumpToTarget(jit, exit, false);
LinkBuffer patchBuffer(*vm, jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldDumpDisassembly() || Options::verboseOSR() || Options::verboseFTLOSRExit(),
patchBuffer,
("FTL OSR exit #%u (%s, %s) from %s, with operands = %s, and record = %s",
exitID, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(exit.m_values)).data(),
toCString(*record).data()));
}
