void JIT::compileCallEvalSlowCase(Instruction* instruction, Vector<SlowCaseEntry>::iterator& iter)
{
CallLinkInfo* info = m_codeBlock->addCallLinkInfo();
info->setUpCall(CallLinkInfo::Call, CodeOrigin(m_bytecodeOffset), regT0);
linkSlowCase(iter);
int registerOffset = -instruction[4].u.operand;
addPtr(TrustedImm32(registerOffset * sizeof(Register) + sizeof(CallerFrameAndPC)), callFrameRegister, stackPointerRegister);
loadPtr(Address(stackPointerRegister, sizeof(Register) * JSStack::Callee - sizeof(CallerFrameAndPC)), regT0);
loadPtr(Address(stackPointerRegister, sizeof(Register) * JSStack::Callee - sizeof(CallerFrameAndPC)), regT1);
move(TrustedImmPtr(info), regT2);
emitLoad(JSStack::Callee, regT1, regT0);
MacroAssemblerCodeRef virtualThunk = virtualThunkFor(m_vm, *info);
info->setSlowStub(createJITStubRoutine(virtualThunk, *m_vm, nullptr, true));
emitNakedCall(virtualThunk.code());
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
checkStackPointerAlignment();
sampleCodeBlock(m_codeBlock);
emitPutCallResult(instruction);
}
void initializeSeparatedWXHeaps(void* stubBase, size_t stubSize, void* jitBase, size_t jitSize)
{
mach_vm_address_t writableAddr = 0;
// Create a second mapping of the JIT region at a random address.
vm_prot_t cur, max;
int remapFlags = VM_FLAGS_ANYWHERE;
#if defined(VM_FLAGS_RANDOM_ADDR)
remapFlags |= VM_FLAGS_RANDOM_ADDR;
#endif
kern_return_t ret = mach_vm_remap(mach_task_self(), &writableAddr, jitSize, 0,
remapFlags,
mach_task_self(), (mach_vm_address_t)jitBase, FALSE,
&cur, &max, VM_INHERIT_DEFAULT);
bool remapSucceeded = (ret == KERN_SUCCESS);
if (!remapSucceeded)
return;
// Assemble a thunk that will serve as the means for writing into the JIT region.
MacroAssemblerCodeRef writeThunk = jitWriteThunkGenerator(reinterpret_cast<void*>(writableAddr), stubBase, stubSize);
int result = 0;
#if USE(EXECUTE_ONLY_JIT_WRITE_FUNCTION)
// Prevent reading the write thunk code.
result = mprotect(stubBase, stubSize, VM_PROT_EXECUTE_ONLY);
RELEASE_ASSERT(!result);
#endif
// Prevent writing into the executable JIT mapping.
result = mprotect(jitBase, jitSize, VM_PROT_READ | VM_PROT_EXECUTE);
RELEASE_ASSERT(!result);
// Prevent execution in the writable JIT mapping.
result = mprotect((void*)writableAddr, jitSize, VM_PROT_READ | VM_PROT_WRITE);
RELEASE_ASSERT(!result);
// Zero out writableAddr to avoid leaking the address of the writable mapping.
memset_s(&writableAddr, sizeof(writableAddr), 0, sizeof(writableAddr));
jitWriteFunction = reinterpret_cast<JITWriteFunction>(writeThunk.code().executableAddress());
}
NativeExecutable* JITThunks::hostFunctionStub(VM* vm, NativeFunction function, NativeFunction constructor, ThunkGenerator generator, Intrinsic intrinsic, const String& name)
{
ASSERT(!isCompilationThread());
ASSERT(vm->canUseJIT());
if (NativeExecutable* nativeExecutable = m_hostFunctionStubMap->get(std::make_tuple(function, constructor, name)))
return nativeExecutable;
RefPtr<JITCode> forCall;
if (generator) {
MacroAssemblerCodeRef entry = generator(vm);
forCall = adoptRef(new DirectJITCode(entry, entry.code(), JITCode::HostCallThunk));
} else
forCall = adoptRef(new NativeJITCode(JIT::compileCTINativeCall(vm, function), JITCode::HostCallThunk));
RefPtr<JITCode> forConstruct = adoptRef(new NativeJITCode(MacroAssemblerCodeRef::createSelfManagedCodeRef(ctiNativeConstruct(vm)), JITCode::HostCallThunk));
NativeExecutable* nativeExecutable = NativeExecutable::create(*vm, forCall, function, forConstruct, constructor, intrinsic, name);
weakAdd(*m_hostFunctionStubMap, std::make_tuple(function, constructor, name), Weak<NativeExecutable>(nativeExecutable, this));
return nativeExecutable;
}
void JITCompiler::link(LinkBuffer& linkBuffer)
{
// Link the code, populate data in CodeBlock data structures.
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLogF("JIT code for %p start at [%p, %p). Size = %zu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize());
#endif
// Link all calls out from the JIT code to their respective functions.
for (unsigned i = 0; i < m_calls.size(); ++i)
linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);
m_codeBlock->callReturnIndexVector().reserveCapacity(m_exceptionChecks.size());
for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) {
unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call);
CodeOrigin codeOrigin = m_exceptionChecks[i].m_codeOrigin;
while (codeOrigin.inlineCallFrame)
codeOrigin = codeOrigin.inlineCallFrame->caller;
unsigned exceptionInfo = codeOrigin.bytecodeIndex;
m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo));
}
Vector<CodeOriginAtCallReturnOffset, 0, UnsafeVectorOverflow>& codeOrigins = m_codeBlock->codeOrigins();
codeOrigins.resize(m_exceptionChecks.size());
for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) {
CallExceptionRecord& record = m_exceptionChecks[i];
unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call);
codeOrigins[i].codeOrigin = record.m_codeOrigin;
codeOrigins[i].callReturnOffset = returnAddressOffset;
}
m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size());
for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) {
StructureStubInfo& info = m_codeBlock->structureStubInfo(i);
CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->call());
info.codeOrigin = m_propertyAccesses[i].m_codeOrigin;
info.callReturnLocation = callReturnLocation;
info.patch.dfg.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_structureImm), callReturnLocation);
info.patch.dfg.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_structureCheck));
#if USE(JSVALUE64)
info.patch.dfg.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_loadOrStore));
#else
info.patch.dfg.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_tagLoadOrStore));
info.patch.dfg.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_payloadLoadOrStore));
#endif
info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->label()));
info.patch.dfg.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_done));
info.patch.dfg.deltaCallToStorageLoad = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_propertyStorageLoad));
info.patch.dfg.baseGPR = m_propertyAccesses[i].m_baseGPR;
#if USE(JSVALUE64)
info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR;
#else
info.patch.dfg.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR;
info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR;
#endif
m_propertyAccesses[i].m_usedRegisters.copyInfo(info.patch.dfg.usedRegisters);
info.patch.dfg.registersFlushed = m_propertyAccesses[i].m_registerMode == PropertyAccessRecord::RegistersFlushed;
}
m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size());
for (unsigned i = 0; i < m_jsCalls.size(); ++i) {
CallLinkInfo& info = m_codeBlock->callLinkInfo(i);
info.callType = m_jsCalls[i].m_callType;
info.isDFG = true;
info.codeOrigin = m_jsCalls[i].m_codeOrigin;
linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_vm->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress()));
info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall);
info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck);
info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall);
info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee);
}
MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator);
CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
for (unsigned i = 0; i < codeBlock()->numberOfOSRExits(); ++i) {
OSRExit& exit = codeBlock()->osrExit(i);
linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target);
exit.correctJump(linkBuffer);
if (exit.m_watchpointIndex != std::numeric_limits<unsigned>::max())
codeBlock()->watchpoint(exit.m_watchpointIndex).correctLabels(linkBuffer);
}
if (m_graph.m_compilation) {
ASSERT(m_exitSiteLabels.size() == codeBlock()->numberOfOSRExits());
for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) {
Vector<Label>& labels = m_exitSiteLabels[i];
Vector<const void*> addresses;
for (unsigned j = 0; j < labels.size(); ++j)
addresses.append(linkBuffer.locationOf(labels[j]).executableAddress());
m_graph.m_compilation->addOSRExitSite(addresses);
}
} else
ASSERT(!m_exitSiteLabels.size());
codeBlock()->saveCompilation(m_graph.m_compilation);
codeBlock()->shrinkToFit(CodeBlock::LateShrink);
}
void JITCompiler::link(LinkBuffer& linkBuffer)
{
// Link the code, populate data in CodeBlock data structures.
m_jitCode->common.frameRegisterCount = m_graph.frameRegisterCount();
m_jitCode->common.requiredRegisterCountForExit = m_graph.requiredRegisterCountForExit();
if (!m_graph.m_plan.inlineCallFrames->isEmpty())
m_jitCode->common.inlineCallFrames = m_graph.m_plan.inlineCallFrames;
#if USE(JSVALUE32_64)
m_jitCode->common.doubleConstants = WTFMove(m_graph.m_doubleConstants);
#endif
m_graph.registerFrozenValues();
BitVector usedJumpTables;
for (Bag<SwitchData>::iterator iter = m_graph.m_switchData.begin(); !!iter; ++iter) {
SwitchData& data = **iter;
if (!data.didUseJumpTable)
continue;
if (data.kind == SwitchString)
continue;
RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar);
usedJumpTables.set(data.switchTableIndex);
SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex);
table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough.block->index]);
table.ctiOffsets.grow(table.branchOffsets.size());
for (unsigned j = table.ctiOffsets.size(); j--;)
table.ctiOffsets[j] = table.ctiDefault;
for (unsigned j = data.cases.size(); j--;) {
SwitchCase& myCase = data.cases[j];
table.ctiOffsets[myCase.value.switchLookupValue(data.kind) - table.min] =
linkBuffer.locationOf(m_blockHeads[myCase.target.block->index]);
}
}
for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) {
if (usedJumpTables.get(i))
continue;
m_codeBlock->switchJumpTable(i).clear();
}
// NOTE: we cannot clear string switch tables because (1) we're running concurrently
// and we cannot deref StringImpl's and (2) it would be weird to deref those
// StringImpl's since we refer to them.
for (Bag<SwitchData>::iterator switchDataIter = m_graph.m_switchData.begin(); !!switchDataIter; ++switchDataIter) {
SwitchData& data = **switchDataIter;
if (!data.didUseJumpTable)
continue;
if (data.kind != SwitchString)
continue;
StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex);
table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough.block->index]);
StringJumpTable::StringOffsetTable::iterator iter;
StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end();
for (iter = table.offsetTable.begin(); iter != end; ++iter)
iter->value.ctiOffset = table.ctiDefault;
for (unsigned j = data.cases.size(); j--;) {
SwitchCase& myCase = data.cases[j];
iter = table.offsetTable.find(myCase.value.stringImpl());
RELEASE_ASSERT(iter != end);
iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target.block->index]);
}
}
// Link all calls out from the JIT code to their respective functions.
for (unsigned i = 0; i < m_calls.size(); ++i)
linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);
for (unsigned i = m_getByIds.size(); i--;)
m_getByIds[i].finalize(linkBuffer);
for (unsigned i = m_putByIds.size(); i--;)
m_putByIds[i].finalize(linkBuffer);
for (unsigned i = 0; i < m_ins.size(); ++i) {
StructureStubInfo& info = *m_ins[i].m_stubInfo;
CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call());
info.patch.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_done));
info.patch.deltaCallToJump = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_jump));
info.callReturnLocation = callReturnLocation;
info.patch.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label()));
}
for (unsigned i = 0; i < m_jsCalls.size(); ++i) {
JSCallRecord& record = m_jsCalls[i];
CallLinkInfo& info = *record.m_info;
linkBuffer.link(record.m_slowCall, FunctionPtr(m_vm->getCTIStub(linkCallThunkGenerator).code().executableAddress()));
info.setCallLocations(linkBuffer.locationOfNearCall(record.m_slowCall),
linkBuffer.locationOf(record.m_targetToCheck),
linkBuffer.locationOfNearCall(record.m_fastCall));
}
MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator);
CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) {
OSRExit& exit = m_jitCode->osrExit[i];
OSRExitCompilationInfo& info = m_exitCompilationInfo[i];
linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target);
exit.correctJump(linkBuffer);
if (info.m_replacementSource.isSet()) {
m_jitCode->common.jumpReplacements.append(JumpReplacement(
linkBuffer.locationOf(info.m_replacementSource),
linkBuffer.locationOf(info.m_replacementDestination)));
}
}
if (m_graph.compilation()) {
ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size());
for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) {
Vector<Label>& labels = m_exitSiteLabels[i];
Vector<const void*> addresses;
for (unsigned j = 0; j < labels.size(); ++j)
addresses.append(linkBuffer.locationOf(labels[j]).executableAddress());
m_graph.compilation()->addOSRExitSite(addresses);
}
} else
ASSERT(!m_exitSiteLabels.size());
m_jitCode->common.compilation = m_graph.compilation();
// Link new DFG exception handlers and remove baseline JIT handlers.
m_codeBlock->clearExceptionHandlers();
for (unsigned i = 0; i < m_exceptionHandlerOSRExitCallSites.size(); i++) {
OSRExitCompilationInfo& info = m_exceptionHandlerOSRExitCallSites[i].exitInfo;
if (info.m_replacementDestination.isSet()) {
// If this is is *not* set, it means that we already jumped to the OSR exit in pure generated control flow.
// i.e, we explicitly emitted an exceptionCheck that we know will be caught in this machine frame.
// If this *is set*, it means we will be landing at this code location from genericUnwind from an
// exception thrown in a child call frame.
CodeLocationLabel catchLabel = linkBuffer.locationOf(info.m_replacementDestination);
HandlerInfo newExceptionHandler = m_exceptionHandlerOSRExitCallSites[i].baselineExceptionHandler;
CallSiteIndex callSite = m_exceptionHandlerOSRExitCallSites[i].callSiteIndex;
newExceptionHandler.start = callSite.bits();
newExceptionHandler.end = callSite.bits() + 1;
newExceptionHandler.nativeCode = catchLabel;
m_codeBlock->appendExceptionHandler(newExceptionHandler);
}
}
if (m_pcToCodeOriginMapBuilder.didBuildMapping())
m_codeBlock->setPCToCodeOriginMap(std::make_unique<PCToCodeOriginMap>(WTFMove(m_pcToCodeOriginMapBuilder), linkBuffer));
}
void JITCompiler::link(LinkBuffer& linkBuffer)
{
// Link the code, populate data in CodeBlock data structures.
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLogF("JIT code for %p start at [%p, %p). Size = %zu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize());
#endif
if (!m_graph.m_inlineCallFrames->isEmpty()) {
m_graph.m_inlineCallFrames->shrinkToFit();
m_jitCode->common.inlineCallFrames = m_graph.m_inlineCallFrames.release();
}
BitVector usedJumpTables;
for (unsigned i = m_graph.m_switchData.size(); i--;) {
SwitchData& data = m_graph.m_switchData[i];
if (!data.didUseJumpTable)
continue;
if (data.kind == SwitchString)
continue;
RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar);
usedJumpTables.set(data.switchTableIndex);
SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex);
table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]);
table.ctiOffsets.grow(table.branchOffsets.size());
for (unsigned j = table.ctiOffsets.size(); j--;)
table.ctiOffsets[j] = table.ctiDefault;
for (unsigned j = data.cases.size(); j--;) {
SwitchCase& myCase = data.cases[j];
table.ctiOffsets[myCase.value.switchLookupValue() - table.min] =
linkBuffer.locationOf(m_blockHeads[myCase.target->index]);
}
}
for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) {
if (usedJumpTables.get(i))
continue;
m_codeBlock->switchJumpTable(i).clear();
}
// NOTE: we cannot clear string switch tables because (1) we're running concurrently
// and we cannot deref StringImpl's and (2) it would be weird to deref those
// StringImpl's since we refer to them.
for (unsigned i = m_graph.m_switchData.size(); i--;) {
SwitchData& data = m_graph.m_switchData[i];
if (!data.didUseJumpTable)
continue;
if (data.kind != SwitchString)
continue;
StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex);
table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]);
StringJumpTable::StringOffsetTable::iterator iter;
StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end();
for (iter = table.offsetTable.begin(); iter != end; ++iter)
iter->value.ctiOffset = table.ctiDefault;
for (unsigned j = data.cases.size(); j--;) {
SwitchCase& myCase = data.cases[j];
iter = table.offsetTable.find(myCase.value.stringImpl());
RELEASE_ASSERT(iter != end);
iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target->index]);
}
}
// Link all calls out from the JIT code to their respective functions.
for (unsigned i = 0; i < m_calls.size(); ++i)
linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);
m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size() + m_ins.size());
for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) {
StructureStubInfo& info = m_codeBlock->structureStubInfo(i);
CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->call());
info.codeOrigin = m_propertyAccesses[i].m_codeOrigin;
info.callReturnLocation = callReturnLocation;
info.patch.dfg.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_structureImm), callReturnLocation);
info.patch.dfg.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_structureCheck));
#if USE(JSVALUE64)
info.patch.dfg.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_loadOrStore));
#else
info.patch.dfg.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_tagLoadOrStore));
info.patch.dfg.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_payloadLoadOrStore));
#endif
info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->label()));
info.patch.dfg.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_done));
info.patch.dfg.deltaCallToStorageLoad = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_propertyStorageLoad));
info.patch.dfg.baseGPR = m_propertyAccesses[i].m_baseGPR;
#if USE(JSVALUE64)
info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR;
#else
info.patch.dfg.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR;
info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR;
#endif
m_propertyAccesses[i].m_usedRegisters.copyInfo(info.patch.dfg.usedRegisters);
info.patch.dfg.registersFlushed = m_propertyAccesses[i].m_registerMode == PropertyAccessRecord::RegistersFlushed;
}
for (unsigned i = 0; i < m_ins.size(); ++i) {
StructureStubInfo& info = m_codeBlock->structureStubInfo(m_propertyAccesses.size() + i);
CodeLocationLabel jump = linkBuffer.locationOf(m_ins[i].m_jump);
CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call());
info.codeOrigin = m_ins[i].m_codeOrigin;
info.hotPathBegin = jump;
info.callReturnLocation = callReturnLocation;
info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label()));
info.patch.dfg.baseGPR = m_ins[i].m_baseGPR;
info.patch.dfg.valueGPR = m_ins[i].m_resultGPR;
m_ins[i].m_usedRegisters.copyInfo(info.patch.dfg.usedRegisters);
info.patch.dfg.registersFlushed = false;
}
m_codeBlock->sortStructureStubInfos();
m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size());
for (unsigned i = 0; i < m_jsCalls.size(); ++i) {
CallLinkInfo& info = m_codeBlock->callLinkInfo(i);
info.callType = m_jsCalls[i].m_callType;
info.isDFG = true;
info.codeOrigin = m_jsCalls[i].m_codeOrigin;
linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_vm->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress()));
info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall);
info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck);
info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall);
info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee);
}
MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator);
CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) {
OSRExit& exit = m_jitCode->osrExit[i];
linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target);
exit.correctJump(linkBuffer);
if (exit.m_watchpointIndex != std::numeric_limits<unsigned>::max())
m_jitCode->watchpoints[exit.m_watchpointIndex].correctLabels(linkBuffer);
}
if (m_graph.compilation()) {
ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size());
for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) {
Vector<Label>& labels = m_exitSiteLabels[i];
Vector<const void*> addresses;
for (unsigned j = 0; j < labels.size(); ++j)
addresses.append(linkBuffer.locationOf(labels[j]).executableAddress());
m_graph.compilation()->addOSRExitSite(addresses);
}
} else
ASSERT(!m_exitSiteLabels.size());
m_jitCode->common.compilation = m_graph.compilation();
}
void JITCompiler::link(LinkBuffer& linkBuffer)
{
// Link the code, populate data in CodeBlock data structures.
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLog("JIT code for %p start at [%p, %p). Size = %zu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize());
#endif
// Link all calls out from the JIT code to their respective functions.
for (unsigned i = 0; i < m_calls.size(); ++i)
linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);
m_codeBlock->callReturnIndexVector().reserveCapacity(m_exceptionChecks.size());
for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) {
unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call);
CodeOrigin codeOrigin = m_exceptionChecks[i].m_codeOrigin;
while (codeOrigin.inlineCallFrame)
codeOrigin = codeOrigin.inlineCallFrame->caller;
unsigned exceptionInfo = codeOrigin.bytecodeIndex;
m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo));
}
Vector<CodeOriginAtCallReturnOffset>& codeOrigins = m_codeBlock->codeOrigins();
codeOrigins.resize(m_exceptionChecks.size());
for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) {
CallExceptionRecord& record = m_exceptionChecks[i];
unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call);
codeOrigins[i].codeOrigin = record.m_codeOrigin;
codeOrigins[i].callReturnOffset = returnAddressOffset;
record.m_token.assertCodeOriginIndex(i);
}
m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size());
for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) {
StructureStubInfo& info = m_codeBlock->structureStubInfo(i);
CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->call());
info.codeOrigin = m_propertyAccesses[i].m_codeOrigin;
info.callReturnLocation = callReturnLocation;
info.patch.dfg.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_structureImm), callReturnLocation);
info.patch.dfg.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_structureCheck));
#if USE(JSVALUE64)
info.patch.dfg.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_loadOrStore));
#else
info.patch.dfg.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_tagLoadOrStore));
info.patch.dfg.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_payloadLoadOrStore));
#endif
info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->label()));
info.patch.dfg.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_done));
info.patch.dfg.baseGPR = m_propertyAccesses[i].m_baseGPR;
#if USE(JSVALUE64)
info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR;
#else
info.patch.dfg.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR;
info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR;
#endif
info.patch.dfg.scratchGPR = m_propertyAccesses[i].m_scratchGPR;
info.patch.dfg.registersFlushed = m_propertyAccesses[i].m_registerMode == PropertyAccessRecord::RegistersFlushed;
}
m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size());
for (unsigned i = 0; i < m_jsCalls.size(); ++i) {
CallLinkInfo& info = m_codeBlock->callLinkInfo(i);
info.callType = m_jsCalls[i].m_callType;
info.isDFG = true;
info.callReturnLocation = CodeLocationLabel(linkBuffer.locationOf(m_jsCalls[i].m_slowCall));
info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck);
info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall);
}
MacroAssemblerCodeRef osrExitThunk = globalData()->getCTIStub(osrExitGenerationThunkGenerator);
CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
for (unsigned i = 0; i < codeBlock()->numberOfOSRExits(); ++i) {
OSRExit& exit = codeBlock()->osrExit(i);
linkBuffer.link(exit.m_check.lateJump(), target);
exit.m_check.correctLateJump(linkBuffer);
if (exit.m_watchpointIndex != std::numeric_limits<unsigned>::max())
codeBlock()->watchpoint(exit.m_watchpointIndex).correctLabels(linkBuffer);
}
codeBlock()->shrinkToFit(CodeBlock::LateShrink);
}
void JITCompiler::link(LinkBuffer& linkBuffer)
{
// Link the code, populate data in CodeBlock data structures.
m_jitCode->common.frameRegisterCount = m_graph.frameRegisterCount();
m_jitCode->common.requiredRegisterCountForExit = m_graph.requiredRegisterCountForExit();
if (!m_graph.m_inlineCallFrames->isEmpty())
m_jitCode->common.inlineCallFrames = m_graph.m_inlineCallFrames.release();
m_jitCode->common.machineCaptureStart = m_graph.m_machineCaptureStart;
m_jitCode->common.slowArguments = std::move(m_graph.m_slowArguments);
BitVector usedJumpTables;
for (unsigned i = m_graph.m_switchData.size(); i--;) {
SwitchData& data = m_graph.m_switchData[i];
if (!data.didUseJumpTable)
continue;
if (data.kind == SwitchString)
continue;
RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar);
usedJumpTables.set(data.switchTableIndex);
SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex);
table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]);
table.ctiOffsets.grow(table.branchOffsets.size());
for (unsigned j = table.ctiOffsets.size(); j--;)
table.ctiOffsets[j] = table.ctiDefault;
for (unsigned j = data.cases.size(); j--;) {
SwitchCase& myCase = data.cases[j];
table.ctiOffsets[myCase.value.switchLookupValue() - table.min] =
linkBuffer.locationOf(m_blockHeads[myCase.target->index]);
}
}
for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) {
if (usedJumpTables.get(i))
continue;
m_codeBlock->switchJumpTable(i).clear();
}
// NOTE: we cannot clear string switch tables because (1) we're running concurrently
// and we cannot deref StringImpl's and (2) it would be weird to deref those
// StringImpl's since we refer to them.
for (unsigned i = m_graph.m_switchData.size(); i--;) {
SwitchData& data = m_graph.m_switchData[i];
if (!data.didUseJumpTable)
continue;
if (data.kind != SwitchString)
continue;
StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex);
table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]);
StringJumpTable::StringOffsetTable::iterator iter;
StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end();
for (iter = table.offsetTable.begin(); iter != end; ++iter)
iter->value.ctiOffset = table.ctiDefault;
for (unsigned j = data.cases.size(); j--;) {
SwitchCase& myCase = data.cases[j];
iter = table.offsetTable.find(myCase.value.stringImpl());
RELEASE_ASSERT(iter != end);
iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target->index]);
}
}
// Link all calls out from the JIT code to their respective functions.
for (unsigned i = 0; i < m_calls.size(); ++i)
linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);
for (unsigned i = m_getByIds.size(); i--;)
m_getByIds[i].finalize(linkBuffer);
for (unsigned i = m_putByIds.size(); i--;)
m_putByIds[i].finalize(linkBuffer);
for (unsigned i = 0; i < m_ins.size(); ++i) {
StructureStubInfo& info = *m_ins[i].m_stubInfo;
CodeLocationLabel jump = linkBuffer.locationOf(m_ins[i].m_jump);
CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call());
info.hotPathBegin = jump;
info.callReturnLocation = callReturnLocation;
info.patch.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label()));
}
m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size());
for (unsigned i = 0; i < m_jsCalls.size(); ++i) {
CallLinkInfo& info = m_codeBlock->callLinkInfo(i);
info.callType = m_jsCalls[i].m_callType;
info.isDFG = true;
info.codeOrigin = m_jsCalls[i].m_codeOrigin;
linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_vm->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress()));
info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall);
info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck);
info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall);
info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee);
}
MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator);
CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) {
OSRExit& exit = m_jitCode->osrExit[i];
OSRExitCompilationInfo& info = m_exitCompilationInfo[i];
linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target);
exit.correctJump(linkBuffer);
if (info.m_replacementSource.isSet()) {
m_jitCode->common.jumpReplacements.append(JumpReplacement(
linkBuffer.locationOf(info.m_replacementSource),
linkBuffer.locationOf(info.m_replacementDestination)));
}
}
if (m_graph.compilation()) {
ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size());
for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) {
Vector<Label>& labels = m_exitSiteLabels[i];
Vector<const void*> addresses;
for (unsigned j = 0; j < labels.size(); ++j)
addresses.append(linkBuffer.locationOf(labels[j]).executableAddress());
m_graph.compilation()->addOSRExitSite(addresses);
}
} else
ASSERT(!m_exitSiteLabels.size());
m_jitCode->common.compilation = m_graph.compilation();
}
void JITCompiler::link(LinkBuffer& linkBuffer)
{
// Link the code, populate data in CodeBlock data structures.
#if DFG_ENABLE(DEBUG_VERBOSE)
fprintf(stderr, "JIT code for %p start at [%p, %p). Size = %lu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize());
#endif
// Link all calls out from the JIT code to their respective functions.
for (unsigned i = 0; i < m_calls.size(); ++i)
linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function);
if (m_codeBlock->needsCallReturnIndices()) {
m_codeBlock->callReturnIndexVector().reserveCapacity(m_exceptionChecks.size());
for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) {
unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call);
CodeOrigin codeOrigin = m_exceptionChecks[i].m_codeOrigin;
while (codeOrigin.inlineCallFrame)
codeOrigin = codeOrigin.inlineCallFrame->caller;
unsigned exceptionInfo = codeOrigin.bytecodeIndex;
m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo));
}
}
unsigned numCallsFromInlineCode = 0;
for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) {
if (m_exceptionChecks[i].m_codeOrigin.inlineCallFrame)
numCallsFromInlineCode++;
}
if (numCallsFromInlineCode) {
Vector<CodeOriginAtCallReturnOffset>& codeOrigins = m_codeBlock->codeOrigins();
codeOrigins.resize(numCallsFromInlineCode);
for (unsigned i = 0, j = 0; i < m_exceptionChecks.size(); ++i) {
CallExceptionRecord& record = m_exceptionChecks[i];
if (record.m_codeOrigin.inlineCallFrame) {
unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call);
codeOrigins[j].codeOrigin = record.m_codeOrigin;
codeOrigins[j].callReturnOffset = returnAddressOffset;
j++;
}
}
}
m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size());
for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) {
StructureStubInfo& info = m_codeBlock->structureStubInfo(i);
CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_functionCall);
info.callReturnLocation = callReturnLocation;
info.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCheckImmToCall), callReturnLocation);
info.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToStructCheck));
#if USE(JSVALUE64)
info.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToLoadOrStore));
#else
info.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToTagLoadOrStore));
info.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToPayloadLoadOrStore));
#endif
info.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToSlowCase));
info.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToDone));
info.baseGPR = m_propertyAccesses[i].m_baseGPR;
#if USE(JSVALUE64)
info.valueGPR = m_propertyAccesses[i].m_valueGPR;
#else
info.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR;
info.valueGPR = m_propertyAccesses[i].m_valueGPR;
#endif
info.scratchGPR = m_propertyAccesses[i].m_scratchGPR;
}
m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size());
for (unsigned i = 0; i < m_jsCalls.size(); ++i) {
CallLinkInfo& info = m_codeBlock->callLinkInfo(i);
info.callType = m_jsCalls[i].m_callType;
info.isDFG = true;
info.callReturnLocation = CodeLocationLabel(linkBuffer.locationOf(m_jsCalls[i].m_slowCall));
info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck);
info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall);
}
MacroAssemblerCodeRef osrExitThunk = globalData()->getCTIStub(osrExitGenerationThunkGenerator);
CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code());
for (unsigned i = 0; i < codeBlock()->numberOfOSRExits(); ++i) {
OSRExit& exit = codeBlock()->osrExit(i);
linkBuffer.link(exit.m_check.lateJump(), target);
exit.m_check.correctLateJump(linkBuffer);
}
codeBlock()->shrinkWeakReferencesToFit();
codeBlock()->shrinkWeakReferenceTransitionsToFit();
}
