void compile(State& state, Safepoint::Result& safepointResult)
{
char* error = 0;
{
GraphSafepoint safepoint(state.graph, safepointResult);
LLVMMCJITCompilerOptions options;
llvm->InitializeMCJITCompilerOptions(&options, sizeof(options));
options.OptLevel = Options::llvmBackendOptimizationLevel();
options.NoFramePointerElim = true;
if (Options::useLLVMSmallCodeModel())
options.CodeModel = LLVMCodeModelSmall;
options.EnableFastISel = Options::enableLLVMFastISel();
options.MCJMM = llvm->CreateSimpleMCJITMemoryManager(
&state, mmAllocateCodeSection, mmAllocateDataSection, mmApplyPermissions, mmDestroy);
LLVMExecutionEngineRef engine;
if (isARM64())
#if OS(DARWIN)
llvm->SetTarget(state.module, "arm64-apple-ios");
#elif OS(LINUX)
llvm->SetTarget(state.module, "aarch64-linux-gnu");
#else
#error "Unrecognized OS"
#endif
if (llvm->CreateMCJITCompilerForModule(&engine, state.module, &options, sizeof(options), &error)) {
dataLog("FATAL: Could not create LLVM execution engine: ", error, "\n");
CRASH();
}
// The data layout also has to be set in the module. Get the data layout from the MCJIT and apply
// it to the module.
LLVMTargetMachineRef targetMachine = llvm->GetExecutionEngineTargetMachine(engine);
LLVMTargetDataRef targetData = llvm->GetExecutionEngineTargetData(engine);
char* stringRepOfTargetData = llvm->CopyStringRepOfTargetData(targetData);
llvm->SetDataLayout(state.module, stringRepOfTargetData);
free(stringRepOfTargetData);
LLVMPassManagerRef functionPasses = 0;
LLVMPassManagerRef modulePasses;
if (Options::llvmSimpleOpt()) {
modulePasses = llvm->CreatePassManager();
llvm->AddTargetData(targetData, modulePasses);
llvm->AddAnalysisPasses(targetMachine, modulePasses);
llvm->AddPromoteMemoryToRegisterPass(modulePasses);
llvm->AddGlobalOptimizerPass(modulePasses);
llvm->AddFunctionInliningPass(modulePasses);
llvm->AddPruneEHPass(modulePasses);
llvm->AddGlobalDCEPass(modulePasses);
llvm->AddConstantPropagationPass(modulePasses);
llvm->AddAggressiveDCEPass(modulePasses);
llvm->AddInstructionCombiningPass(modulePasses);
// BEGIN - DO NOT CHANGE THE ORDER OF THE ALIAS ANALYSIS PASSES
llvm->AddTypeBasedAliasAnalysisPass(modulePasses);
llvm->AddBasicAliasAnalysisPass(modulePasses);
// END - DO NOT CHANGE THE ORDER OF THE ALIAS ANALYSIS PASSES
llvm->AddGVNPass(modulePasses);
llvm->AddCFGSimplificationPass(modulePasses);
llvm->AddDeadStoreEliminationPass(modulePasses);
llvm->RunPassManager(modulePasses, state.module);
} else {
LLVMPassManagerBuilderRef passBuilder = llvm->PassManagerBuilderCreate();
llvm->PassManagerBuilderSetOptLevel(passBuilder, Options::llvmOptimizationLevel());
llvm->PassManagerBuilderUseInlinerWithThreshold(passBuilder, 275);
llvm->PassManagerBuilderSetSizeLevel(passBuilder, Options::llvmSizeLevel());
functionPasses = llvm->CreateFunctionPassManagerForModule(state.module);
modulePasses = llvm->CreatePassManager();
llvm->AddTargetData(llvm->GetExecutionEngineTargetData(engine), modulePasses);
llvm->PassManagerBuilderPopulateFunctionPassManager(passBuilder, functionPasses);
llvm->PassManagerBuilderPopulateModulePassManager(passBuilder, modulePasses);
llvm->PassManagerBuilderDispose(passBuilder);
llvm->InitializeFunctionPassManager(functionPasses);
for (LValue function = llvm->GetFirstFunction(state.module); function; function = llvm->GetNextFunction(function))
llvm->RunFunctionPassManager(functionPasses, function);
llvm->FinalizeFunctionPassManager(functionPasses);
llvm->RunPassManager(modulePasses, state.module);
}
if (shouldShowDisassembly() || verboseCompilationEnabled())
state.dumpState("after optimization");
// FIXME: Need to add support for the case where JIT memory allocation failed.
// https://bugs.webkit.org/show_bug.cgi?id=113620
state.generatedFunction = reinterpret_cast<GeneratedFunction>(llvm->GetPointerToGlobal(engine, state.function));
if (functionPasses)
llvm->DisposePassManager(functionPasses);
llvm->DisposePassManager(modulePasses);
llvm->DisposeExecutionEngine(engine);
}
if (safepointResult.didGetCancelled())
return;
RELEASE_ASSERT(!state.graph.m_vm.heap.isCollecting());
if (shouldShowDisassembly()) {
for (unsigned i = 0; i < state.jitCode->handles().size(); ++i) {
ExecutableMemoryHandle* handle = state.jitCode->handles()[i].get();
dataLog(
"Generated LLVM code for ",
CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT),
" #", i, ", ", state.codeSectionNames[i], ":\n");
disassemble(
MacroAssemblerCodePtr(handle->start()), handle->sizeInBytes(),
" ", WTF::dataFile(), LLVMSubset);
}
for (unsigned i = 0; i < state.jitCode->dataSections().size(); ++i) {
DataSection* section = state.jitCode->dataSections()[i].get();
dataLog(
"Generated LLVM data section for ",
CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT),
" #", i, ", ", state.dataSectionNames[i], ":\n");
dumpDataSection(section, " ");
}
}
bool didSeeUnwindInfo = state.jitCode->unwindInfo.parse(
state.unwindDataSection, state.unwindDataSectionSize,
state.generatedFunction);
if (shouldShowDisassembly()) {
dataLog("Unwind info for ", CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT), ":\n");
if (didSeeUnwindInfo)
dataLog(" ", state.jitCode->unwindInfo, "\n");
else
dataLog(" <no unwind info>\n");
}
if (state.stackmapsSection && state.stackmapsSection->size()) {
if (shouldShowDisassembly()) {
dataLog(
"Generated LLVM stackmaps section for ",
CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT), ":\n");
dataLog(" Raw data:\n");
dumpDataSection(state.stackmapsSection.get(), " ");
}
RefPtr<DataView> stackmapsData = DataView::create(
ArrayBuffer::create(state.stackmapsSection->base(), state.stackmapsSection->size()));
state.jitCode->stackmaps.parse(stackmapsData.get());
if (shouldShowDisassembly()) {
dataLog(" Structured data:\n");
state.jitCode->stackmaps.dumpMultiline(WTF::dataFile(), " ");
}
StackMaps::RecordMap recordMap = state.jitCode->stackmaps.computeRecordMap();
fixFunctionBasedOnStackMaps(
state, state.graph.m_codeBlock, state.jitCode.get(), state.generatedFunction,
recordMap, didSeeUnwindInfo);
if (shouldShowDisassembly()) {
for (unsigned i = 0; i < state.jitCode->handles().size(); ++i) {
if (state.codeSectionNames[i] != SECTION_NAME("text"))
continue;
ExecutableMemoryHandle* handle = state.jitCode->handles()[i].get();
dataLog(
"Generated LLVM code after stackmap-based fix-up for ",
CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::FTLJIT),
" in ", state.graph.m_plan.mode, " #", i, ", ",
state.codeSectionNames[i], ":\n");
disassemble(
MacroAssemblerCodePtr(handle->start()), handle->sizeInBytes(),
" ", WTF::dataFile(), LLVMSubset);
}
}
}
state.module = 0; // We no longer own the module.
}
void compile(State& state)
{
char* error = 0;
LLVMMCJITCompilerOptions options;
LLVMInitializeMCJITCompilerOptions(&options, sizeof(options));
options.OptLevel = Options::llvmBackendOptimizationLevel();
options.NoFramePointerElim = true;
if (Options::useLLVMSmallCodeModel())
options.CodeModel = LLVMCodeModelSmall;
options.EnableFastISel = Options::enableLLVMFastISel();
options.MCJMM = LLVMCreateSimpleMCJITMemoryManager(
&state, mmAllocateCodeSection, mmAllocateDataSection, mmApplyPermissions, mmDestroy);
LLVMExecutionEngineRef engine;
if (LLVMCreateMCJITCompilerForModule(&engine, state.module, &options, sizeof(options), &error)) {
dataLog("FATAL: Could not create LLVM execution engine: ", error, "\n");
CRASH();
}
LLVMPassManagerBuilderRef passBuilder = LLVMPassManagerBuilderCreate();
LLVMPassManagerBuilderSetOptLevel(passBuilder, Options::llvmOptimizationLevel());
LLVMPassManagerBuilderSetSizeLevel(passBuilder, Options::llvmSizeLevel());
LLVMPassManagerRef functionPasses = LLVMCreateFunctionPassManagerForModule(state.module);
LLVMPassManagerRef modulePasses = LLVMCreatePassManager();
LLVMAddTargetData(LLVMGetExecutionEngineTargetData(engine), modulePasses);
LLVMPassManagerBuilderPopulateFunctionPassManager(passBuilder, functionPasses);
LLVMPassManagerBuilderPopulateModulePassManager(passBuilder, modulePasses);
LLVMPassManagerBuilderDispose(passBuilder);
LLVMInitializeFunctionPassManager(functionPasses);
for (LValue function = LLVMGetFirstFunction(state.module); function; function = LLVMGetNextFunction(function))
LLVMRunFunctionPassManager(functionPasses, function);
LLVMFinalizeFunctionPassManager(functionPasses);
LLVMRunPassManager(modulePasses, state.module);
if (DFG::shouldShowDisassembly() || DFG::verboseCompilationEnabled())
state.dumpState("after optimization");
// FIXME: Need to add support for the case where JIT memory allocation failed.
// https://bugs.webkit.org/show_bug.cgi?id=113620
state.generatedFunction = reinterpret_cast<GeneratedFunction>(LLVMGetPointerToGlobal(engine, state.function));
LLVMDisposePassManager(functionPasses);
LLVMDisposePassManager(modulePasses);
LLVMDisposeExecutionEngine(engine);
if (shouldShowDisassembly()) {
// FIXME: fourthTier: FTL memory allocator should be able to tell us which of
// these things is actually code or data.
// https://bugs.webkit.org/show_bug.cgi?id=116189
for (unsigned i = 0; i < state.jitCode->handles().size(); ++i) {
ExecutableMemoryHandle* handle = state.jitCode->handles()[i].get();
dataLog(
"Generated LLVM code for ",
CodeBlockWithJITType(state.graph.m_codeBlock, JITCode::DFGJIT),
" #", i, ":\n");
disassemble(
MacroAssemblerCodePtr(handle->start()), handle->sizeInBytes(),
" ", WTF::dataFile(), LLVMSubset);
}
}
}
void link(State& state)
{
Graph& graph = state.graph;
CodeBlock* codeBlock = graph.m_codeBlock;
VM& vm = graph.m_vm;
// LLVM will create its own jump tables as needed.
codeBlock->clearSwitchJumpTables();
#if !FTL_USES_B3
// What LLVM's stackmaps call stackSizeForLocals and what we call frameRegisterCount have a simple
// relationship, though it's not obvious from reading the code. The easiest way to understand them
// is to look at stackOffset, i.e. what you have to add to FP to get SP. For LLVM that is just:
//
// stackOffset == -state.jitCode->stackmaps.stackSizeForLocals()
//
// The way we define frameRegisterCount is that it satisfies this equality:
//
// stackOffset == virtualRegisterForLocal(frameRegisterCount - 1).offset() * sizeof(Register)
//
// We can simplify this when we apply virtualRegisterForLocal():
//
// stackOffset == (-1 - (frameRegisterCount - 1)) * sizeof(Register)
// stackOffset == (-1 - frameRegisterCount + 1) * sizeof(Register)
// stackOffset == -frameRegisterCount * sizeof(Register)
//
// Therefore we just have:
//
// frameRegisterCount == -stackOffset / sizeof(Register)
//
// If we substitute what we have above, we get:
//
// frameRegisterCount == -(-state.jitCode->stackmaps.stackSizeForLocals()) / sizeof(Register)
// frameRegisterCount == state.jitCode->stackmaps.stackSizeForLocals() / sizeof(Register)
state.jitCode->common.frameRegisterCount = state.jitCode->stackmaps.stackSizeForLocals() / sizeof(void*);
#endif
state.jitCode->common.requiredRegisterCountForExit = graph.requiredRegisterCountForExit();
if (!graph.m_plan.inlineCallFrames->isEmpty())
state.jitCode->common.inlineCallFrames = graph.m_plan.inlineCallFrames;
graph.registerFrozenValues();
// Create the entrypoint. Note that we use this entrypoint totally differently
// depending on whether we're doing OSR entry or not.
CCallHelpers jit(&vm, codeBlock);
std::unique_ptr<LinkBuffer> linkBuffer;
CCallHelpers::Address frame = CCallHelpers::Address(
CCallHelpers::stackPointerRegister, -static_cast<int32_t>(AssemblyHelpers::prologueStackPointerDelta()));
if (Profiler::Compilation* compilation = graph.compilation()) {
compilation->addDescription(
Profiler::OriginStack(),
toCString("Generated FTL JIT code for ", CodeBlockWithJITType(codeBlock, JITCode::FTLJIT), ", instruction count = ", graph.m_codeBlock->instructionCount(), ":\n"));
graph.ensureDominators();
graph.ensureNaturalLoops();
const char* prefix = " ";
DumpContext dumpContext;
StringPrintStream out;
Node* lastNode = 0;
for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); ++blockIndex) {
BasicBlock* block = graph.block(blockIndex);
if (!block)
continue;
graph.dumpBlockHeader(out, prefix, block, Graph::DumpLivePhisOnly, &dumpContext);
compilation->addDescription(Profiler::OriginStack(), out.toCString());
out.reset();
for (size_t nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
Node* node = block->at(nodeIndex);
Profiler::OriginStack stack;
if (node->origin.semantic.isSet()) {
stack = Profiler::OriginStack(
*vm.m_perBytecodeProfiler, codeBlock, node->origin.semantic);
}
if (graph.dumpCodeOrigin(out, prefix, lastNode, node, &dumpContext)) {
compilation->addDescription(stack, out.toCString());
out.reset();
}
graph.dump(out, prefix, node, &dumpContext);
compilation->addDescription(stack, out.toCString());
out.reset();
if (node->origin.semantic.isSet())
lastNode = node;
}
}
dumpContext.dump(out, prefix);
compilation->addDescription(Profiler::OriginStack(), out.toCString());
out.reset();
out.print(" Disassembly:\n");
#if FTL_USES_B3
out.print(" <not implemented yet>\n");
#else
for (unsigned i = 0; i < state.jitCode->handles().size(); ++i) {
if (state.codeSectionNames[i] != SECTION_NAME("text"))
continue;
ExecutableMemoryHandle* handle = state.jitCode->handles()[i].get();
disassemble(
MacroAssemblerCodePtr(handle->start()), handle->sizeInBytes(),
" ", out, LLVMSubset);
}
#endif
compilation->addDescription(Profiler::OriginStack(), out.toCString());
out.reset();
state.jitCode->common.compilation = compilation;
}
switch (graph.m_plan.mode) {
case FTLMode: {
CCallHelpers::JumpList mainPathJumps;
jit.load32(
frame.withOffset(sizeof(Register) * JSStack::ArgumentCount),
GPRInfo::regT1);
mainPathJumps.append(jit.branch32(
CCallHelpers::AboveOrEqual, GPRInfo::regT1,
CCallHelpers::TrustedImm32(codeBlock->numParameters())));
jit.emitFunctionPrologue();
jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
jit.store32(
CCallHelpers::TrustedImm32(CallSiteIndex(0).bits()),
CCallHelpers::tagFor(JSStack::ArgumentCount));
jit.storePtr(GPRInfo::callFrameRegister, &vm.topCallFrame);
CCallHelpers::Call callArityCheck = jit.call();
#if !ASSERT_DISABLED
// FIXME: need to make this call register with exception handling somehow. This is
// part of a bigger problem: FTL should be able to handle exceptions.
// https://bugs.webkit.org/show_bug.cgi?id=113622
// Until then, use a JIT ASSERT.
jit.load64(vm.addressOfException(), GPRInfo::regT1);
jit.jitAssertIsNull(GPRInfo::regT1);
#endif
jit.move(GPRInfo::returnValueGPR, GPRInfo::argumentGPR0);
jit.emitFunctionEpilogue();
mainPathJumps.append(jit.branchTest32(CCallHelpers::Zero, GPRInfo::argumentGPR0));
jit.emitFunctionPrologue();
CCallHelpers::Call callArityFixup = jit.call();
jit.emitFunctionEpilogue();
mainPathJumps.append(jit.jump());
linkBuffer = std::make_unique<LinkBuffer>(vm, jit, codeBlock, JITCompilationCanFail);
if (linkBuffer->didFailToAllocate()) {
state.allocationFailed = true;
return;
}
linkBuffer->link(callArityCheck, codeBlock->m_isConstructor ? operationConstructArityCheck : operationCallArityCheck);
linkBuffer->link(callArityFixup, FunctionPtr((vm.getCTIStub(arityFixupGenerator)).code().executableAddress()));
linkBuffer->link(mainPathJumps, CodeLocationLabel(bitwise_cast<void*>(state.generatedFunction)));
state.jitCode->initializeAddressForCall(MacroAssemblerCodePtr(bitwise_cast<void*>(state.generatedFunction)));
break;
}
case FTLForOSREntryMode: {
// We jump to here straight from DFG code, after having boxed up all of the
// values into the scratch buffer. Everything should be good to go - at this
// point we've even done the stack check. Basically we just have to make the
// call to the LLVM-generated code.
CCallHelpers::Label start = jit.label();
jit.emitFunctionEpilogue();
CCallHelpers::Jump mainPathJump = jit.jump();
linkBuffer = std::make_unique<LinkBuffer>(vm, jit, codeBlock, JITCompilationCanFail);
if (linkBuffer->didFailToAllocate()) {
state.allocationFailed = true;
return;
}
linkBuffer->link(mainPathJump, CodeLocationLabel(bitwise_cast<void*>(state.generatedFunction)));
state.jitCode->initializeAddressForCall(linkBuffer->locationOf(start));
break;
}
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
state.finalizer->entrypointLinkBuffer = WTFMove(linkBuffer);
state.finalizer->function = state.generatedFunction;
state.finalizer->jitCode = state.jitCode;
}
void link(State& state)
{
Graph& graph = state.graph;
CodeBlock* codeBlock = graph.m_codeBlock;
VM& vm = graph.m_vm;
// LLVM will create its own jump tables as needed.
codeBlock->clearSwitchJumpTables();
// FIXME: Need to know the real frame register count.
// https://bugs.webkit.org/show_bug.cgi?id=125727
state.jitCode->common.frameRegisterCount = 1000;
state.jitCode->common.requiredRegisterCountForExit = graph.requiredRegisterCountForExit();
if (!graph.m_plan.inlineCallFrames->isEmpty())
state.jitCode->common.inlineCallFrames = graph.m_plan.inlineCallFrames;
graph.registerFrozenValues();
// Create the entrypoint. Note that we use this entrypoint totally differently
// depending on whether we're doing OSR entry or not.
CCallHelpers jit(&vm, codeBlock);
std::unique_ptr<LinkBuffer> linkBuffer;
CCallHelpers::Address frame = CCallHelpers::Address(
CCallHelpers::stackPointerRegister, -static_cast<int32_t>(AssemblyHelpers::prologueStackPointerDelta()));
if (Profiler::Compilation* compilation = graph.compilation()) {
compilation->addDescription(
Profiler::OriginStack(),
toCString("Generated FTL JIT code for ", CodeBlockWithJITType(codeBlock, JITCode::FTLJIT), ", instruction count = ", graph.m_codeBlock->instructionCount(), ":\n"));
graph.m_dominators.computeIfNecessary(graph);
graph.m_naturalLoops.computeIfNecessary(graph);
const char* prefix = " ";
DumpContext dumpContext;
StringPrintStream out;
Node* lastNode = 0;
for (size_t blockIndex = 0; blockIndex < graph.numBlocks(); ++blockIndex) {
BasicBlock* block = graph.block(blockIndex);
if (!block)
continue;
graph.dumpBlockHeader(out, prefix, block, Graph::DumpLivePhisOnly, &dumpContext);
compilation->addDescription(Profiler::OriginStack(), out.toCString());
out.reset();
for (size_t nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
Node* node = block->at(nodeIndex);
if (!node->willHaveCodeGenOrOSR() && !Options::showAllDFGNodes())
continue;
Profiler::OriginStack stack;
if (node->origin.semantic.isSet()) {
stack = Profiler::OriginStack(
*vm.m_perBytecodeProfiler, codeBlock, node->origin.semantic);
}
if (graph.dumpCodeOrigin(out, prefix, lastNode, node, &dumpContext)) {
compilation->addDescription(stack, out.toCString());
out.reset();
}
graph.dump(out, prefix, node, &dumpContext);
compilation->addDescription(stack, out.toCString());
out.reset();
if (node->origin.semantic.isSet())
lastNode = node;
}
}
dumpContext.dump(out, prefix);
compilation->addDescription(Profiler::OriginStack(), out.toCString());
out.reset();
out.print(" Disassembly:\n");
for (unsigned i = 0; i < state.jitCode->handles().size(); ++i) {
if (state.codeSectionNames[i] != SECTION_NAME("text"))
continue;
ExecutableMemoryHandle* handle = state.jitCode->handles()[i].get();
disassemble(
MacroAssemblerCodePtr(handle->start()), handle->sizeInBytes(),
" ", out, LLVMSubset);
}
compilation->addDescription(Profiler::OriginStack(), out.toCString());
out.reset();
state.jitCode->common.compilation = compilation;
}
switch (graph.m_plan.mode) {
case FTLMode: {
CCallHelpers::JumpList mainPathJumps;
jit.load32(
frame.withOffset(sizeof(Register) * JSStack::ArgumentCount),
GPRInfo::regT1);
mainPathJumps.append(jit.branch32(
CCallHelpers::AboveOrEqual, GPRInfo::regT1,
CCallHelpers::TrustedImm32(codeBlock->numParameters())));
jit.emitFunctionPrologue();
jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR0);
jit.store32(
CCallHelpers::TrustedImm32(CallFrame::Location::encodeAsBytecodeOffset(0)),
CCallHelpers::tagFor(JSStack::ArgumentCount));
jit.storePtr(GPRInfo::callFrameRegister, &vm.topCallFrame);
CCallHelpers::Call callArityCheck = jit.call();
#if !ASSERT_DISABLED
// FIXME: need to make this call register with exception handling somehow. This is
// part of a bigger problem: FTL should be able to handle exceptions.
// https://bugs.webkit.org/show_bug.cgi?id=113622
// Until then, use a JIT ASSERT.
jit.load64(vm.addressOfException(), GPRInfo::regT1);
jit.jitAssertIsNull(GPRInfo::regT1);
#endif
jit.move(GPRInfo::returnValueGPR, GPRInfo::regT0);
jit.emitFunctionEpilogue();
mainPathJumps.append(jit.branchTest32(CCallHelpers::Zero, GPRInfo::regT0));
jit.emitFunctionPrologue();
jit.move(CCallHelpers::TrustedImmPtr(vm.arityCheckFailReturnThunks->returnPCsFor(vm, codeBlock->numParameters())), GPRInfo::regT7);
jit.loadPtr(CCallHelpers::BaseIndex(GPRInfo::regT7, GPRInfo::regT0, CCallHelpers::timesPtr()), GPRInfo::regT7);
CCallHelpers::Call callArityFixup = jit.call();
jit.emitFunctionEpilogue();
mainPathJumps.append(jit.jump());
linkBuffer = std::make_unique<LinkBuffer>(vm, jit, codeBlock, JITCompilationMustSucceed);
linkBuffer->link(callArityCheck, codeBlock->m_isConstructor ? operationConstructArityCheck : operationCallArityCheck);
linkBuffer->link(callArityFixup, FunctionPtr((vm.getCTIStub(arityFixupGenerator)).code().executableAddress()));
linkBuffer->link(mainPathJumps, CodeLocationLabel(bitwise_cast<void*>(state.generatedFunction)));
state.jitCode->initializeAddressForCall(MacroAssemblerCodePtr(bitwise_cast<void*>(state.generatedFunction)));
break;
}
case FTLForOSREntryMode: {
// We jump to here straight from DFG code, after having boxed up all of the
// values into the scratch buffer. Everything should be good to go - at this
// point we've even done the stack check. Basically we just have to make the
// call to the LLVM-generated code.
CCallHelpers::Label start = jit.label();
jit.emitFunctionEpilogue();
CCallHelpers::Jump mainPathJump = jit.jump();
linkBuffer = std::make_unique<LinkBuffer>(vm, jit, codeBlock, JITCompilationMustSucceed);
linkBuffer->link(mainPathJump, CodeLocationLabel(bitwise_cast<void*>(state.generatedFunction)));
state.jitCode->initializeAddressForCall(linkBuffer->locationOf(start));
break;
}
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
state.finalizer->entrypointLinkBuffer = WTF::move(linkBuffer);
state.finalizer->function = state.generatedFunction;
state.finalizer->jitCode = state.jitCode;
}
