void JITCompiler::disassemble(LinkBuffer& linkBuffer)
{
if (shouldDumpDisassembly()) {
m_disassembler->dump(linkBuffer);
linkBuffer.didAlreadyDisassemble();
}
if (m_graph.m_plan.compilation)
m_disassembler->reportToProfiler(m_graph.m_plan.compilation.get(), linkBuffer);
}
JITCompiler::JITCompiler(Graph& dfg)
: CCallHelpers(&dfg.m_vm, dfg.m_codeBlock)
, m_graph(dfg)
, m_jitCode(adoptRef(new JITCode()))
, m_blockHeads(dfg.numBlocks())
, m_pcToCodeOriginMapBuilder(dfg.m_vm)
{
if (shouldDumpDisassembly() || m_graph.m_vm.m_perBytecodeProfiler)
m_disassembler = std::make_unique<Disassembler>(dfg);
#if ENABLE(FTL_JIT)
m_jitCode->tierUpInLoopHierarchy = WTFMove(m_graph.m_plan.tierUpInLoopHierarchy);
for (unsigned tierUpBytecode : m_graph.m_plan.tierUpAndOSREnterBytecodes)
m_jitCode->tierUpEntryTriggers.add(tierUpBytecode, 0);
#endif
}
Plan::CompilationPath Plan::compileInThreadImpl(LongLivedState& longLivedState)
{
cleanMustHandleValuesIfNecessary();
if (verboseCompilationEnabled(mode) && osrEntryBytecodeIndex != UINT_MAX) {
dataLog("\n");
dataLog("Compiler must handle OSR entry from bc#", osrEntryBytecodeIndex, " with values: ", mustHandleValues, "\n");
dataLog("\n");
}
Graph dfg(*vm, *this, longLivedState);
if (!parse(dfg)) {
finalizer = std::make_unique<FailedFinalizer>(*this);
return FailPath;
}
codeBlock->setCalleeSaveRegisters(RegisterSet::dfgCalleeSaveRegisters());
// By this point the DFG bytecode parser will have potentially mutated various tables
// in the CodeBlock. This is a good time to perform an early shrink, which is more
// powerful than a late one. It's safe to do so because we haven't generated any code
// that references any of the tables directly, yet.
codeBlock->shrinkToFit(CodeBlock::EarlyShrink);
if (validationEnabled())
validate(dfg);
if (Options::dumpGraphAfterParsing()) {
dataLog("Graph after parsing:\n");
dfg.dump();
}
performLiveCatchVariablePreservationPhase(dfg);
if (Options::useMaximalFlushInsertionPhase())
performMaximalFlushInsertion(dfg);
performCPSRethreading(dfg);
performUnification(dfg);
performPredictionInjection(dfg);
performStaticExecutionCountEstimation(dfg);
if (mode == FTLForOSREntryMode) {
bool result = performOSREntrypointCreation(dfg);
if (!result) {
finalizer = std::make_unique<FailedFinalizer>(*this);
return FailPath;
}
performCPSRethreading(dfg);
}
if (validationEnabled())
validate(dfg);
performBackwardsPropagation(dfg);
performPredictionPropagation(dfg);
performFixup(dfg);
performStructureRegistration(dfg);
performInvalidationPointInjection(dfg);
performTypeCheckHoisting(dfg);
dfg.m_fixpointState = FixpointNotConverged;
// For now we're back to avoiding a fixpoint. Note that we've ping-ponged on this decision
// many times. For maximum throughput, it's best to fixpoint. But the throughput benefit is
// small and not likely to show up in FTL anyway. On the other hand, not fixpointing means
// that the compiler compiles more quickly. We want the third tier to compile quickly, which
// not fixpointing accomplishes; and the fourth tier shouldn't need a fixpoint.
if (validationEnabled())
validate(dfg);
performStrengthReduction(dfg);
performCPSRethreading(dfg);
performCFA(dfg);
performConstantFolding(dfg);
bool changed = false;
changed |= performCFGSimplification(dfg);
changed |= performLocalCSE(dfg);
if (validationEnabled())
validate(dfg);
performCPSRethreading(dfg);
if (!isFTL(mode)) {
// Only run this if we're not FTLing, because currently for a LoadVarargs that is forwardable and
// in a non-varargs inlined call frame, this will generate ForwardVarargs while the FTL
// ArgumentsEliminationPhase will create a sequence of GetStack+PutStacks. The GetStack+PutStack
// sequence then gets sunk, eliminating anything that looks like an escape for subsequent phases,
// while the ForwardVarargs doesn't get simplified until later (or not at all) and looks like an
// escape for all of the arguments. This then disables object allocation sinking.
//
// So, for now, we just disable this phase for the FTL.
//
// If we wanted to enable it, we'd have to do any of the following:
// - Enable ForwardVarargs->GetStack+PutStack strength reduction, and have that run before
// PutStack sinking and object allocation sinking.
// - Make VarargsForwarding emit a GetLocal+SetLocal sequence, that we can later turn into
// GetStack+PutStack.
//
// But, it's not super valuable to enable those optimizations, since the FTL
// ArgumentsEliminationPhase does everything that this phase does, and it doesn't introduce this
// pathology.
changed |= performVarargsForwarding(dfg); // Do this after CFG simplification and CPS rethreading.
}
if (changed) {
performCFA(dfg);
performConstantFolding(dfg);
}
// If we're doing validation, then run some analyses, to give them an opportunity
// to self-validate. Now is as good a time as any to do this.
if (validationEnabled()) {
dfg.ensureDominators();
dfg.ensureNaturalLoops();
dfg.ensurePrePostNumbering();
}
switch (mode) {
case DFGMode: {
dfg.m_fixpointState = FixpointConverged;
performTierUpCheckInjection(dfg);
performFastStoreBarrierInsertion(dfg);
performStoreBarrierClustering(dfg);
performCleanUp(dfg);
performCPSRethreading(dfg);
performDCE(dfg);
performPhantomInsertion(dfg);
performStackLayout(dfg);
performVirtualRegisterAllocation(dfg);
performWatchpointCollection(dfg);
dumpAndVerifyGraph(dfg, "Graph after optimization:");
JITCompiler dataFlowJIT(dfg);
if (codeBlock->codeType() == FunctionCode)
dataFlowJIT.compileFunction();
else
dataFlowJIT.compile();
return DFGPath;
}
case FTLMode:
case FTLForOSREntryMode: {
#if ENABLE(FTL_JIT)
if (FTL::canCompile(dfg) == FTL::CannotCompile) {
finalizer = std::make_unique<FailedFinalizer>(*this);
return FailPath;
}
performCleanUp(dfg); // Reduce the graph size a bit.
performCriticalEdgeBreaking(dfg);
if (Options::createPreHeaders())
performLoopPreHeaderCreation(dfg);
performCPSRethreading(dfg);
performSSAConversion(dfg);
performSSALowering(dfg);
// Ideally, these would be run to fixpoint with the object allocation sinking phase.
performArgumentsElimination(dfg);
if (Options::usePutStackSinking())
performPutStackSinking(dfg);
performConstantHoisting(dfg);
performGlobalCSE(dfg);
performLivenessAnalysis(dfg);
performCFA(dfg);
performConstantFolding(dfg);
performCleanUp(dfg); // Reduce the graph size a lot.
changed = false;
changed |= performStrengthReduction(dfg);
if (Options::useObjectAllocationSinking()) {
changed |= performCriticalEdgeBreaking(dfg);
changed |= performObjectAllocationSinking(dfg);
}
if (changed) {
// State-at-tail and state-at-head will be invalid if we did strength reduction since
// it might increase live ranges.
performLivenessAnalysis(dfg);
performCFA(dfg);
performConstantFolding(dfg);
}
// Currently, this relies on pre-headers still being valid. That precludes running CFG
// simplification before it, unless we re-created the pre-headers. There wouldn't be anything
// wrong with running LICM earlier, if we wanted to put other CFG transforms above this point.
// Alternatively, we could run loop pre-header creation after SSA conversion - but if we did that
// then we'd need to do some simple SSA fix-up.
performLivenessAnalysis(dfg);
performCFA(dfg);
performLICM(dfg);
// FIXME: Currently: IntegerRangeOptimization *must* be run after LICM.
//
// IntegerRangeOptimization makes changes on nodes based on preceding blocks
// and nodes. LICM moves nodes which can invalidates assumptions used
// by IntegerRangeOptimization.
//
// Ideally, the dependencies should be explicit. See https://bugs.webkit.org/show_bug.cgi?id=157534.
performLivenessAnalysis(dfg);
performIntegerRangeOptimization(dfg);
performCleanUp(dfg);
performIntegerCheckCombining(dfg);
performGlobalCSE(dfg);
// At this point we're not allowed to do any further code motion because our reasoning
// about code motion assumes that it's OK to insert GC points in random places.
dfg.m_fixpointState = FixpointConverged;
performLivenessAnalysis(dfg);
performCFA(dfg);
performGlobalStoreBarrierInsertion(dfg);
performStoreBarrierClustering(dfg);
if (Options::useMovHintRemoval())
performMovHintRemoval(dfg);
performCleanUp(dfg);
performDCE(dfg); // We rely on this to kill dead code that won't be recognized as dead by B3.
performStackLayout(dfg);
performLivenessAnalysis(dfg);
performOSRAvailabilityAnalysis(dfg);
performWatchpointCollection(dfg);
if (FTL::canCompile(dfg) == FTL::CannotCompile) {
finalizer = std::make_unique<FailedFinalizer>(*this);
return FailPath;
}
dumpAndVerifyGraph(dfg, "Graph just before FTL lowering:", shouldDumpDisassembly(mode));
// Flash a safepoint in case the GC wants some action.
Safepoint::Result safepointResult;
{
GraphSafepoint safepoint(dfg, safepointResult);
}
if (safepointResult.didGetCancelled())
return CancelPath;
FTL::State state(dfg);
FTL::lowerDFGToB3(state);
if (UNLIKELY(computeCompileTimes()))
m_timeBeforeFTL = monotonicallyIncreasingTimeMS();
if (Options::b3AlwaysFailsBeforeCompile()) {
FTL::fail(state);
return FTLPath;
}
FTL::compile(state, safepointResult);
if (safepointResult.didGetCancelled())
return CancelPath;
if (Options::b3AlwaysFailsBeforeLink()) {
FTL::fail(state);
return FTLPath;
}
if (state.allocationFailed) {
FTL::fail(state);
return FTLPath;
}
FTL::link(state);
if (state.allocationFailed) {
FTL::fail(state);
return FTLPath;
}
return FTLPath;
#else
RELEASE_ASSERT_NOT_REACHED();
return FailPath;
#endif // ENABLE(FTL_JIT)
}
default:
RELEASE_ASSERT_NOT_REACHED();
return FailPath;
}
}
void compileOSRExit(ExecState* exec)
{
if (exec->vm().callFrameForCatch)
RELEASE_ASSERT(exec->vm().callFrameForCatch == exec);
CodeBlock* codeBlock = exec->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->jitType() == JITCode::DFGJIT);
VM* vm = &exec->vm();
// It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
// really be profitable.
DeferGCForAWhile deferGC(vm->heap);
uint32_t exitIndex = vm->osrExitIndex;
OSRExit& exit = codeBlock->jitCode()->dfg()->osrExit[exitIndex];
if (vm->callFrameForCatch)
ASSERT(exit.m_kind == GenericUnwind);
if (exit.isExceptionHandler())
ASSERT(!!vm->exception());
prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
// Compute the value recoveries.
Operands<ValueRecovery> operands;
codeBlock->jitCode()->dfg()->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->jitCode()->dfg()->minifiedDFG, exit.m_streamIndex, operands);
SpeculationRecovery* recovery = 0;
if (exit.m_recoveryIndex != UINT_MAX)
recovery = &codeBlock->jitCode()->dfg()->speculationRecovery[exit.m_recoveryIndex];
{
CCallHelpers jit(vm, codeBlock);
OSRExitCompiler exitCompiler(jit);
if (exit.m_kind == GenericUnwind) {
// We are acting as a defacto op_catch because we arrive here from genericUnwind().
// So, we must restore our call frame and stack pointer.
jit.restoreCalleeSavesFromVMEntryFrameCalleeSavesBuffer();
jit.loadPtr(vm->addressOfCallFrameForCatch(), GPRInfo::callFrameRegister);
jit.addPtr(CCallHelpers::TrustedImm32(codeBlock->stackPointerOffset() * sizeof(Register)),
GPRInfo::callFrameRegister, CCallHelpers::stackPointerRegister);
}
jit.jitAssertHasValidCallFrame();
if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
Profiler::Database& database = *vm->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
exitCompiler.compileExit(exit, operands, recovery);
LinkBuffer patchBuffer(*vm, jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldDumpDisassembly() || Options::verboseOSR(),
patchBuffer,
("DFG OSR exit #%u (%s, %s) from %s, with operands = %s",
exitIndex, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(operands)).data()));
}
MacroAssembler::repatchJump(exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
vm->osrExitJumpDestination = exit.m_code.code().executableAddress();
}
void compileOSRExit(ExecState* exec)
{
SamplingRegion samplingRegion("DFG OSR Exit Compilation");
CodeBlock* codeBlock = exec->codeBlock();
ASSERT(codeBlock);
ASSERT(codeBlock->jitType() == JITCode::DFGJIT);
VM* vm = &exec->vm();
// It's sort of preferable that we don't GC while in here. Anyways, doing so wouldn't
// really be profitable.
DeferGCForAWhile deferGC(vm->heap);
uint32_t exitIndex = vm->osrExitIndex;
OSRExit& exit = codeBlock->jitCode()->dfg()->osrExit[exitIndex];
prepareCodeOriginForOSRExit(exec, exit.m_codeOrigin);
// Compute the value recoveries.
Operands<ValueRecovery> operands;
codeBlock->jitCode()->dfg()->variableEventStream.reconstruct(codeBlock, exit.m_codeOrigin, codeBlock->jitCode()->dfg()->minifiedDFG, exit.m_streamIndex, operands);
SpeculationRecovery* recovery = 0;
if (exit.m_recoveryIndex != UINT_MAX)
recovery = &codeBlock->jitCode()->dfg()->speculationRecovery[exit.m_recoveryIndex];
{
CCallHelpers jit(vm, codeBlock);
OSRExitCompiler exitCompiler(jit);
jit.jitAssertHasValidCallFrame();
if (vm->m_perBytecodeProfiler && codeBlock->jitCode()->dfgCommon()->compilation) {
Profiler::Database& database = *vm->m_perBytecodeProfiler;
Profiler::Compilation* compilation = codeBlock->jitCode()->dfgCommon()->compilation.get();
Profiler::OSRExit* profilerExit = compilation->addOSRExit(
exitIndex, Profiler::OriginStack(database, codeBlock, exit.m_codeOrigin),
exit.m_kind, exit.m_kind == UncountableInvalidation);
jit.add64(CCallHelpers::TrustedImm32(1), CCallHelpers::AbsoluteAddress(profilerExit->counterAddress()));
}
exitCompiler.compileExit(exit, operands, recovery);
LinkBuffer patchBuffer(*vm, jit, codeBlock);
exit.m_code = FINALIZE_CODE_IF(
shouldDumpDisassembly() || Options::verboseOSR(),
patchBuffer,
("DFG OSR exit #%u (%s, %s) from %s, with operands = %s",
exitIndex, toCString(exit.m_codeOrigin).data(),
exitKindToString(exit.m_kind), toCString(*codeBlock).data(),
toCString(ignoringContext<DumpContext>(operands)).data()));
}
{
RepatchBuffer repatchBuffer(codeBlock);
repatchBuffer.relink(exit.codeLocationForRepatch(codeBlock), CodeLocationLabel(exit.m_code.code()));
}
vm->osrExitJumpDestination = exit.m_code.code().executableAddress();
}
