bool InliningDecider::ContinueInliningUserDefinedFunctions(uint32 bytecodeInlinedCount) const { #if ENABLE_DEBUG_CONFIG_OPTIONS char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; #endif if (PHASE_FORCE(Js::InlinePhase, this->topFunc) || bytecodeInlinedCount <= (uint)this->threshold.inlineCountMax) { return true; } INLINE_TESTTRACE(_u("INLINING: Skip Inline: InlineCountMax threshold %d, reached: %s (#%s)\n"), (uint)this->threshold.inlineCountMax, this->topFunc->GetDisplayName(), this->topFunc->GetDebugNumberSet(debugStringBuffer)); return false; }
// This only enables collection of the inlinee data, we are much more aggressive here. // Actual decision of whether something is inlined or not is taken in CommitInlineIntoInliner bool InliningDecider::DeciderInlineIntoInliner(Js::FunctionBody * inlinee, Js::FunctionBody * inliner, bool isConstructorCall, bool isPolymorphicCall, uint16 constantArgInfo, uint recursiveInlineDepth, bool allowRecursiveInlining) { if (!CanRecursivelyInline(inlinee, inliner, allowRecursiveInlining, recursiveInlineDepth)) { return false; } if (inlinee->GetIsAsmjsMode() || inliner->GetIsAsmjsMode()) { return false; } if (PHASE_FORCE(Js::InlinePhase, this->topFunc) || PHASE_FORCE(Js::InlinePhase, inliner) || PHASE_FORCE(Js::InlinePhase, inlinee)) { return true; } if (PHASE_OFF(Js::InlinePhase, this->topFunc) || PHASE_OFF(Js::InlinePhase, inliner) || PHASE_OFF(Js::InlinePhase, inlinee)) { return false; } if (PHASE_FORCE(Js::InlineTreePhase, this->topFunc) || PHASE_FORCE(Js::InlineTreePhase, inliner)) { return true; } if (PHASE_FORCE(Js::InlineAtEveryCallerPhase, inlinee)) { return true; } uint inlineeByteCodeCount = inlinee->GetByteCodeWithoutLDACount(); // Heuristics are hit in the following order (Note *order* is important) // 1. Leaf function: If the inlinee is a leaf (but not a constructor or a polymorphic call) inline threshold is LeafInlineThreshold (60). Also it can have max 1 loop // 2. Constant Function Argument: If the inlinee candidate has a constant argument and that argument is used for branching, then the inline threshold is ConstantArgumentInlineThreshold (157) // 3. InlineThreshold: If an inlinee candidate exceeds InlineThreshold just don't inline no matter what. // Following are additional constraint for an inlinee which meets InlineThreshold (Rule no 3) // 4. Rule for inlinee with loops: // 4a. Only single loop in inlinee is permitted. // 4b. Should not have polymorphic field access. // 4c. Should not be a constructor. // 4d. Should meet LoopInlineThreshold (25) // 5. Rule for polymorphic inlinee: // 4a. Should meet PolymorphicInlineThreshold (32) // 6. Rule for constructors: // 5a. Always inline if inlinee has polymorphic field access (as we have cloned runtime data). // 5b. If inlinee is monomorphic, inline only small constructors. They are governed by ConstructorInlineThreshold (21) // 7. Rule for inlinee which is not interpreted enough (as we might not have all the profile data): // 7a. As of now it is still governed by the InlineThreshold. Plan to play with this in future. // 8. Rest should be inlined. uint16 mask = constantArgInfo & inlinee->m_argUsedForBranch; if (mask && inlineeByteCodeCount < (uint)CONFIG_FLAG(ConstantArgumentInlineThreshold)) { return true; } int inlineThreshold = threshold.inlineThreshold; if (!isPolymorphicCall && !isConstructorCall && IsInlineeLeaf(inlinee) && (inlinee->GetLoopCount() <= 2)) { // Inlinee is a leaf function if (inlinee->GetLoopCount() == 0 || GetNumberOfInlineesWithLoop() <= (uint)threshold.maxNumberOfInlineesWithLoop) // Don't inlinee too many inlinees with loops. { // Negative LeafInlineThreshold disable the threshold if (threshold.leafInlineThreshold >= 0) { inlineThreshold += threshold.leafInlineThreshold - threshold.inlineThreshold; } } } #if ENABLE_DEBUG_CONFIG_OPTIONS char16 debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; char16 debugStringBuffer2[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; char16 debugStringBuffer3[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; #endif if (inlinee->GetHasLoops()) { if (threshold.loopInlineThreshold < 0 || // Negative LoopInlineThreshold disable inlining with loop GetNumberOfInlineesWithLoop() >(uint)threshold.maxNumberOfInlineesWithLoop || // See if we are inlining too many inlinees with loops. (inlinee->GetLoopCount() > 2) || // Allow at most 2 loops. inlinee->GetHasNestedLoop() || // Nested loops are not a good inlinee candidate isConstructorCall || // If the function is constructor with loops, don't inline. PHASE_OFF(Js::InlineFunctionsWithLoopsPhase, this->topFunc)) { INLINE_TESTTRACE(_u("INLINING: Skip Inline: Has loops \tBytecode size: %d \tgetNumberOfInlineesWithLoop: %d\tloopCount: %d\thasNestedLoop: %B\tisConstructorCall:%B\tInlinee: %s (%s)\tCaller: %s (%s) \tRoot: %s (%s)\n"), inlinee->GetByteCodeCount(), GetNumberOfInlineesWithLoop(), inlinee->GetLoopCount(), inlinee->GetHasNestedLoop(), isConstructorCall, inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer2), topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer3)); // Don't inline function with loops return false; } else { inlineThreshold -= (threshold.inlineThreshold > threshold.loopInlineThreshold) ? threshold.inlineThreshold - threshold.loopInlineThreshold : 0; } } if (isPolymorphicCall) { if (threshold.polymorphicInlineThreshold < 0 || // Negative PolymorphicInlineThreshold disable inlining isConstructorCall) { INLINE_TESTTRACE(_u("INLINING: Skip Inline: Polymorphic call under PolymorphicInlineThreshold: %d \tBytecode size: %d\tInlinee: %s (%s)\tCaller: %s (%s) \tRoot: %s (%s)\n"), threshold.polymorphicInlineThreshold, inlinee->GetByteCodeCount(), inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer2), topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer3)); return false; } else { inlineThreshold -= (threshold.inlineThreshold > threshold.polymorphicInlineThreshold) ? threshold.inlineThreshold - threshold.polymorphicInlineThreshold : 0; } } if (isConstructorCall) { #pragma prefast(suppress: 6285, "logical-or of constants is by design") if (PHASE_OFF(Js::InlineConstructorsPhase, this->topFunc) || PHASE_OFF(Js::InlineConstructorsPhase, inliner) || PHASE_OFF(Js::InlineConstructorsPhase, inlinee) || !CONFIG_FLAG(CloneInlinedPolymorphicCaches)) { return false; } if (PHASE_FORCE(Js::InlineConstructorsPhase, this->topFunc) || PHASE_FORCE(Js::InlineConstructorsPhase, inliner) || PHASE_FORCE(Js::InlineConstructorsPhase, inlinee)) { return true; } if (inlinee->HasDynamicProfileInfo() && inlinee->GetAnyDynamicProfileInfo()->HasPolymorphicFldAccess()) { // As of now this is not dependent on bytecodeInlinedThreshold. return true; } // Negative ConstructorInlineThreshold always disable constructor inlining if (threshold.constructorInlineThreshold < 0) { INLINE_TESTTRACE(_u("INLINING: Skip Inline: Constructor with no polymorphic field access \tBytecode size: %d\tInlinee: %s (%s)\tCaller: %s (%s) \tRoot: %s (%s)\n"), inlinee->GetByteCodeCount(), inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer2), topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer3)); // Don't inline constructor that does not have a polymorphic field access, or if cloning polymorphic inline // caches is disabled return false; } else { inlineThreshold -= (threshold.inlineThreshold > threshold.constructorInlineThreshold) ? threshold.inlineThreshold - threshold.constructorInlineThreshold : 0; } } if (threshold.forLoopBody) { inlineThreshold /= CONFIG_FLAG(InlineInLoopBodyScaleDownFactor); } if (inlineThreshold > 0 && inlineeByteCodeCount <= (uint)inlineThreshold) { if (inlinee->GetLoopCount()) { IncrementNumberOfInlineesWithLoop(); } return true; } else { return false; } }
// Called from background thread to commit inlining. bool InliningHeuristics::BackendInlineIntoInliner(Js::FunctionBody* inlinee, Js::FunctionBody *inliner, Func *topFunction, Js::ProfileId callSiteId, bool isConstructorCall, bool isFixedMethodCall, // Reserved bool isCallOutsideLoopInTopFunc, // There is a loop for sure and this call is outside loop bool isCallInsideLoop, uint recursiveInlineDepth, uint16 constantArguments ) { // We have one piece of additional data in backend, whether we are outside loop or inside // This function decides to inline or not based on that additional data. Most of the filtering is already done by DeciderInlineIntoInliner which is called // during work item creation. // This is additional filtering during actual inlining phase. // Note *order* is important // Following are // 1. Constructor is always inlined (irrespective of inside or outside) // 2. If the inlinee candidate has constant argument and that argument is used for a branch and the inlinee size is within ConstantArgumentInlineThreshold(157) we inline // 3. Inside loops: // 3a. Leaf function will always get inlined (irrespective of leaf has loop or not) // 3b. If the inlinee has loops, don't inline it (Basically avoiding inlining a loop within another loop unless its leaf). // 4. Outside loop (inliner has loops): // 4a. Only inline small inlinees. Governed by OutsideLoopInlineThreshold (16) // 5. Rest are inlined. #if ENABLE_DEBUG_CONFIG_OPTIONS wchar_t debugStringBuffer[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; wchar_t debugStringBuffer2[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; wchar_t debugStringBuffer3[MAX_FUNCTION_BODY_DEBUG_STRING_SIZE]; #endif bool doBackEndAggressiveInline = (constantArguments & inlinee->m_argUsedForBranch) != 0; if (!PHASE_OFF(Js::InlineRecursivePhase, inliner) && inlinee == inliner && (!inlinee->CanInlineRecursively(recursiveInlineDepth, doBackEndAggressiveInline))) { INLINE_TESTTRACE(L"INLINING: Skip Inline (backend): Recursive inlining\tInlinee: %s (#%s)\tCaller: %s (#%s) \tRoot: %s (#%s) Depth: %d\n", inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer2), topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer3), recursiveInlineDepth); return false; } if(PHASE_FORCE(Js::InlinePhase, this->topFunc) || PHASE_FORCE(Js::InlinePhase, inliner) || PHASE_FORCE(Js::InlinePhase, inlinee)) { return true; } if (PHASE_FORCE(Js::InlineTreePhase, this->topFunc) || PHASE_FORCE(Js::InlineTreePhase, inliner)) { return true; } if (PHASE_FORCE(Js::InlineAtEveryCallerPhase, inlinee)) { return true; } Js::DynamicProfileInfo *dynamicProfile = inliner->GetAnyDynamicProfileInfo(); bool doConstantArgumentInlining = (dynamicProfile->GetConstantArgInfo(callSiteId) & inlinee->m_argUsedForBranch) != 0; if (doConstantArgumentInlining && inlinee->GetByteCodeWithoutLDACount() < (uint)threshold.constantArgumentInlineThreshold) { return true; } if (topFunction->m_workItem->RecyclableData()->JitTimeData()->GetIsAggressiveInliningEnabled()) { return true; } if (isConstructorCall) { return true; } if (isCallInsideLoop && IsInlineeLeaf(inlinee)) { return true; } if (isCallInsideLoop && inlinee->GetHasLoops() ) // Don't inline function with loops inside another loop unless it is a leaf { INLINE_TESTTRACE(L"INLINING: Skip Inline (backend): Recursive loop inlining\tInlinee: %s (#%s)\tCaller: %s (#%s) \tRoot: %s (#%s)\n", inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer2), topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer3)); return false; } byte scale = 1; if (doBackEndAggressiveInline) { scale = 2; } if (isCallOutsideLoopInTopFunc && (threshold.outsideLoopInlineThreshold < 0 || inlinee->GetByteCodeWithoutLDACount() > (uint)threshold.outsideLoopInlineThreshold * scale)) { Assert(!isCallInsideLoop); INLINE_TESTTRACE(L"INLINING: Skip Inline (backend): Inlining outside loop doesn't meet OutsideLoopInlineThreshold: %d \tBytecode size: %d\tInlinee: %s (#%s)\tCaller: %s (#%s) \tRoot: %s (#%s)\n", threshold.outsideLoopInlineThreshold, inlinee->GetByteCodeCount(), inlinee->GetDisplayName(), inlinee->GetDebugNumberSet(debugStringBuffer), inliner->GetDisplayName(), inliner->GetDebugNumberSet(debugStringBuffer2), topFunc->GetDisplayName(), topFunc->GetDebugNumberSet(debugStringBuffer3)); return false; } return true; }