void breakCriticalEdges(Procedure& proc)
{
    BlockInsertionSet insertionSet(proc);
    
    for (BasicBlock* block : proc) {
        // Non-void terminals that are the moral equivalent of jumps trigger critical edge breaking
        // because of fixSSA's demoteValues.
        if (block->numSuccessors() <= 1
            && block->last()->type() == Void)
            continue;

        for (BasicBlock*& successor : block->successorBlocks()) {
            if (successor->numPredecessors() <= 1)
                continue;

            BasicBlock* pad =
                insertionSet.insertBefore(successor, successor->frequency());
            pad->appendNew<Value>(proc, Jump, successor->at(0)->origin());
            pad->setSuccessors(FrequentedBlock(successor));
            pad->addPredecessor(block);
            successor->replacePredecessor(block, pad);
            successor = pad;
        }
    }

    if (insertionSet.execute())
        proc.invalidateCFG();
}
    bool run()
    {
        DFG_ASSERT(m_graph, nullptr, m_graph.m_form == LoadStore);

        InsertionSet insertionSet(m_graph);
        for (BasicBlock* block : m_graph.blocksInNaturalOrder()) {
            treatRegularBlock(block, insertionSet);
            insertionSet.execute(block);
        }

        treatRootBlock(m_graph.block(0), insertionSet);
        insertionSet.execute(m_graph.block(0));

        return true;
    }
    bool run()
    {
        DFG_ASSERT(m_graph, nullptr, m_graph.m_form == LoadStore);

        if (!m_graph.m_hasExceptionHandlers)
            return false;

        InsertionSet insertionSet(m_graph);
        if (m_graph.m_hasExceptionHandlers) {
            for (BasicBlock* block : m_graph.blocksInNaturalOrder()) {
                handleBlockForTryCatch(block, insertionSet);
                insertionSet.execute(block);
            }
        }

        return true;
    }
    bool run()
    {
        InsertionSet insertionSet(m_graph);
        
        for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
            BasicBlock* block = m_graph.block(blockIndex);
            if (!block)
                continue;

            for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
                Node* node = block->at(nodeIndex);
                if (!node->hasResult())
                    continue;
                insertionSet.insertNode(
                    nodeIndex + 1, SpecNone, Phantom, node->origin, Edge(node));
            }
            
            insertionSet.execute(block);
        }
        
        return true;
    }
void breakCriticalEdges(Code& code)
{
    BlockInsertionSet insertionSet(code);

    for (BasicBlock* block : code) {
        if (block->numSuccessors() <= 1)
            continue;

        for (BasicBlock*& successor : block->successorBlocks()) {
            if (successor->numPredecessors() <= 1)
                continue;

            BasicBlock* pad = insertionSet.insertBefore(successor, successor->frequency());
            pad->append(Jump, successor->at(0).origin);
            pad->setSuccessors(successor);
            pad->addPredecessor(block);
            successor->replacePredecessor(block, pad);
            successor = pad;
        }
    }

    insertionSet.execute();
}
void lowerStackArgs(Code& code)
{
    PhaseScope phaseScope(code, "lowerStackArgs");

    // Now we need to deduce how much argument area we need.
    for (BasicBlock* block : code) {
        for (Inst& inst : *block) {
            for (Arg& arg : inst.args) {
                if (arg.isCallArg()) {
                    // For now, we assume that we use 8 bytes of the call arg. But that's not
                    // such an awesome assumption.
                    // FIXME: https://bugs.webkit.org/show_bug.cgi?id=150454
                    ASSERT(arg.offset() >= 0);
                    code.requestCallArgAreaSizeInBytes(arg.offset() + 8);
                }
            }
        }
    }

    code.setFrameSize(code.frameSize() + code.callArgAreaSizeInBytes());

    // Finally, transform the code to use Addr's instead of StackSlot's. This is a lossless
    // transformation since we can search the StackSlots array to figure out which StackSlot any
    // offset-from-FP refers to.

    InsertionSet insertionSet(code);
    for (BasicBlock* block : code) {
        // FIXME We can keep track of the last large offset which was materialized in this block, and reuse the register
        // if it hasn't been clobbered instead of renetating imm+add+addr every time. https://bugs.webkit.org/show_bug.cgi?id=171387

        for (unsigned instIndex = 0; instIndex < block->size(); ++instIndex) {
            Inst& inst = block->at(instIndex);

            inst.forEachArg(
                [&] (Arg& arg, Arg::Role role, Bank, Width width) {
                    auto stackAddr = [&] (Value::OffsetType offsetFromFP) -> Arg {
                        int32_t offsetFromSP = offsetFromFP + code.frameSize();

                        if (inst.admitsExtendedOffsetAddr(arg)) {
                            // Stackmaps and patchpoints expect addr inputs relative to SP or FP only. We might as well
                            // not even bother generating an addr with valid form for these opcodes since extended offset
                            // addr is always valid.
                            return Arg::extendedOffsetAddr(offsetFromFP);
                        }

                        Arg result = Arg::addr(Air::Tmp(GPRInfo::callFrameRegister), offsetFromFP);
                        if (result.isValidForm(width))
                            return result;

                        result = Arg::addr(Air::Tmp(MacroAssembler::stackPointerRegister), offsetFromSP);
                        if (result.isValidForm(width))
                            return result;
#if CPU(ARM64)
                        ASSERT(pinnedExtendedOffsetAddrRegister());
                        Air::Tmp tmp = Air::Tmp(*pinnedExtendedOffsetAddrRegister());

                        Arg largeOffset = Arg::isValidImmForm(offsetFromSP) ? Arg::imm(offsetFromSP) : Arg::bigImm(offsetFromSP);
                        insertionSet.insert(instIndex, Move, inst.origin, largeOffset, tmp);
                        insertionSet.insert(instIndex, Add64, inst.origin, Air::Tmp(MacroAssembler::stackPointerRegister), tmp);
                        result = Arg::addr(tmp, 0);
                        return result;
#elif CPU(X86_64)
                        // Can't happen on x86: immediates are always big enough for frame size.
                        RELEASE_ASSERT_NOT_REACHED();
#else
#error Unhandled architecture.
#endif
                    };

                    switch (arg.kind()) {
                    case Arg::Stack: {
                        StackSlot* slot = arg.stackSlot();
                        if (Arg::isZDef(role)
                            && slot->kind() == StackSlotKind::Spill
                            && slot->byteSize() > bytes(width)) {
                            // Currently we only handle this simple case because it's the only one
                            // that arises: ZDef's are only 32-bit right now. So, when we hit these
                            // assertions it means that we need to implement those other kinds of
                            // zero fills.
                            RELEASE_ASSERT(slot->byteSize() == 8);
                            RELEASE_ASSERT(width == Width32);

                            RELEASE_ASSERT(isValidForm(StoreZero32, Arg::Stack));
                            insertionSet.insert(
                                instIndex + 1, StoreZero32, inst.origin,
                                stackAddr(arg.offset() + 4 + slot->offsetFromFP()));
                        }
                        arg = stackAddr(arg.offset() + slot->offsetFromFP());
                        break;
                    }
                    case Arg::CallArg:
                        arg = stackAddr(arg.offset() - code.frameSize());
                        break;
                    default:
                        break;
                    }
                }
            );
        }
        insertionSet.execute(block);
    }
}
void spillEverything(Code& code)
{
    PhaseScope phaseScope(code, "spillEverything");

    // We want to know the set of registers used at every point in every basic block.
    IndexMap<BasicBlock, Vector<RegisterSet>> usedRegisters(code.size());
    Liveness<Tmp> liveness(code);
    for (BasicBlock* block : code) {
        Liveness<Tmp>::LocalCalc localCalc(liveness, block);
        usedRegisters[block].resize(block->size() + 1);

        auto setUsedRegisters = [&] (unsigned index, Inst& inst) {
            RegisterSet& registerSet = usedRegisters[block][index];
            for (Tmp tmp : localCalc.live()) {
                if (tmp.isReg())
                    registerSet.set(tmp.reg());
            }

            // Gotta account for dead assignments to registers. These may happen because the input
            // code is suboptimal.
            inst.forEachArg(
                [&] (Arg& arg, Arg::Role role, Arg::Type) {
                    if (Arg::isDef(role) && arg.isReg())
                        registerSet.set(arg.reg());
                });
        };

        for (unsigned instIndex = block->size(); instIndex--;) {
            Inst& inst = block->at(instIndex);
            setUsedRegisters(instIndex + 1, inst);
            localCalc.execute(inst);
        }

        Inst nop;
        setUsedRegisters(0, nop);
    }

    // Allocate a stack slot for each tmp.
    Vector<StackSlot*> allStackSlots[Arg::numTypes];
    for (unsigned typeIndex = 0; typeIndex < Arg::numTypes; ++typeIndex) {
        Vector<StackSlot*>& stackSlots = allStackSlots[typeIndex];
        Arg::Type type = static_cast<Arg::Type>(typeIndex);
        stackSlots.resize(code.numTmps(type));
        for (unsigned tmpIndex = code.numTmps(type); tmpIndex--;)
            stackSlots[tmpIndex] = code.addStackSlot(8, StackSlotKind::Anonymous);
    }

    InsertionSet insertionSet(code);
    for (BasicBlock* block : code) {
        for (unsigned instIndex = 0; instIndex < block->size(); ++instIndex) {
            RegisterSet& setBefore = usedRegisters[block][instIndex];
            RegisterSet& setAfter = usedRegisters[block][instIndex + 1];
            Inst& inst = block->at(instIndex);
            inst.forEachTmp(
                [&] (Tmp& tmp, Arg::Role role, Arg::Type type) {
                    if (tmp.isReg())
                        return;

                    StackSlot* stackSlot = allStackSlots[type][tmp.tmpIndex()];
                    Arg arg = Arg::stack(stackSlot);

                    // Need to figure out a register to use. How we do that depends on the role.
                    Reg chosenReg;
                    switch (role) {
                    case Arg::Use:
                        for (Reg reg : regsInPriorityOrder(type)) {
                            if (!setBefore.get(reg)) {
                                setBefore.set(reg);
                                chosenReg = reg;
                                break;
                            }
                        }
                        break;
                    case Arg::Def:
                        for (Reg reg : regsInPriorityOrder(type)) {
                            if (!setAfter.get(reg)) {
                                setAfter.set(reg);
                                chosenReg = reg;
                                break;
                            }
                        }
                        break;
                    case Arg::UseDef:
                        for (Reg reg : regsInPriorityOrder(type)) {
                            if (!setBefore.get(reg) && !setAfter.get(reg)) {
                                setAfter.set(reg);
                                setBefore.set(reg);
                                chosenReg = reg;
                                break;
                            }
                        }
                        break;
                    }
                    RELEASE_ASSERT(chosenReg);

                    tmp = Tmp(chosenReg);

                    Opcode move = type == Arg::GP ? Move : MoveDouble;

                    if (Arg::isUse(role)) {
                        insertionSet.insert(
                            instIndex, move, inst.origin, arg, tmp);
                    }
                    if (Arg::isDef(role)) {
                        insertionSet.insert(
                            instIndex + 1, move, inst.origin, tmp, arg);
                    }
                });
        }
        insertionSet.execute(block);
    }
}
    bool run()
    {
        ASSERT(m_graph.m_form == ThreadedCPS);
        
        for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
            VariableAccessData* variable = &m_graph.m_variableAccessData[i];
            if (!variable->isRoot())
                continue;
            variable->clearVotes();
        }
        
        // Identify the set of variables that are always subject to the same structure
        // checks. For now, only consider monomorphic structure checks (one structure).
        
        for (BlockIndex blockIndex = 0; blockIndex < m_graph.m_blocks.size(); ++blockIndex) {
            BasicBlock* block = m_graph.m_blocks[blockIndex].get();
            if (!block)
                continue;
            for (unsigned indexInBlock = 0; indexInBlock < block->size(); ++indexInBlock) {
                Node* node = block->at(indexInBlock);
                switch (node->op()) {
                case CheckStructure:
                case StructureTransitionWatchpoint: {
                    Node* child = node->child1().node();
                    if (child->op() != GetLocal)
                        break;
                    VariableAccessData* variable = child->variableAccessData();
                    variable->vote(VoteStructureCheck);
                    if (!shouldConsiderForHoisting(variable))
                        break;
                    noticeStructureCheck(variable, node->structureSet());
                    break;
                }
                    
                case ForwardCheckStructure:
                case ForwardStructureTransitionWatchpoint:
                    // We currently rely on the fact that we're the only ones who would
                    // insert this node.
                    RELEASE_ASSERT_NOT_REACHED();
                    break;
                    
                case GetByOffset:
                case PutByOffset:
                case PutStructure:
                case AllocatePropertyStorage:
                case ReallocatePropertyStorage:
                case GetButterfly:
                case GetByVal:
                case PutByVal:
                case PutByValAlias:
                case GetArrayLength:
                case CheckArray:
                case GetIndexedPropertyStorage:
                case Phantom:
                    // Don't count these uses.
                    break;
                    
                case ArrayifyToStructure:
                case Arrayify:
                    if (node->arrayMode().conversion() == Array::RageConvert) {
                        // Rage conversion changes structures. We should avoid tying to do
                        // any kind of hoisting when rage conversion is in play.
                        Node* child = node->child1().node();
                        if (child->op() != GetLocal)
                            break;
                        VariableAccessData* variable = child->variableAccessData();
                        variable->vote(VoteOther);
                        if (!shouldConsiderForHoisting(variable))
                            break;
                        noticeStructureCheck(variable, 0);
                    }
                    break;
                    
                case SetLocal: {
                    // Find all uses of the source of the SetLocal. If any of them are a
                    // kind of CheckStructure, then we should notice them to ensure that
                    // we're not hoisting a check that would contravene checks that are
                    // already being performed.
                    VariableAccessData* variable = node->variableAccessData();
                    if (!shouldConsiderForHoisting(variable))
                        break;
                    Node* source = node->child1().node();
                    for (unsigned subIndexInBlock = 0; subIndexInBlock < block->size(); ++subIndexInBlock) {
                        Node* subNode = block->at(subIndexInBlock);
                        switch (subNode->op()) {
                        case CheckStructure: {
                            if (subNode->child1() != source)
                                break;
                            
                            noticeStructureCheck(variable, subNode->structureSet());
                            break;
                        }
                        case StructureTransitionWatchpoint: {
                            if (subNode->child1() != source)
                                break;
                            
                            noticeStructureCheck(variable, subNode->structure());
                            break;
                        }
                        default:
                            break;
                        }
                    }
                    
                    m_graph.voteChildren(node, VoteOther);
                    break;
                }
                case GarbageValue:
                    break;
                    
                default:
                    m_graph.voteChildren(node, VoteOther);
                    break;
                }
            }
        }
        
        // Disable structure hoisting on variables that appear to mostly be used in
        // contexts where it doesn't make sense.
        
        for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
            VariableAccessData* variable = &m_graph.m_variableAccessData[i];
            if (!variable->isRoot())
                continue;
            if (variable->voteRatio() >= Options::structureCheckVoteRatioForHoisting())
                continue;
            HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
            if (iter == m_map.end())
                continue;
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
            dataLog(
                "Zeroing the structure to hoist for ", VariableAccessDataDump(m_graph, variable),
                " because the ratio is ", variable->voteRatio(), ".\n");
#endif
            iter->value.m_structure = 0;
        }
        
        // Disable structure check hoisting for variables that cross the OSR entry that
        // we're currently taking, and where the value currently does not have the
        // structure we want.
        
        for (BlockIndex blockIndex = 0; blockIndex < m_graph.m_blocks.size(); ++blockIndex) {
            BasicBlock* block = m_graph.m_blocks[blockIndex].get();
            if (!block)
                continue;
            ASSERT(block->isReachable);
            if (!block->isOSRTarget)
                continue;
            if (block->bytecodeBegin != m_graph.m_osrEntryBytecodeIndex)
                continue;
            for (size_t i = 0; i < m_graph.m_mustHandleValues.size(); ++i) {
                int operand = m_graph.m_mustHandleValues.operandForIndex(i);
                Node* node = block->variablesAtHead.operand(operand);
                if (!node)
                    continue;
                VariableAccessData* variable = node->variableAccessData();
                HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
                if (iter == m_map.end())
                    continue;
                if (!iter->value.m_structure)
                    continue;
                JSValue value = m_graph.m_mustHandleValues[i];
                if (!value || !value.isCell()) {
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
                    dataLog(
                        "Zeroing the structure to hoist for ", VariableAccessDataDump(m_graph, variable),
                        " because the OSR entry value is not a cell: ", value, ".\n");
#endif
                    iter->value.m_structure = 0;
                    continue;
                }
                if (value.asCell()->structure() != iter->value.m_structure) {
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
                    dataLog(
                        "Zeroing the structure to hoist for ", VariableAccessDataDump(m_graph, variable),
                        " because the OSR entry value has structure ",
                        RawPointer(value.asCell()->structure()), " and we wanted ",
                        RawPointer(iter->value.m_structure), ".\n");
#endif
                    iter->value.m_structure = 0;
                    continue;
                }
            }
        }

        bool changed = false;

#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
        for (HashMap<VariableAccessData*, CheckData>::iterator it = m_map.begin();
            it != m_map.end(); ++it) {
            if (!it->value.m_structure) {
                dataLog(
                    "Not hoisting checks for ", VariableAccessDataDump(m_graph, it->key),
                    " because of heuristics.\n");
                continue;
            }
            dataLog("Hoisting checks for ", VariableAccessDataDump(m_graph, it->key), "\n");
        }
#endif // DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
        
        // Place CheckStructure's at SetLocal sites.
        
        InsertionSet insertionSet(m_graph);
        for (BlockIndex blockIndex = 0; blockIndex < m_graph.m_blocks.size(); ++blockIndex) {
            BasicBlock* block = m_graph.m_blocks[blockIndex].get();
            if (!block)
                continue;
            for (unsigned indexInBlock = 0; indexInBlock < block->size(); ++indexInBlock) {
                Node* node = block->at(indexInBlock);
                // Be careful not to use 'node' after appending to the graph. In those switch
                // cases where we need to append, we first carefully extract everything we need
                // from the node, before doing any appending.
                switch (node->op()) {
                case SetArgument: {
                    ASSERT(!blockIndex);
                    // Insert a GetLocal and a CheckStructure immediately following this
                    // SetArgument, if the variable was a candidate for structure hoisting.
                    // If the basic block previously only had the SetArgument as its
                    // variable-at-tail, then replace it with this GetLocal.
                    VariableAccessData* variable = node->variableAccessData();
                    HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
                    if (iter == m_map.end())
                        break;
                    if (!iter->value.m_structure)
                        break;
                    
                    CodeOrigin codeOrigin = node->codeOrigin;
                    
                    Node* getLocal = insertionSet.insertNode(
                        indexInBlock + 1, variable->prediction(), GetLocal, codeOrigin,
                        OpInfo(variable), Edge(node));
                    insertionSet.insertNode(
                        indexInBlock + 1, SpecNone, CheckStructure, codeOrigin,
                        OpInfo(m_graph.addStructureSet(iter->value.m_structure)),
                        Edge(getLocal, CellUse));

                    if (block->variablesAtTail.operand(variable->local()) == node)
                        block->variablesAtTail.operand(variable->local()) = getLocal;
                    
                    m_graph.substituteGetLocal(*block, indexInBlock, variable, getLocal);
                    
                    changed = true;
                    break;
                }
                    
                case SetLocal: {
                    VariableAccessData* variable = node->variableAccessData();
                    HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
                    if (iter == m_map.end())
                        break;
                    if (!iter->value.m_structure)
                        break;

                    // First insert a dead SetLocal to tell OSR that the child's value should
                    // be dropped into this bytecode variable if the CheckStructure decides
                    // to exit.
                    
                    CodeOrigin codeOrigin = node->codeOrigin;
                    Edge child1 = node->child1();
                    
                    insertionSet.insertNode(
                        indexInBlock, SpecNone, SetLocal, codeOrigin, OpInfo(variable), child1);

                    // Use a ForwardCheckStructure to indicate that we should exit to the
                    // next bytecode instruction rather than reexecuting the current one.
                    insertionSet.insertNode(
                        indexInBlock, SpecNone, ForwardCheckStructure, codeOrigin,
                        OpInfo(m_graph.addStructureSet(iter->value.m_structure)),
                        Edge(child1.node(), CellUse));
                    changed = true;
                    break;
                }
                    
                default:
                    break;
                }
            }
            insertionSet.execute(block);
        }
        
        return changed;
    }
void handleCalleeSaves(Code& code)
{
    PhaseScope phaseScope(code, "handleCalleeSaves");

    RegisterSet usedCalleeSaves;

    for (BasicBlock* block : code) {
        for (Inst& inst : *block) {
            inst.forEachTmpFast(
                [&] (Tmp& tmp) {
                    // At first we just record all used regs.
                    usedCalleeSaves.set(tmp.reg());
                });

            if (inst.hasSpecial())
                usedCalleeSaves.merge(inst.extraClobberedRegs());
        }
    }

    // Now we filter to really get the callee saves.
    usedCalleeSaves.filter(RegisterSet::calleeSaveRegisters());
    usedCalleeSaves.exclude(RegisterSet::stackRegisters()); // We don't need to save FP here.

    if (!usedCalleeSaves.numberOfSetRegisters())
        return;

    code.calleeSaveRegisters() = RegisterAtOffsetList(usedCalleeSaves);

    size_t byteSize = 0;
    for (const RegisterAtOffset& entry : code.calleeSaveRegisters())
        byteSize = std::max(static_cast<size_t>(-entry.offset()), byteSize);

    StackSlot* savesArea = code.addStackSlot(byteSize, StackSlotKind::Locked);
    // This is a bit weird since we could have already pinned a different stack slot to this
    // area. Also, our runtime does not require us to pin the saves area. Maybe we shouldn't pin it?
    savesArea->setOffsetFromFP(-byteSize);

    auto argFor = [&] (const RegisterAtOffset& entry) -> Arg {
        return Arg::stack(savesArea, entry.offset() + byteSize);
    };

    InsertionSet insertionSet(code);
    
    // First insert saving code in the prologue.
    for (const RegisterAtOffset& entry : code.calleeSaveRegisters()) {
        insertionSet.insert(
            0, entry.reg().isGPR() ? Move : MoveDouble, code[0]->at(0).origin,
            Tmp(entry.reg()), argFor(entry));
    }
    insertionSet.execute(code[0]);

    // Now insert restore code at epilogues.
    for (BasicBlock* block : code) {
        Inst& last = block->last();
        if (!isReturn(last.opcode))
            continue;

        for (const RegisterAtOffset& entry : code.calleeSaveRegisters()) {
            insertionSet.insert(
                block->size() - 1, entry.reg().isGPR() ? Move : MoveDouble, last.origin,
                argFor(entry), Tmp(entry.reg()));
        }
        insertionSet.execute(block);
    }
}
Exemple #10
0
void allocateStack(Code& code)
{
    PhaseScope phaseScope(code, "allocateStack");

    // Perform an escape analysis over stack slots. An escaping stack slot is one that is locked or
    // is explicitly escaped in the code.
    IndexSet<StackSlot> escapingStackSlots;
    for (StackSlot* slot : code.stackSlots()) {
        if (slot->isLocked())
            escapingStackSlots.add(slot);
    }
    for (BasicBlock* block : code) {
        for (Inst& inst : *block) {
            inst.forEachArg(
                [&] (Arg& arg, Arg::Role role, Arg::Type, Arg::Width) {
                    if (role == Arg::UseAddr && arg.isStack())
                        escapingStackSlots.add(arg.stackSlot());
                });
        }
    }

    // Allocate all of the escaped slots in order. This is kind of a crazy algorithm to allow for
    // the possibility of stack slots being assigned frame offsets before we even get here.
    ASSERT(!code.frameSize());
    Vector<StackSlot*> assignedEscapedStackSlots;
    Vector<StackSlot*> escapedStackSlotsWorklist;
    for (StackSlot* slot : code.stackSlots()) {
        if (escapingStackSlots.contains(slot)) {
            if (slot->offsetFromFP())
                assignedEscapedStackSlots.append(slot);
            else
                escapedStackSlotsWorklist.append(slot);
        } else {
            // It would be super strange to have an unlocked stack slot that has an offset already.
            ASSERT(!slot->offsetFromFP());
        }
    }
    // This is a fairly expensive loop, but it's OK because we'll usually only have a handful of
    // escaped stack slots.
    while (!escapedStackSlotsWorklist.isEmpty()) {
        StackSlot* slot = escapedStackSlotsWorklist.takeLast();
        assign(slot, assignedEscapedStackSlots);
        assignedEscapedStackSlots.append(slot);
    }

    // Now we handle the anonymous slots.
    StackSlotLiveness liveness(code);
    IndexMap<StackSlot, HashSet<StackSlot*>> interference(code.stackSlots().size());
    Vector<StackSlot*> slots;

    for (BasicBlock* block : code) {
        StackSlotLiveness::LocalCalc localCalc(liveness, block);

        auto interfere = [&] (unsigned instIndex) {
            if (verbose)
                dataLog("Interfering: ", WTF::pointerListDump(localCalc.live()), "\n");

            Inst::forEachDef<Arg>(
                block->get(instIndex), block->get(instIndex + 1),
                [&] (Arg& arg, Arg::Role, Arg::Type, Arg::Width) {
                    if (!arg.isStack())
                        return;
                    StackSlot* slot = arg.stackSlot();
                    if (slot->kind() != StackSlotKind::Anonymous)
                        return;

                    for (StackSlot* otherSlot : localCalc.live()) {
                        interference[slot].add(otherSlot);
                        interference[otherSlot].add(slot);
                    }
                });
        };

        for (unsigned instIndex = block->size(); instIndex--;) {
            if (verbose)
                dataLog("Analyzing: ", block->at(instIndex), "\n");

            // Kill dead stores. For simplicity we say that a store is killable if it has only late
            // defs and those late defs are to things that are dead right now. We only do that
            // because that's the only kind of dead stack store we will see here.
            Inst& inst = block->at(instIndex);
            if (!inst.hasNonArgEffects()) {
                bool ok = true;
                inst.forEachArg(
                    [&] (Arg& arg, Arg::Role role, Arg::Type, Arg::Width) {
                        if (Arg::isEarlyDef(role)) {
                            ok = false;
                            return;
                        }
                        if (!Arg::isLateDef(role))
                            return;
                        if (!arg.isStack()) {
                            ok = false;
                            return;
                        }
                        StackSlot* slot = arg.stackSlot();
                        if (slot->kind() != StackSlotKind::Anonymous) {
                            ok = false;
                            return;
                        }

                        if (localCalc.isLive(slot)) {
                            ok = false;
                            return;
                        }
                    });
                if (ok)
                    inst = Inst();
            }
            
            interfere(instIndex);
            localCalc.execute(instIndex);
        }
        interfere(-1);
        
        block->insts().removeAllMatching(
            [&] (const Inst& inst) -> bool {
                return !inst;
            });
    }

    if (verbose) {
        for (StackSlot* slot : code.stackSlots())
            dataLog("Interference of ", pointerDump(slot), ": ", pointerListDump(interference[slot]), "\n");
    }

    // Now we assign stack locations. At its heart this algorithm is just first-fit. For each
    // StackSlot we just want to find the offsetFromFP that is closest to zero while ensuring no
    // overlap with other StackSlots that this overlaps with.
    Vector<StackSlot*> otherSlots = assignedEscapedStackSlots;
    for (StackSlot* slot : code.stackSlots()) {
        if (slot->offsetFromFP()) {
            // Already assigned an offset.
            continue;
        }

        HashSet<StackSlot*>& interferingSlots = interference[slot];
        otherSlots.resize(assignedEscapedStackSlots.size());
        otherSlots.resize(assignedEscapedStackSlots.size() + interferingSlots.size());
        unsigned nextIndex = assignedEscapedStackSlots.size();
        for (StackSlot* otherSlot : interferingSlots)
            otherSlots[nextIndex++] = otherSlot;

        assign(slot, otherSlots);
    }

    // Figure out how much stack we're using for stack slots.
    unsigned frameSizeForStackSlots = 0;
    for (StackSlot* slot : code.stackSlots()) {
        frameSizeForStackSlots = std::max(
            frameSizeForStackSlots,
            static_cast<unsigned>(-slot->offsetFromFP()));
    }

    frameSizeForStackSlots = WTF::roundUpToMultipleOf(stackAlignmentBytes(), frameSizeForStackSlots);

    // Now we need to deduce how much argument area we need.
    for (BasicBlock* block : code) {
        for (Inst& inst : *block) {
            for (Arg& arg : inst.args) {
                if (arg.isCallArg()) {
                    // For now, we assume that we use 8 bytes of the call arg. But that's not
                    // such an awesome assumption.
                    // FIXME: https://bugs.webkit.org/show_bug.cgi?id=150454
                    ASSERT(arg.offset() >= 0);
                    code.requestCallArgAreaSize(arg.offset() + 8);
                }
            }
        }
    }

    code.setFrameSize(frameSizeForStackSlots + code.callArgAreaSize());

    // Finally, transform the code to use Addr's instead of StackSlot's. This is a lossless
    // transformation since we can search the StackSlots array to figure out which StackSlot any
    // offset-from-FP refers to.

    // FIXME: This may produce addresses that aren't valid if we end up with a ginormous stack frame.
    // We would have to scavenge for temporaries if this happened. Fortunately, this case will be
    // extremely rare so we can do crazy things when it arises.
    // https://bugs.webkit.org/show_bug.cgi?id=152530

    InsertionSet insertionSet(code);
    for (BasicBlock* block : code) {
        for (unsigned instIndex = 0; instIndex < block->size(); ++instIndex) {
            Inst& inst = block->at(instIndex);
            inst.forEachArg(
                [&] (Arg& arg, Arg::Role role, Arg::Type, Arg::Width width) {
                    auto stackAddr = [&] (int32_t offset) -> Arg {
                        return Arg::stackAddr(offset, code.frameSize(), width);
                    };
                    
                    switch (arg.kind()) {
                    case Arg::Stack: {
                        StackSlot* slot = arg.stackSlot();
                        if (Arg::isZDef(role)
                            && slot->kind() == StackSlotKind::Anonymous
                            && slot->byteSize() > Arg::bytes(width)) {
                            // Currently we only handle this simple case because it's the only one
                            // that arises: ZDef's are only 32-bit right now. So, when we hit these
                            // assertions it means that we need to implement those other kinds of
                            // zero fills.
                            RELEASE_ASSERT(slot->byteSize() == 8);
                            RELEASE_ASSERT(width == Arg::Width32);

                            RELEASE_ASSERT(isValidForm(StoreZero32, Arg::Stack));
                            insertionSet.insert(
                                instIndex + 1, StoreZero32, inst.origin,
                                stackAddr(arg.offset() + 4 + slot->offsetFromFP()));
                        }
                        arg = stackAddr(arg.offset() + slot->offsetFromFP());
                        break;
                    }
                    case Arg::CallArg:
                        arg = stackAddr(arg.offset() - code.frameSize());
                        break;
                    default:
                        break;
                    }
                }
            );
        }
        insertionSet.execute(block);
    }
}
    bool run()
    {
        RELEASE_ASSERT(m_graph.m_plan.mode == FTLForOSREntryMode);
        RELEASE_ASSERT(m_graph.m_form == ThreadedCPS);
        
        unsigned bytecodeIndex = m_graph.m_plan.osrEntryBytecodeIndex;
        RELEASE_ASSERT(bytecodeIndex);
        RELEASE_ASSERT(bytecodeIndex != UINT_MAX);
        
        // Needed by createPreHeader().
        m_graph.ensureDominators();
        
        CodeBlock* baseline = m_graph.m_profiledBlock;
        
        BasicBlock* target = 0;
        for (unsigned blockIndex = m_graph.numBlocks(); blockIndex--;) {
            BasicBlock* block = m_graph.block(blockIndex);
            if (!block)
                continue;
            unsigned nodeIndex = 0;
            Node* firstNode = block->at(0);
            while (firstNode->isSemanticallySkippable())
                firstNode = block->at(++nodeIndex);
            if (firstNode->op() == LoopHint
                && firstNode->origin.semantic == CodeOrigin(bytecodeIndex)) {
                target = block;
                break;
            }
        }

        if (!target) {
            // This is a terrible outcome. It shouldn't often happen but it might
            // happen and so we should defend against it. If it happens, then this
            // compilation is a failure.
            return false;
        }
        
        BlockInsertionSet insertionSet(m_graph);
        
        // We say that the execution count of the entry block is 1, because we know for sure
        // that this must be the case. Under our definition of executionCount, "1" means "once
        // per invocation". We could have said NaN here, since that would ask any clients of
        // executionCount to use best judgement - but that seems unnecessary since we know for
        // sure what the executionCount should be in this case.
        BasicBlock* newRoot = insertionSet.insert(0, 1);

        // We'd really like to use an unset origin, but ThreadedCPS won't allow that.
        NodeOrigin origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), false);
        
        Vector<Node*> locals(baseline->m_numCalleeLocals);
        for (int local = 0; local < baseline->m_numCalleeLocals; ++local) {
            Node* previousHead = target->variablesAtHead.local(local);
            if (!previousHead)
                continue;
            VariableAccessData* variable = previousHead->variableAccessData();
            locals[local] = newRoot->appendNode(
                m_graph, variable->prediction(), ExtractOSREntryLocal, origin,
                OpInfo(variable->local().offset()));
            
            newRoot->appendNode(
                m_graph, SpecNone, MovHint, origin, OpInfo(variable->local().offset()),
                Edge(locals[local]));
        }

        // Now use the origin of the target, since it's not OK to exit, and we will probably hoist
        // type checks to here.
        origin = target->at(0)->origin;
        
        for (int argument = 0; argument < baseline->numParameters(); ++argument) {
            Node* oldNode = target->variablesAtHead.argument(argument);
            if (!oldNode) {
                // Just for sanity, always have a SetArgument even if it's not needed.
                oldNode = m_graph.m_arguments[argument];
            }
            Node* node = newRoot->appendNode(
                m_graph, SpecNone, SetArgument, origin,
                OpInfo(oldNode->variableAccessData()));
            m_graph.m_arguments[argument] = node;
        }

        for (int local = 0; local < baseline->m_numCalleeLocals; ++local) {
            Node* previousHead = target->variablesAtHead.local(local);
            if (!previousHead)
                continue;
            VariableAccessData* variable = previousHead->variableAccessData();
            Node* node = locals[local];
            newRoot->appendNode(
                m_graph, SpecNone, SetLocal, origin, OpInfo(variable), Edge(node));
        }
        
        newRoot->appendNode(
            m_graph, SpecNone, Jump, origin,
            OpInfo(createPreHeader(m_graph, insertionSet, target)));
        
        insertionSet.execute();
        m_graph.resetReachability();
        m_graph.killUnreachableBlocks();
        return true;
    }
Exemple #12
0
void demoteValues(Procedure& proc, const IndexSet<Value>& values)
{
    HashMap<Value*, StackSlotValue*> map;
    HashMap<Value*, StackSlotValue*> phiMap;

    // Create stack slots.
    InsertionSet insertionSet(proc);
    for (Value* value : values.values(proc.values())) {
        StackSlotValue* stack = insertionSet.insert<StackSlotValue>(
            0, value->origin(), sizeofType(value->type()), StackSlotKind::Anonymous);
        map.add(value, stack);

        if (value->opcode() == Phi) {
            StackSlotValue* phiStack = insertionSet.insert<StackSlotValue>(
                0, value->origin(), sizeofType(value->type()), StackSlotKind::Anonymous);
            phiMap.add(value, phiStack);
        }
    }
    insertionSet.execute(proc[0]);

    if (verbose) {
        dataLog("Demoting values as follows:\n");
        dataLog("   map = ");
        CommaPrinter comma;
        for (auto& entry : map)
            dataLog(comma, *entry.key, "=>", *entry.value);
        dataLog("\n");
        dataLog("   phiMap = ");
        comma = CommaPrinter();
        for (auto& entry : phiMap)
            dataLog(comma, *entry.key, "=>", *entry.value);
        dataLog("\n");
    }

    // Change accesses to the values to accesses to the stack slots.
    for (BasicBlock* block : proc) {
        for (unsigned valueIndex = 0; valueIndex < block->size(); ++valueIndex) {
            Value* value = block->at(valueIndex);

            if (value->opcode() == Phi) {
                if (StackSlotValue* stack = phiMap.get(value)) {
                    value->replaceWithIdentity(
                        insertionSet.insert<MemoryValue>(
                            valueIndex, Load, value->type(), value->origin(), stack));
                }
            } else {
                for (Value*& child : value->children()) {
                    if (StackSlotValue* stack = map.get(child)) {
                        child = insertionSet.insert<MemoryValue>(
                            valueIndex, Load, child->type(), value->origin(), stack);
                    }
                }

                if (UpsilonValue* upsilon = value->as<UpsilonValue>()) {
                    if (StackSlotValue* stack = phiMap.get(upsilon->phi())) {
                        insertionSet.insert<MemoryValue>(
                            valueIndex, Store, upsilon->origin(), upsilon->child(0), stack);
                        value->replaceWithNop();
                    }
                }
            }

            if (StackSlotValue* stack = map.get(value)) {
                insertionSet.insert<MemoryValue>(
                    valueIndex + 1, Store, value->origin(), value, stack);
            }
        }
        insertionSet.execute(block);
    }
}
Exemple #13
0
bool fixSSA(Procedure& proc)
{
    PhaseScope phaseScope(proc, "fixSSA");
    
    // Collect the stack "variables". If there aren't any, then we don't have anything to do.
    // That's a fairly common case.
    HashMap<StackSlotValue*, Type> stackVariable;
    for (Value* value : proc.values()) {
        if (StackSlotValue* stack = value->as<StackSlotValue>()) {
            if (stack->kind() == StackSlotKind::Anonymous)
                stackVariable.add(stack, Void);
        }
    }

    if (stackVariable.isEmpty())
        return false;

    // Make sure that we know how to optimize all of these. We only know how to handle Load and
    // Store on anonymous variables.
    for (Value* value : proc.values()) {
        auto reject = [&] (Value* value) {
            if (StackSlotValue* stack = value->as<StackSlotValue>())
                stackVariable.remove(stack);
        };
        
        auto handleAccess = [&] (Value* access, Type type) {
            StackSlotValue* stack = access->lastChild()->as<StackSlotValue>();
            if (!stack)
                return;
            
            if (value->as<MemoryValue>()->offset()) {
                stackVariable.remove(stack);
                return;
            }

            auto result = stackVariable.find(stack);
            if (result == stackVariable.end())
                return;
            if (result->value == Void) {
                result->value = type;
                return;
            }
            if (result->value == type)
                return;
            stackVariable.remove(result);
        };
        
        switch (value->opcode()) {
        case Load:
            // We're OK with loads from stack variables at an offset of zero.
            handleAccess(value, value->type());
            break;
        case Store:
            // We're OK with stores to stack variables, but not storing stack variables.
            reject(value->child(0));
            handleAccess(value, value->child(0)->type());
            break;
        default:
            for (Value* child : value->children())
                reject(child);
            break;
        }
    }

    Vector<StackSlotValue*> deadValues;
    for (auto& entry : stackVariable) {
        if (entry.value == Void)
            deadValues.append(entry.key);
    }

    for (StackSlotValue* deadValue : deadValues) {
        deadValue->replaceWithNop();
        stackVariable.remove(deadValue);
    }

    if (stackVariable.isEmpty())
        return false;

    // We know that we have variables to optimize, so do that now.
    breakCriticalEdges(proc);

    SSACalculator ssa(proc);

    // Create a SSACalculator::Variable for every stack variable.
    Vector<StackSlotValue*> variableToStack;
    HashMap<StackSlotValue*, SSACalculator::Variable*> stackToVariable;

    for (auto& entry : stackVariable) {
        StackSlotValue* stack = entry.key;
        SSACalculator::Variable* variable = ssa.newVariable();
        RELEASE_ASSERT(variable->index() == variableToStack.size());
        variableToStack.append(stack);
        stackToVariable.add(stack, variable);
    }

    // Create Defs for all of the stores to the stack variable.
    for (BasicBlock* block : proc) {
        for (Value* value : *block) {
            if (value->opcode() != Store)
                continue;

            StackSlotValue* stack = value->child(1)->as<StackSlotValue>();
            if (!stack)
                continue;

            if (SSACalculator::Variable* variable = stackToVariable.get(stack))
                ssa.newDef(variable, block, value->child(0));
        }
    }

    // Decide where Phis are to be inserted. This creates them but does not insert them.
    ssa.computePhis(
        [&] (SSACalculator::Variable* variable, BasicBlock* block) -> Value* {
            StackSlotValue* stack = variableToStack[variable->index()];
            Value* phi = proc.add<Value>(Phi, stackVariable.get(stack), stack->origin());
            if (verbose) {
                dataLog(
                    "Adding Phi for ", pointerDump(stack), " at ", *block, ": ",
                    deepDump(proc, phi), "\n");
            }
            return phi;
        });

    // Now perform the conversion.
    InsertionSet insertionSet(proc);
    HashMap<StackSlotValue*, Value*> mapping;
    for (BasicBlock* block : proc.blocksInPreOrder()) {
        mapping.clear();

        for (auto& entry : stackToVariable) {
            StackSlotValue* stack = entry.key;
            SSACalculator::Variable* variable = entry.value;

            SSACalculator::Def* def = ssa.reachingDefAtHead(block, variable);
            if (def)
                mapping.set(stack, def->value());
        }

        for (SSACalculator::Def* phiDef : ssa.phisForBlock(block)) {
            StackSlotValue* stack = variableToStack[phiDef->variable()->index()];

            insertionSet.insertValue(0, phiDef->value());
            mapping.set(stack, phiDef->value());
        }

        for (unsigned valueIndex = 0; valueIndex < block->size(); ++valueIndex) {
            Value* value = block->at(valueIndex);
            value->performSubstitution();

            switch (value->opcode()) {
            case Load: {
                if (StackSlotValue* stack = value->child(0)->as<StackSlotValue>()) {
                    if (Value* replacement = mapping.get(stack))
                        value->replaceWithIdentity(replacement);
                }
                break;
            }
                
            case Store: {
                if (StackSlotValue* stack = value->child(1)->as<StackSlotValue>()) {
                    if (stackToVariable.contains(stack)) {
                        mapping.set(stack, value->child(0));
                        value->replaceWithNop();
                    }
                }
                break;
            }

            default:
                break;
            }
        }

        unsigned upsilonInsertionPoint = block->size() - 1;
        Origin upsilonOrigin = block->last()->origin();
        for (BasicBlock* successorBlock : block->successorBlocks()) {
            for (SSACalculator::Def* phiDef : ssa.phisForBlock(successorBlock)) {
                Value* phi = phiDef->value();
                SSACalculator::Variable* variable = phiDef->variable();
                StackSlotValue* stack = variableToStack[variable->index()];

                Value* mappedValue = mapping.get(stack);
                if (verbose) {
                    dataLog(
                        "Mapped value for ", *stack, " with successor Phi ", *phi, " at end of ",
                        *block, ": ", pointerDump(mappedValue), "\n");
                }
                
                if (!mappedValue)
                    mappedValue = insertionSet.insertBottom(upsilonInsertionPoint, phi);
                
                insertionSet.insert<UpsilonValue>(
                    upsilonInsertionPoint, upsilonOrigin, mappedValue, phi);
            }
        }

        insertionSet.execute(block);
    }

    // Finally, kill the stack slots.
    for (StackSlotValue* stack : variableToStack)
        stack->replaceWithNop();

    if (verbose) {
        dataLog("B3 after SSA conversion:\n");
        dataLog(proc);
    }

    return true;
}
    bool run()
    {
        RELEASE_ASSERT(m_graph.m_plan.mode == DFGMode);
        
        if (!Options::useFTLJIT())
            return false;
        
        if (m_graph.m_profiledBlock->m_didFailFTLCompilation)
            return false;

        if (!Options::bytecodeRangeToFTLCompile().isInRange(m_graph.m_profiledBlock->instructionCount()))
            return false;

#if ENABLE(FTL_JIT)
        FTL::CapabilityLevel level = FTL::canCompile(m_graph);
        if (level == FTL::CannotCompile)
            return false;
        
        if (!Options::useOSREntryToFTL())
            level = FTL::CanCompile;

        m_graph.ensureNaturalLoops();
        NaturalLoops& naturalLoops = *m_graph.m_naturalLoops;
        HashMap<const NaturalLoop*, unsigned> naturalLoopToLoopHint = buildNaturalLoopToLoopHintMap(naturalLoops);

        HashMap<unsigned, LoopHintDescriptor> tierUpHierarchy;

        InsertionSet insertionSet(m_graph);
        for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
            BasicBlock* block = m_graph.block(blockIndex);
            if (!block)
                continue;

            for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
                Node* node = block->at(nodeIndex);
                if (node->op() != LoopHint)
                    continue;

                NodeOrigin origin = node->origin;
                bool canOSREnter = canOSREnterAtLoopHint(level, block, nodeIndex);

                NodeType tierUpType = CheckTierUpAndOSREnter;
                if (!canOSREnter)
                    tierUpType = CheckTierUpInLoop;
                insertionSet.insertNode(nodeIndex + 1, SpecNone, tierUpType, origin);

                unsigned bytecodeIndex = origin.semantic.bytecodeIndex;
                if (canOSREnter)
                    m_graph.m_plan.tierUpAndOSREnterBytecodes.append(bytecodeIndex);

                if (const NaturalLoop* loop = naturalLoops.innerMostLoopOf(block)) {
                    LoopHintDescriptor descriptor;
                    descriptor.canOSREnter = canOSREnter;

                    const NaturalLoop* outerLoop = loop;
                    while ((outerLoop = naturalLoops.innerMostOuterLoop(*outerLoop))) {
                        auto it = naturalLoopToLoopHint.find(outerLoop);
                        if (it != naturalLoopToLoopHint.end())
                            descriptor.osrEntryCandidates.append(it->value);
                    }
                    if (!descriptor.osrEntryCandidates.isEmpty())
                        tierUpHierarchy.add(bytecodeIndex, WTFMove(descriptor));
                }
                break;
            }

            NodeAndIndex terminal = block->findTerminal();
            if (terminal.node->isFunctionTerminal()) {
                insertionSet.insertNode(
                    terminal.index, SpecNone, CheckTierUpAtReturn, terminal.node->origin);
            }

            insertionSet.execute(block);
        }

        // Add all the candidates that can be OSR Entered.
        for (auto entry : tierUpHierarchy) {
            Vector<unsigned> tierUpCandidates;
            for (unsigned bytecodeIndex : entry.value.osrEntryCandidates) {
                auto descriptorIt = tierUpHierarchy.find(bytecodeIndex);
                if (descriptorIt != tierUpHierarchy.end()
                    && descriptorIt->value.canOSREnter)
                    tierUpCandidates.append(bytecodeIndex);
            }

            if (!tierUpCandidates.isEmpty())
                m_graph.m_plan.tierUpInLoopHierarchy.add(entry.key, WTFMove(tierUpCandidates));
        }
        m_graph.m_plan.willTryToTierUp = true;
        return true;
#else // ENABLE(FTL_JIT)
        RELEASE_ASSERT_NOT_REACHED();
        return false;
#endif // ENABLE(FTL_JIT)
    }
void spillEverything(Code& code)
{
    PhaseScope phaseScope(code, "spillEverything");

    // We want to know the set of registers used at every point in every basic block.
    IndexMap<BasicBlock, Vector<RegisterSet>> usedRegisters(code.size());
    GPLiveness gpLiveness(code);
    FPLiveness fpLiveness(code);
    for (BasicBlock* block : code) {
        GPLiveness::LocalCalc gpLocalCalc(gpLiveness, block);
        FPLiveness::LocalCalc fpLocalCalc(fpLiveness, block);

        usedRegisters[block].resize(block->size() + 1);

        auto setUsedRegisters = [&] (unsigned index) {
            RegisterSet& registerSet = usedRegisters[block][index];
            for (Tmp tmp : gpLocalCalc.live()) {
                if (tmp.isReg())
                    registerSet.set(tmp.reg());
            }
            for (Tmp tmp : fpLocalCalc.live()) {
                if (tmp.isReg())
                    registerSet.set(tmp.reg());
            }

            // Gotta account for dead assignments to registers. These may happen because the input
            // code is suboptimal.
            Inst::forEachDefWithExtraClobberedRegs<Tmp>(
                block->get(index - 1), block->get(index),
                [&] (const Tmp& tmp, Arg::Role, Arg::Type, Arg::Width) {
                    if (tmp.isReg())
                        registerSet.set(tmp.reg());
                });
        };

        for (unsigned instIndex = block->size(); instIndex--;) {
            setUsedRegisters(instIndex + 1);
            gpLocalCalc.execute(instIndex);
            fpLocalCalc.execute(instIndex);
        }
        setUsedRegisters(0);
    }

    // Allocate a stack slot for each tmp.
    Vector<StackSlot*> allStackSlots[Arg::numTypes];
    for (unsigned typeIndex = 0; typeIndex < Arg::numTypes; ++typeIndex) {
        Vector<StackSlot*>& stackSlots = allStackSlots[typeIndex];
        Arg::Type type = static_cast<Arg::Type>(typeIndex);
        stackSlots.resize(code.numTmps(type));
        for (unsigned tmpIndex = code.numTmps(type); tmpIndex--;)
            stackSlots[tmpIndex] = code.addStackSlot(8, StackSlotKind::Anonymous);
    }

    InsertionSet insertionSet(code);
    for (BasicBlock* block : code) {
        for (unsigned instIndex = 0; instIndex < block->size(); ++instIndex) {
            RegisterSet& setBefore = usedRegisters[block][instIndex];
            RegisterSet& setAfter = usedRegisters[block][instIndex + 1];
            Inst& inst = block->at(instIndex);

            // First try to spill directly.
            for (unsigned i = 0; i < inst.args.size(); ++i) {
                Arg& arg = inst.args[i];

                if (arg.isTmp()) {
                    if (arg.isReg())
                        continue;

                    if (inst.admitsStack(i)) {
                        StackSlot* stackSlot = allStackSlots[arg.type()][arg.tmpIndex()];
                        arg = Arg::stack(stackSlot);
                        continue;
                    }
                }
            }

            // Now fall back on spilling using separate Move's to load/store the tmp.
            inst.forEachTmp(
                [&] (Tmp& tmp, Arg::Role role, Arg::Type type, Arg::Width) {
                    if (tmp.isReg())
                        return;
                    
                    StackSlot* stackSlot = allStackSlots[type][tmp.tmpIndex()];
                    Arg arg = Arg::stack(stackSlot);

                    // Need to figure out a register to use. How we do that depends on the role.
                    Reg chosenReg;
                    switch (role) {
                    case Arg::Use:
                    case Arg::ColdUse:
                        for (Reg reg : regsInPriorityOrder(type)) {
                            if (!setBefore.get(reg)) {
                                setBefore.set(reg);
                                chosenReg = reg;
                                break;
                            }
                        }
                        break;
                    case Arg::Def:
                    case Arg::ZDef:
                        for (Reg reg : regsInPriorityOrder(type)) {
                            if (!setAfter.get(reg)) {
                                setAfter.set(reg);
                                chosenReg = reg;
                                break;
                            }
                        }
                        break;
                    case Arg::UseDef:
                    case Arg::UseZDef:
                    case Arg::LateUse:
                    case Arg::LateColdUse:
                    case Arg::Scratch:
                    case Arg::EarlyDef:
                        for (Reg reg : regsInPriorityOrder(type)) {
                            if (!setBefore.get(reg) && !setAfter.get(reg)) {
                                setAfter.set(reg);
                                setBefore.set(reg);
                                chosenReg = reg;
                                break;
                            }
                        }
                        break;
                    case Arg::UseAddr:
                        // We will never UseAddr a Tmp, that doesn't make sense.
                        RELEASE_ASSERT_NOT_REACHED();
                        break;
                    }
                    RELEASE_ASSERT(chosenReg);

                    tmp = Tmp(chosenReg);

                    Opcode move = type == Arg::GP ? Move : MoveDouble;

                    if (Arg::isAnyUse(role) && role != Arg::Scratch)
                        insertionSet.insert(instIndex, move, inst.origin, arg, tmp);
                    if (Arg::isAnyDef(role))
                        insertionSet.insert(instIndex + 1, move, inst.origin, tmp, arg);
                });
        }
        insertionSet.execute(block);
    }
}
 bool run()
 {
     ASSERT(m_graph.m_form == SSA);
     
     m_graph.clearFlagsOnAllNodes(NodeNeedsPhantom | NodeNeedsHardPhantom | NodeRelevantToOSR);
     
     for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
         BasicBlock* block = m_graph.block(blockIndex);
         if (!block)
             continue;
         
         for (unsigned i = block->size(); i--;) {
             Node* node = block->at(i);
             if (node->op() == MovHint)
                 node->child1()->mergeFlags(NodeRelevantToOSR);
         }
     }
     
     for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
         BasicBlock* block = m_graph.block(blockIndex);
         if (!block)
             continue;
         
         unsigned sourceIndex = 0;
         unsigned targetIndex = 0;
         while (sourceIndex < block->size()) {
             Node* node = block->at(sourceIndex++);
             if (node->op() == HardPhantom || node->op() == Phantom || node->op() == Check) {
                 for (unsigned i = 0; i < AdjacencyList::Size; ++i) {
                     Edge edge = node->children.child(i);
                     if (!edge)
                         break;
                     if (node->op() == HardPhantom)
                         edge->mergeFlags(NodeNeedsHardPhantom);
                     if ((edge->flags() & NodeRelevantToOSR) && node->op() == Phantom) {
                         // A Phantom on a node that is RelevantToOSR means that we need to keep
                         // a Phantom on this node instead of just having a Check.
                         edge->mergeFlags(NodeNeedsPhantom);
                     }
                     if (edge.willHaveCheck())
                         continue; // Keep the type check.
                     
                     node->children.removeEdge(i--);
                 }
                 
                 if (node->children.isEmpty()) {
                     m_graph.m_allocator.free(node);
                     continue;
                 }
                 
                 node->convertToCheck();
             }
             block->at(targetIndex++) = node;
         }
         block->resize(targetIndex);
     }
     
     InsertionSet insertionSet(m_graph);
     for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
         BasicBlock* block = m_graph.block(blockIndex);
         if (!block)
             continue;
         
         for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
             Node* node = block->at(nodeIndex);
             if (node->flags() & NodeNeedsHardPhantom) {
                 insertionSet.insertNode(
                     nodeIndex + 1, SpecNone, HardPhantom, node->origin, node->defaultEdge());
             } else if (node->flags() & NodeNeedsPhantom) {
                 insertionSet.insertNode(
                     nodeIndex + 1, SpecNone, Phantom, node->origin, node->defaultEdge());
             }
         }
         insertionSet.execute(block);
     }
     
     return true;
 }
void fixPartialRegisterStalls(Code& code)
{
    if (!isX86())
        return;

    PhaseScope phaseScope(code, "fixPartialRegisterStalls");

    Vector<BasicBlock*> candidates;

    for (BasicBlock* block : code) {
        for (const Inst& inst : *block) {
            if (hasPartialXmmRegUpdate(inst)) {
                candidates.append(block);
                break;
            }
        }
    }

    // Fortunately, Partial Stalls are rarely used. Return early if no block
    // cares about them.
    if (candidates.isEmpty())
        return;

    // For each block, this provides the distance to the last instruction setting each register
    // on block *entry*.
    IndexMap<BasicBlock, FPDefDistance> lastDefDistance(code.size());

    // Blocks with dirty distance at head.
    IndexSet<BasicBlock> dirty;

    // First, we compute the local distance for each block and push it to the successors.
    for (BasicBlock* block : code) {
        FPDefDistance localDistance;

        unsigned distanceToBlockEnd = block->size();
        for (Inst& inst : *block)
            updateDistances(inst, localDistance, distanceToBlockEnd);

        for (BasicBlock* successor : block->successorBlocks()) {
            if (lastDefDistance[successor].updateFromPrecessor(localDistance))
                dirty.add(successor);
        }
    }

    // Now we propagate the minimums accross blocks.
    bool changed;
    do {
        changed = false;

        for (BasicBlock* block : code) {
            if (!dirty.remove(block))
                continue;

            // Little shortcut: if the block is big enough, propagating it won't add any information.
            if (block->size() >= minimumSafeDistance)
                continue;

            unsigned blockSize = block->size();
            FPDefDistance& blockDistance = lastDefDistance[block];
            for (BasicBlock* successor : block->successorBlocks()) {
                if (lastDefDistance[successor].updateFromPrecessor(blockDistance, blockSize)) {
                    dirty.add(successor);
                    changed = true;
                }
            }
        }
    } while (changed);

    // Finally, update each block as needed.
    InsertionSet insertionSet(code);
    for (BasicBlock* block : candidates) {
        unsigned distanceToBlockEnd = block->size();
        FPDefDistance& localDistance = lastDefDistance[block];

        for (unsigned i = 0; i < block->size(); ++i) {
            Inst& inst = block->at(i);

            if (hasPartialXmmRegUpdate(inst)) {
                RegisterSet defs;
                RegisterSet uses;
                inst.forEachTmp([&] (Tmp& tmp, Arg::Role role, Arg::Type) {
                    if (tmp.isFPR()) {
                        if (Arg::isDef(role))
                            defs.set(tmp.fpr());
                        if (Arg::isAnyUse(role))
                            uses.set(tmp.fpr());
                    }
                });
                // We only care about values we define but not use. Otherwise we have to wait
                // for the value to be resolved anyway.
                defs.exclude(uses);

                defs.forEach([&] (Reg reg) {
                    if (localDistance.distance[MacroAssembler::fpRegisterIndex(reg.fpr())] < minimumSafeDistance)
                        insertionSet.insert(i, MoveZeroToDouble, inst.origin, Tmp(reg));
                });
            }

            updateDistances(inst, localDistance, distanceToBlockEnd);
        }
        insertionSet.execute(block);
    }
}
    bool run()
    {
        RELEASE_ASSERT(m_graph.m_plan.mode == DFGMode);
        
        if (!Options::useFTLJIT())
            return false;
        
        if (m_graph.m_profiledBlock->m_didFailFTLCompilation)
            return false;

        if (!Options::bytecodeRangeToFTLCompile().isInRange(m_graph.m_profiledBlock->instructionCount()))
            return false;

#if ENABLE(FTL_JIT)
        FTL::CapabilityLevel level = FTL::canCompile(m_graph);
        if (level == FTL::CannotCompile)
            return false;
        
        if (!Options::useOSREntryToFTL())
            level = FTL::CanCompile;

        // First we find all the loops that contain a LoopHint for which we cannot OSR enter.
        // We use that information to decide if we need CheckTierUpAndOSREnter or CheckTierUpWithNestedTriggerAndOSREnter.
        m_graph.ensureNaturalLoops();
        NaturalLoops& naturalLoops = *m_graph.m_naturalLoops;

        HashSet<const NaturalLoop*> loopsContainingLoopHintWithoutOSREnter = findLoopsContainingLoopHintWithoutOSREnter(naturalLoops, level);

        bool canTierUpAndOSREnter = false;
        
        InsertionSet insertionSet(m_graph);
        for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
            BasicBlock* block = m_graph.block(blockIndex);
            if (!block)
                continue;
            
            for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
                Node* node = block->at(nodeIndex);
                if (node->op() != LoopHint)
                    continue;

                NodeOrigin origin = node->origin;
                if (canOSREnterAtLoopHint(level, block, nodeIndex)) {
                    canTierUpAndOSREnter = true;
                    const NaturalLoop* loop = naturalLoops.innerMostLoopOf(block);
                    if (loop && loopsContainingLoopHintWithoutOSREnter.contains(loop))
                        insertionSet.insertNode(nodeIndex + 1, SpecNone, CheckTierUpWithNestedTriggerAndOSREnter, origin);
                    else
                        insertionSet.insertNode(nodeIndex + 1, SpecNone, CheckTierUpAndOSREnter, origin);
                } else
                    insertionSet.insertNode(nodeIndex + 1, SpecNone, CheckTierUpInLoop, origin);
                break;
            }
            
            NodeAndIndex terminal = block->findTerminal();
            if (terminal.node->isFunctionTerminal()) {
                insertionSet.insertNode(
                    terminal.index, SpecNone, CheckTierUpAtReturn, terminal.node->origin);
            }
            
            insertionSet.execute(block);
        }

        m_graph.m_plan.canTierUpAndOSREnter = canTierUpAndOSREnter;
        m_graph.m_plan.willTryToTierUp = true;
        return true;
#else // ENABLE(FTL_JIT)
        RELEASE_ASSERT_NOT_REACHED();
        return false;
#endif // ENABLE(FTL_JIT)
    }
    bool run()
    {
        RELEASE_ASSERT(m_graph.m_plan.mode == DFGMode);

        if (!Options::useFTLJIT())
            return false;

        if (m_graph.m_profiledBlock->m_didFailFTLCompilation)
            return false;

#if ENABLE(FTL_JIT)
        FTL::CapabilityLevel level = FTL::canCompile(m_graph);
        if (level == FTL::CannotCompile)
            return false;

        if (!Options::enableOSREntryToFTL())
            level = FTL::CanCompile;

        InsertionSet insertionSet(m_graph);
        for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
            BasicBlock* block = m_graph.block(blockIndex);
            if (!block)
                continue;

            for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
                Node* node = block->at(nodeIndex);
                if (node->op() != LoopHint)
                    continue;

                // We only put OSR checks for the first LoopHint in the block. Note that
                // more than one LoopHint could happen in cases where we did a lot of CFG
                // simplification in the bytecode parser, but it should be very rare.

                NodeOrigin origin = node->origin;

                if (level != FTL::CanCompileAndOSREnter || origin.semantic.inlineCallFrame) {
                    insertionSet.insertNode(
                        nodeIndex + 1, SpecNone, CheckTierUpInLoop, origin);
                    break;
                }

                bool isAtTop = true;
                for (unsigned subNodeIndex = nodeIndex; subNodeIndex--;) {
                    if (!block->at(subNodeIndex)->isSemanticallySkippable()) {
                        isAtTop = false;
                        break;
                    }
                }

                if (!isAtTop) {
                    insertionSet.insertNode(
                        nodeIndex + 1, SpecNone, CheckTierUpInLoop, origin);
                    break;
                }

                insertionSet.insertNode(
                    nodeIndex + 1, SpecNone, CheckTierUpAndOSREnter, origin);
                break;
            }

            NodeAndIndex terminal = block->findTerminal();
            if (terminal.node->op() == Return) {
                insertionSet.insertNode(
                    terminal.index, SpecNone, CheckTierUpAtReturn, terminal.node->origin);
            }

            insertionSet.execute(block);
        }

        m_graph.m_plan.willTryToTierUp = true;
        return true;
#else // ENABLE(FTL_JIT)
        RELEASE_ASSERT_NOT_REACHED();
        return false;
#endif // ENABLE(FTL_JIT)
    }
    bool run()
    {
        RELEASE_ASSERT(m_graph.m_plan.mode == FTLForOSREntryMode);
        RELEASE_ASSERT(m_graph.m_form == ThreadedCPS);

        unsigned bytecodeIndex = m_graph.m_plan.osrEntryBytecodeIndex;
        RELEASE_ASSERT(bytecodeIndex);
        RELEASE_ASSERT(bytecodeIndex != UINT_MAX);

        // Needed by createPreHeader().
        m_graph.m_dominators.computeIfNecessary(m_graph);

        CodeBlock* baseline = m_graph.m_profiledBlock;

        BasicBlock* target = 0;
        for (unsigned blockIndex = m_graph.numBlocks(); blockIndex--;) {
            BasicBlock* block = m_graph.block(blockIndex);
            if (!block)
                continue;
            unsigned nodeIndex = 0;
            Node* firstNode = block->at(0);
            while (firstNode->isSemanticallySkippable())
                firstNode = block->at(++nodeIndex);
            if (firstNode->op() == LoopHint
                    && firstNode->origin.semantic == CodeOrigin(bytecodeIndex)) {
                target = block;
                break;
            }
        }

        if (!target) {
            // This is a terrible outcome. It shouldn't often happen but it might
            // happen and so we should defend against it. If it happens, then this
            // compilation is a failure.
            return false;
        }

        BlockInsertionSet insertionSet(m_graph);

        BasicBlock* newRoot = insertionSet.insert(0, QNaN);
        NodeOrigin origin = target->at(0)->origin;

        Vector<Node*> locals(baseline->m_numCalleeRegisters);
        for (int local = 0; local < baseline->m_numCalleeRegisters; ++local) {
            Node* previousHead = target->variablesAtHead.local(local);
            if (!previousHead)
                continue;
            VariableAccessData* variable = previousHead->variableAccessData();
            locals[local] = newRoot->appendNode(
                                m_graph, variable->prediction(), ExtractOSREntryLocal, origin,
                                OpInfo(variable->local().offset()));

            newRoot->appendNode(
                m_graph, SpecNone, MovHint, origin, OpInfo(variable->local().offset()),
                Edge(locals[local]));
        }

        for (int argument = 0; argument < baseline->numParameters(); ++argument) {
            Node* oldNode = target->variablesAtHead.argument(argument);
            if (!oldNode) {
                // Just for sanity, always have a SetArgument even if it's not needed.
                oldNode = m_graph.m_arguments[argument];
            }
            Node* node = newRoot->appendNode(
                             m_graph, SpecNone, SetArgument, origin,
                             OpInfo(oldNode->variableAccessData()));
            m_graph.m_arguments[argument] = node;
        }

        for (int local = 0; local < baseline->m_numCalleeRegisters; ++local) {
            Node* previousHead = target->variablesAtHead.local(local);
            if (!previousHead)
                continue;
            VariableAccessData* variable = previousHead->variableAccessData();
            Node* node = locals[local];
            newRoot->appendNode(
                m_graph, SpecNone, SetLocal, origin, OpInfo(variable), Edge(node));
        }

        newRoot->appendNode(
            m_graph, SpecNone, Jump, origin,
            OpInfo(createPreHeader(m_graph, insertionSet, target)));

        insertionSet.execute();
        m_graph.resetReachability();
        m_graph.killUnreachableBlocks();
        return true;
    }