UnlinkedCodeBlock::UnlinkedCodeBlock(VM* vm, Structure* structure, CodeType codeType, const ExecutableInfo& info) : Base(*vm, structure) , m_numVars(0) , m_numCalleeRegisters(0) , m_numParameters(0) , m_vm(vm) , m_argumentsRegister(VirtualRegister()) , m_globalObjectRegister(VirtualRegister()) , m_needsFullScopeChain(info.m_needsActivation) , m_usesEval(info.m_usesEval) , m_isNumericCompareFunction(false) , m_isStrictMode(info.m_isStrictMode) , m_isConstructor(info.m_isConstructor) , m_hasCapturedVariables(false) , m_isBuiltinFunction(info.m_isBuiltinFunction) , m_firstLine(0) , m_lineCount(0) , m_endColumn(UINT_MAX) , m_features(0) , m_codeType(codeType) , m_arrayProfileCount(0) , m_arrayAllocationProfileCount(0) , m_objectAllocationProfileCount(0) , m_valueProfileCount(0) , m_llintCallLinkInfoCount(0) #if ENABLE(BYTECODE_COMMENTS) , m_bytecodeCommentIterator(0) #endif { }
void LocalOSRAvailabilityCalculator::executeNode(Node* node) { switch (node->op()) { case PutLocal: { VariableAccessData* variable = node->variableAccessData(); m_availability.m_locals.operand(variable->local()).setFlush(variable->flushedAt()); break; } case KillLocal: { m_availability.m_locals.operand(node->unlinkedLocal()).setFlush(FlushedAt(ConflictingFlush)); break; } case GetLocal: { VariableAccessData* variable = node->variableAccessData(); m_availability.m_locals.operand(variable->local()) = Availability(node, variable->flushedAt()); break; } case MovHint: { m_availability.m_locals.operand(node->unlinkedLocal()).setNode(node->child1().node()); break; } case ZombieHint: { m_availability.m_locals.operand(node->unlinkedLocal()).setNodeUnavailable(); break; } case LoadVarargs: { LoadVarargsData* data = node->loadVarargsData(); m_availability.m_locals.operand(data->count) = Availability(FlushedAt(FlushedInt32, data->machineCount)); for (unsigned i = data->limit; i--;) { m_availability.m_locals.operand(VirtualRegister(data->start.offset() + i)) = Availability(FlushedAt(FlushedJSValue, VirtualRegister(data->machineStart.offset() + i))); } break; } default: break; } promoteHeapAccess( node, [&] (PromotedHeapLocation location, Edge value) { m_availability.m_heap.set(location, Availability(value.node())); }, [&] (PromotedHeapLocation) { }); }
FastBitVector BytecodeLivenessAnalysis::getLivenessInfoAtBytecodeOffset(unsigned bytecodeOffset) { FastBitVector temp; FastBitVector result; getLivenessInfoForNonCapturedVarsAtBytecodeOffset(bytecodeOffset, temp); unsigned numCapturedVars = numberOfCapturedVariables(m_codeBlock); if (numCapturedVars) { int firstCapturedLocal = VirtualRegister(captureStart(m_codeBlock)).toLocal(); result.resize(temp.numBits() + numCapturedVars); for (unsigned i = 0; i < numCapturedVars; ++i) result.set(firstCapturedLocal + i); } else result.resize(temp.numBits()); int tempLength = temp.numBits(); ASSERT(tempLength >= 0); for (int i = 0; i < tempLength; i++) { if (!temp.get(i)) continue; if (!numCapturedVars) { result.set(i); continue; } if (virtualRegisterForLocal(i).offset() > captureStart(m_codeBlock)) result.set(i); else result.set(numCapturedVars + i); } return result; }
void MethodOfGettingAValueProfile::reportValue(JSValue value) { switch (m_kind) { case None: return; case Ready: *u.profile->specFailBucket(0) = JSValue::encode(value); return; case LazyOperand: { LazyOperandValueProfileKey key(u.lazyOperand.bytecodeOffset, VirtualRegister(u.lazyOperand.operand)); ConcurrentJSLocker locker(u.lazyOperand.codeBlock->m_lock); LazyOperandValueProfile* profile = u.lazyOperand.codeBlock->lazyOperandValueProfiles(locker).add(locker, key); *profile->specFailBucket(0) = JSValue::encode(value); return; } case ArithProfileReady: { u.arithProfile->observeResult(value); return; } } RELEASE_ASSERT_NOT_REACHED(); }
void MethodOfGettingAValueProfile::emitReportValue(CCallHelpers& jit, JSValueRegs regs) const { switch (m_kind) { case None: return; case Ready: jit.storeValue(regs, u.profile->specFailBucket(0)); return; case LazyOperand: { LazyOperandValueProfileKey key(u.lazyOperand.bytecodeOffset, VirtualRegister(u.lazyOperand.operand)); ConcurrentJSLocker locker(u.lazyOperand.codeBlock->m_lock); LazyOperandValueProfile* profile = u.lazyOperand.codeBlock->lazyOperandValueProfiles(locker).add(locker, key); jit.storeValue(regs, profile->specFailBucket(0)); return; } case ArithProfileReady: { u.arithProfile->emitObserveResult(jit, regs, DoNotHaveTagRegisters); return; } } RELEASE_ASSERT_NOT_REACHED(); }
UnlinkedCodeBlock::UnlinkedCodeBlock(VM* vm, Structure* structure, CodeType codeType, const ExecutableInfo& info, DebuggerMode debuggerMode) : Base(*vm, structure) , m_numVars(0) , m_numCalleeLocals(0) , m_numParameters(0) , m_globalObjectRegister(VirtualRegister()) , m_usesEval(info.usesEval()) , m_isStrictMode(info.isStrictMode()) , m_isConstructor(info.isConstructor()) , m_hasCapturedVariables(false) , m_isBuiltinFunction(info.isBuiltinFunction()) , m_constructorKind(static_cast<unsigned>(info.constructorKind())) , m_superBinding(static_cast<unsigned>(info.superBinding())) , m_derivedContextType(static_cast<unsigned>(info.derivedContextType())) , m_evalContextType(static_cast<unsigned>(info.evalContextType())) , m_isArrowFunctionContext(info.isArrowFunctionContext()) , m_isClassContext(info.isClassContext()) , m_wasCompiledWithDebuggingOpcodes(debuggerMode == DebuggerMode::DebuggerOn || Options::forceDebuggerBytecodeGeneration()) , m_firstLine(0) , m_lineCount(0) , m_endColumn(UINT_MAX) , m_parseMode(info.parseMode()) , m_features(0) , m_codeType(codeType) , m_arrayProfileCount(0) , m_arrayAllocationProfileCount(0) , m_objectAllocationProfileCount(0) , m_valueProfileCount(0) , m_llintCallLinkInfoCount(0) { for (auto& constantRegisterIndex : m_linkTimeConstants) constantRegisterIndex = 0; ASSERT(m_constructorKind == static_cast<unsigned>(info.constructorKind())); }
static void stepOverInstruction(CodeBlock* codeBlock, BytecodeBasicBlock* block, Vector<std::unique_ptr<BytecodeBasicBlock>>& basicBlocks, unsigned bytecodeOffset, const UseFunctor& use, const DefFunctor& def) { // This abstractly execute the instruction in reverse. Instructions logically first use operands and // then define operands. This logical ordering is necessary for operations that use and def the same // operand, like: // // op_add loc1, loc1, loc2 // // The use of loc1 happens before the def of loc1. That's a semantic requirement since the add // operation cannot travel forward in time to read the value that it will produce after reading that // value. Since we are executing in reverse, this means that we must do defs before uses (reverse of // uses before defs). // // Since this is a liveness analysis, this ordering ends up being particularly important: if we did // uses before defs, then the add operation above would appear to not have loc1 live, since we'd // first add it to the out set (the use), and then we'd remove it (the def). computeDefsForBytecodeOffset( codeBlock, block, bytecodeOffset, [&] (CodeBlock* codeBlock, Instruction*, OpcodeID, int operand) { if (isValidRegisterForLiveness(codeBlock, operand)) def(VirtualRegister(operand).toLocal()); }); computeUsesForBytecodeOffset( codeBlock, block, bytecodeOffset, [&] (CodeBlock* codeBlock, Instruction*, OpcodeID, int operand) { if (isValidRegisterForLiveness(codeBlock, operand)) use(VirtualRegister(operand).toLocal()); }); // If we have an exception handler, we want the live-in variables of the // exception handler block to be included in the live-in of this particular bytecode. if (HandlerInfo* handler = codeBlock->handlerForBytecodeOffset(bytecodeOffset)) { // FIXME: This resume check should not be needed. // https://bugs.webkit.org/show_bug.cgi?id=159281 Interpreter* interpreter = codeBlock->vm()->interpreter; Instruction* instructionsBegin = codeBlock->instructions().begin(); Instruction* instruction = &instructionsBegin[bytecodeOffset]; OpcodeID opcodeID = interpreter->getOpcodeID(instruction->u.opcode); if (opcodeID != op_resume) { BytecodeBasicBlock* handlerBlock = findBasicBlockWithLeaderOffset(basicBlocks, handler->target); ASSERT(handlerBlock); handlerBlock->in().forEachSetBit(use); } } }
void OSREntryData::dumpInContext(PrintStream& out, DumpContext* context) const { out.print("bc#", m_bytecodeIndex, ", machine code offset = ", m_machineCodeOffset); out.print(", stack rules = ["); auto printOperand = [&] (VirtualRegister reg) { out.print(inContext(m_expectedValues.operand(reg), context), " ("); VirtualRegister toReg; bool overwritten = false; for (OSREntryReshuffling reshuffling : m_reshufflings) { if (reg == VirtualRegister(reshuffling.fromOffset)) { toReg = VirtualRegister(reshuffling.toOffset); break; } if (reg == VirtualRegister(reshuffling.toOffset)) overwritten = true; } if (!overwritten && !toReg.isValid()) toReg = reg; if (toReg.isValid()) { if (toReg.isLocal() && !m_machineStackUsed.get(toReg.toLocal())) out.print("ignored"); else out.print("maps to ", toReg); } else out.print("overwritten"); if (reg.isLocal() && m_localsForcedDouble.get(reg.toLocal())) out.print(", forced double"); if (reg.isLocal() && m_localsForcedAnyInt.get(reg.toLocal())) out.print(", forced machine int"); out.print(")"); }; CommaPrinter comma; for (size_t argumentIndex = m_expectedValues.numberOfArguments(); argumentIndex--;) { out.print(comma, "arg", argumentIndex, ":"); printOperand(virtualRegisterForArgument(argumentIndex)); } for (size_t localIndex = 0; localIndex < m_expectedValues.numberOfLocals(); ++localIndex) { out.print(comma, "loc", localIndex, ":"); printOperand(virtualRegisterForLocal(localIndex)); } out.print("], machine stack used = ", m_machineStackUsed); }
bool argumentsInvolveStackSlot(InlineCallFrame* inlineCallFrame, VirtualRegister reg) { if (!inlineCallFrame) return (reg.isArgument() && reg.toArgument()) || reg.isHeader(); if (inlineCallFrame->isClosureCall && reg == VirtualRegister(inlineCallFrame->stackOffset + CallFrameSlot::callee)) return true; if (inlineCallFrame->isVarargs() && reg == VirtualRegister(inlineCallFrame->stackOffset + CallFrameSlot::argumentCount)) return true; unsigned numArguments = inlineCallFrame->arguments.size() - 1; VirtualRegister argumentStart = VirtualRegister(inlineCallFrame->stackOffset) + CallFrame::argumentOffset(0); return reg >= argumentStart && reg < argumentStart + numArguments; }
void JIT_OPERATION operationLoadVarargs(ExecState* exec, int32_t firstElementDest, EncodedJSValue encodedArguments, int32_t offset, int32_t length, int32_t mandatoryMinimum) { VM& vm = exec->vm(); NativeCallFrameTracer tracer(&vm, exec); JSValue arguments = JSValue::decode(encodedArguments); loadVarargs(exec, VirtualRegister(firstElementDest), arguments, offset, length); for (int32_t i = length; i < mandatoryMinimum; ++i) exec->r(firstElementDest + i) = jsUndefined(); }
bool BytecodeLivenessAnalysis::operandIsLiveAtBytecodeOffset(int operand, unsigned bytecodeOffset) { int numCapturedVars = numberOfCapturedVariables(m_codeBlock); if (VirtualRegister(operand).isArgument()) return true; if (operand <= captureStart(m_codeBlock) && operand > captureEnd(m_codeBlock)) return true; FastBitVector result; getLivenessInfoForNonCapturedVarsAtBytecodeOffset(bytecodeOffset, result); return result.get(operand - numCapturedVars); }
VirtualRegister assign(const Vector<unsigned>& allocation, VirtualRegister src) { VirtualRegister result = src; if (result.isLocal()) { unsigned myAllocation = allocation[result.toLocal()]; if (myAllocation == UINT_MAX) result = VirtualRegister(); else result = virtualRegisterForLocal(myAllocation); } return result; }
void JITCode::reconstruct( ExecState* exec, CodeBlock* codeBlock, CodeOrigin codeOrigin, unsigned streamIndex, Operands<JSValue>& result) { Operands<ValueRecovery> recoveries; reconstruct(codeBlock, codeOrigin, streamIndex, recoveries); result = Operands<JSValue>(OperandsLike, recoveries); for (size_t i = result.size(); i--;) { int operand = result.operandForIndex(i); if (operandIsArgument(operand) && !VirtualRegister(operand).toArgument() && codeBlock->codeType() == FunctionCode && codeBlock->specializationKind() == CodeForConstruct) { // Ugh. If we're in a constructor, the 'this' argument may hold garbage. It will // also never be used. It doesn't matter what we put into the value for this, // but it has to be an actual value that can be grokked by subsequent DFG passes, // so we sanitize it here by turning it into Undefined. result[i] = jsUndefined(); continue; } ValueRecovery recovery = recoveries[i]; JSValue value; switch (recovery.technique()) { case AlreadyInJSStack: case AlreadyInJSStackAsUnboxedCell: case AlreadyInJSStackAsUnboxedBoolean: value = exec->r(operand).jsValue(); break; case AlreadyInJSStackAsUnboxedInt32: value = jsNumber(exec->r(operand).unboxedInt32()); break; case AlreadyInJSStackAsUnboxedInt52: value = jsNumber(exec->r(operand).unboxedInt52()); break; case AlreadyInJSStackAsUnboxedDouble: value = jsDoubleNumber(exec->r(operand).unboxedDouble()); break; case Constant: value = recovery.constant(); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } result[i] = value; } }
void emitSetupVarargsFrameFastCase(CCallHelpers& jit, GPRReg numUsedSlotsGPR, GPRReg scratchGPR1, GPRReg scratchGPR2, GPRReg scratchGPR3, InlineCallFrame* inlineCallFrame, unsigned firstVarArgOffset, CCallHelpers::JumpList& slowCase) { ValueRecovery argumentCountRecovery; VirtualRegister firstArgumentReg; if (inlineCallFrame) { if (inlineCallFrame->isVarargs()) { argumentCountRecovery = ValueRecovery::displacedInJSStack( inlineCallFrame->argumentCountRegister, DataFormatInt32); } else { argumentCountRecovery = ValueRecovery::constant( jsNumber(inlineCallFrame->arguments.size())); } if (inlineCallFrame->arguments.size() > 1) firstArgumentReg = inlineCallFrame->arguments[1].virtualRegister(); else firstArgumentReg = VirtualRegister(0); } else { argumentCountRecovery = ValueRecovery::displacedInJSStack( VirtualRegister(JSStack::ArgumentCount), DataFormatInt32); firstArgumentReg = VirtualRegister(CallFrame::argumentOffset(0)); } emitSetupVarargsFrameFastCase(jit, numUsedSlotsGPR, scratchGPR1, scratchGPR2, scratchGPR3, argumentCountRecovery, firstArgumentReg, firstVarArgOffset, slowCase); }
void InlineCallFrame::dumpInContext(PrintStream& out, DumpContext* context) const { out.print(briefFunctionInformation(), ":<", RawPointer(executable.get())); if (executable->isStrictMode()) out.print(" (StrictMode)"); out.print(", bc#", caller.bytecodeIndex, ", ", specializationKind()); if (isClosureCall) out.print(", closure call"); else out.print(", known callee: ", inContext(calleeRecovery.constant(), context)); out.print(", numArgs+this = ", arguments.size()); out.print(", stack < loc", VirtualRegister(stackOffset).toLocal()); out.print(">"); }
ValueRecovery ValueRep::recoveryForJSValue() const { switch (kind()) { case Register: return ValueRecovery::inGPR(gpr(), DataFormatJS); case Stack: RELEASE_ASSERT(!(offsetFromFP() % sizeof(EncodedJSValue))); return ValueRecovery::displacedInJSStack( VirtualRegister(offsetFromFP() / sizeof(EncodedJSValue)), DataFormatJS); case Constant: return ValueRecovery::constant(JSValue::decode(value())); default: RELEASE_ASSERT_NOT_REACHED(); return { }; } }
bool run() { RELEASE_ASSERT(m_graph.m_form == ThreadedCPS); m_graph.clearReplacements(); m_graph.m_dominators.computeIfNecessary(m_graph); if (verbose) { dataLog("Graph before SSA transformation:\n"); m_graph.dump(); } // Create a SSACalculator::Variable for every root VariableAccessData. for (VariableAccessData& variable : m_graph.m_variableAccessData) { if (!variable.isRoot() || variable.isCaptured()) continue; SSACalculator::Variable* ssaVariable = m_calculator.newVariable(); ASSERT(ssaVariable->index() == m_variableForSSAIndex.size()); m_variableForSSAIndex.append(&variable); m_ssaVariableForVariable.add(&variable, ssaVariable); } // Find all SetLocals and create Defs for them. We handle SetArgument by creating a // GetLocal, and recording the flush format. for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; // Must process the block in forward direction because we want to see the last // assignment for every local. for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) { Node* node = block->at(nodeIndex); if (node->op() != SetLocal && node->op() != SetArgument) continue; VariableAccessData* variable = node->variableAccessData(); if (variable->isCaptured()) continue; Node* childNode; if (node->op() == SetLocal) childNode = node->child1().node(); else { ASSERT(node->op() == SetArgument); childNode = m_insertionSet.insertNode( nodeIndex, node->variableAccessData()->prediction(), GetStack, node->origin, OpInfo(m_graph.m_stackAccessData.add(variable->local(), variable->flushFormat()))); if (!ASSERT_DISABLED) m_argumentGetters.add(childNode); m_argumentMapping.add(node, childNode); } m_calculator.newDef( m_ssaVariableForVariable.get(variable), block, childNode); } m_insertionSet.execute(block); } // Decide where Phis are to be inserted. This creates the Phi's but doesn't insert them // yet. We will later know where to insert them because SSACalculator is such a bro. m_calculator.computePhis( [&] (SSACalculator::Variable* ssaVariable, BasicBlock* block) -> Node* { VariableAccessData* variable = m_variableForSSAIndex[ssaVariable->index()]; // Prune by liveness. This doesn't buy us much other than compile times. Node* headNode = block->variablesAtHead.operand(variable->local()); if (!headNode) return nullptr; // There is the possibiltiy of "rebirths". The SSA calculator will already prune // rebirths for the same VariableAccessData. But it will not be able to prune // rebirths that arose from the same local variable number but a different // VariableAccessData. We do that pruning here. // // Here's an example of a rebirth that this would catch: // // var x; // if (foo) { // if (bar) { // x = 42; // } else { // x = 43; // } // print(x); // x = 44; // } else { // x = 45; // } // print(x); // Without this check, we'd have a Phi for x = 42|43 here. // // FIXME: Consider feeding local variable numbers, not VariableAccessData*'s, as // the "variables" for SSACalculator. That would allow us to eliminate this // special case. // https://bugs.webkit.org/show_bug.cgi?id=136641 if (headNode->variableAccessData() != variable) return nullptr; Node* phiNode = m_graph.addNode( variable->prediction(), Phi, NodeOrigin()); FlushFormat format = variable->flushFormat(); NodeFlags result = resultFor(format); phiNode->mergeFlags(result); return phiNode; }); if (verbose) { dataLog("Computed Phis, about to transform the graph.\n"); dataLog("\n"); dataLog("Graph:\n"); m_graph.dump(); dataLog("\n"); dataLog("Mappings:\n"); for (unsigned i = 0; i < m_variableForSSAIndex.size(); ++i) dataLog(" ", i, ": ", VariableAccessDataDump(m_graph, m_variableForSSAIndex[i]), "\n"); dataLog("\n"); dataLog("SSA calculator: ", m_calculator, "\n"); } // Do the bulk of the SSA conversion. For each block, this tracks the operand->Node // mapping based on a combination of what the SSACalculator tells us, and us walking over // the block in forward order. We use our own data structure, valueForOperand, for // determining the local mapping, but we rely on SSACalculator for the non-local mapping. // // This does three things at once: // // - Inserts the Phis in all of the places where they need to go. We've already created // them and they are accounted for in the SSACalculator's data structures, but we // haven't inserted them yet, mostly because we want to insert all of a block's Phis in // one go to amortize the cost of node insertion. // // - Create and insert Upsilons. // // - Convert all of the preexisting SSA nodes (other than the old CPS Phi nodes) into SSA // form by replacing as follows: // // - MovHint has KillLocal prepended to it. // // - GetLocal over captured variables lose their phis and become GetStack. // // - GetLocal over uncaptured variables die and get replaced with references to the node // specified by valueForOperand. // // - SetLocal turns into PutStack if it's flushed, or turns into a Check otherwise. // // - Flush loses its children and turns into a Phantom. // // - PhantomLocal becomes Phantom, and its child is whatever is specified by // valueForOperand. // // - SetArgument is removed. Note that GetStack nodes have already been inserted. Operands<Node*> valueForOperand(OperandsLike, m_graph.block(0)->variablesAtHead); for (BasicBlock* block : m_graph.blocksInPreOrder()) { valueForOperand.clear(); // CPS will claim that the root block has all arguments live. But we have already done // the first step of SSA conversion: argument locals are no longer live at head; // instead we have GetStack nodes for extracting the values of arguments. So, we // skip the at-head available value calculation for the root block. if (block != m_graph.block(0)) { for (size_t i = valueForOperand.size(); i--;) { Node* nodeAtHead = block->variablesAtHead[i]; if (!nodeAtHead) continue; VariableAccessData* variable = nodeAtHead->variableAccessData(); if (variable->isCaptured()) continue; if (verbose) dataLog("Considering live variable ", VariableAccessDataDump(m_graph, variable), " at head of block ", *block, "\n"); SSACalculator::Variable* ssaVariable = m_ssaVariableForVariable.get(variable); SSACalculator::Def* def = m_calculator.reachingDefAtHead(block, ssaVariable); if (!def) { // If we are required to insert a Phi, then we won't have a reaching def // at head. continue; } Node* node = def->value(); if (node->replacement) { // This will occur when a SetLocal had a GetLocal as its source. The // GetLocal would get replaced with an actual SSA value by the time we get // here. Note that the SSA value with which the GetLocal got replaced // would not in turn have a replacement. node = node->replacement; ASSERT(!node->replacement); } if (verbose) dataLog("Mapping: ", VirtualRegister(valueForOperand.operandForIndex(i)), " -> ", node, "\n"); valueForOperand[i] = node; } } // Insert Phis by asking the calculator what phis there are in this block. Also update // valueForOperand with those Phis. For Phis associated with variables that are not // flushed, we also insert a MovHint. size_t phiInsertionPoint = 0; for (SSACalculator::Def* phiDef : m_calculator.phisForBlock(block)) { VariableAccessData* variable = m_variableForSSAIndex[phiDef->variable()->index()]; m_insertionSet.insert(phiInsertionPoint, phiDef->value()); valueForOperand.operand(variable->local()) = phiDef->value(); m_insertionSet.insertNode( phiInsertionPoint, SpecNone, MovHint, NodeOrigin(), OpInfo(variable->local().offset()), phiDef->value()->defaultEdge()); } for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) { Node* node = block->at(nodeIndex); if (verbose) { dataLog("Processing node ", node, ":\n"); m_graph.dump(WTF::dataFile(), " ", node); } m_graph.performSubstitution(node); switch (node->op()) { case MovHint: { m_insertionSet.insertNode( nodeIndex, SpecNone, KillStack, node->origin, OpInfo(node->unlinkedLocal().offset())); break; } case SetLocal: { VariableAccessData* variable = node->variableAccessData(); if (variable->isCaptured() || !!(node->flags() & NodeIsFlushed)) { node->convertToPutStack( m_graph.m_stackAccessData.add( variable->local(), variable->flushFormat())); } else node->setOpAndDefaultFlags(Check); if (!variable->isCaptured()) { if (verbose) dataLog("Mapping: ", variable->local(), " -> ", node->child1().node(), "\n"); valueForOperand.operand(variable->local()) = node->child1().node(); } break; } case GetStack: { ASSERT(m_argumentGetters.contains(node)); valueForOperand.operand(node->stackAccessData()->local) = node; break; } case GetLocal: { VariableAccessData* variable = node->variableAccessData(); node->children.reset(); if (variable->isCaptured()) { node->convertToGetStack(m_graph.m_stackAccessData.add(variable->local(), variable->flushFormat())); break; } node->convertToPhantom(); if (verbose) dataLog("Replacing node ", node, " with ", valueForOperand.operand(variable->local()), "\n"); node->replacement = valueForOperand.operand(variable->local()); break; } case Flush: { node->children.reset(); node->convertToPhantom(); break; } case PhantomLocal: { ASSERT(node->child1().useKind() == UntypedUse); VariableAccessData* variable = node->variableAccessData(); if (variable->isCaptured()) { // This is a fun case. We could have a captured variable that had some // or all of its uses strength reduced to phantoms rather than flushes. // SSA conversion will currently still treat it as flushed, in the sense // that it will just keep the SetLocal. Therefore, there is nothing that // needs to be done here: we don't need to also keep the source value // alive. And even if we did want to keep the source value alive, we // wouldn't be able to, because the variablesAtHead value for a captured // local wouldn't have been computed by the Phi reduction algorithm // above. node->children.reset(); } else node->child1() = valueForOperand.operand(variable->local())->defaultEdge(); node->convertToPhantom(); break; } case SetArgument: { node->convertToPhantom(); break; } default: break; } } // We want to insert Upsilons just before the end of the block. On the surface this // seems dangerous because the Upsilon will have a checking UseKind. But, we will not // actually be performing the check at the point of the Upsilon; the check will // already have been performed at the point where the original SetLocal was. size_t upsilonInsertionPoint = block->size() - 1; NodeOrigin upsilonOrigin = block->last()->origin; for (unsigned successorIndex = block->numSuccessors(); successorIndex--;) { BasicBlock* successorBlock = block->successor(successorIndex); for (SSACalculator::Def* phiDef : m_calculator.phisForBlock(successorBlock)) { Node* phiNode = phiDef->value(); SSACalculator::Variable* ssaVariable = phiDef->variable(); VariableAccessData* variable = m_variableForSSAIndex[ssaVariable->index()]; FlushFormat format = variable->flushFormat(); UseKind useKind = useKindFor(format); m_insertionSet.insertNode( upsilonInsertionPoint, SpecNone, Upsilon, upsilonOrigin, OpInfo(phiNode), Edge( valueForOperand.operand(variable->local()), useKind)); } } m_insertionSet.execute(block); } // Free all CPS phis and reset variables vectors. for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; for (unsigned phiIndex = block->phis.size(); phiIndex--;) m_graph.m_allocator.free(block->phis[phiIndex]); block->phis.clear(); block->variablesAtHead.clear(); block->variablesAtTail.clear(); block->valuesAtHead.clear(); block->valuesAtHead.clear(); block->ssa = std::make_unique<BasicBlock::SSAData>(block); } m_graph.m_argumentFormats.resize(m_graph.m_arguments.size()); for (unsigned i = m_graph.m_arguments.size(); i--;) { FlushFormat format = FlushedJSValue; Node* node = m_argumentMapping.get(m_graph.m_arguments[i]); // m_argumentMapping.get could return null for a captured local. That's fine. We only // track the argument loads of those arguments for which we speculate type. We don't // speculate type for captured arguments. if (node) format = node->stackAccessData()->format; m_graph.m_argumentFormats[i] = format; m_graph.m_arguments[i] = node; // Record the load that loads the arguments for the benefit of exit profiling. } m_graph.m_form = SSA; if (verbose) { dataLog("Graph after SSA transformation:\n"); m_graph.dump(); } return true; }
void JIT::compileLoadVarargs(Instruction* instruction) { int thisValue = instruction[3].u.operand; int arguments = instruction[4].u.operand; int firstFreeRegister = instruction[5].u.operand; JumpList slowCase; JumpList end; bool canOptimize = m_codeBlock->usesArguments() && VirtualRegister(arguments) == m_codeBlock->argumentsRegister() && !m_codeBlock->symbolTable()->slowArguments(); if (canOptimize) { emitLoadTag(arguments, regT1); slowCase.append(branch32(NotEqual, regT1, TrustedImm32(JSValue::EmptyValueTag))); load32(payloadFor(JSStack::ArgumentCount), regT2); slowCase.append(branch32(Above, regT2, TrustedImm32(Arguments::MaxArguments + 1))); // regT2: argumentCountIncludingThis move(regT2, regT3); neg32(regT3); add32(TrustedImm32(firstFreeRegister - JSStack::CallFrameHeaderSize), regT3); lshift32(TrustedImm32(3), regT3); addPtr(callFrameRegister, regT3); // regT3: newCallFrame slowCase.append(branchPtr(Above, AbsoluteAddress(m_vm->interpreter->stack().addressOfEnd()), regT3)); // Initialize ArgumentCount. store32(regT2, payloadFor(JSStack::ArgumentCount, regT3)); // Initialize 'this'. emitLoad(thisValue, regT1, regT0); store32(regT0, Address(regT3, OBJECT_OFFSETOF(JSValue, u.asBits.payload) + (CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))))); store32(regT1, Address(regT3, OBJECT_OFFSETOF(JSValue, u.asBits.tag) + (CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))))); // Copy arguments. end.append(branchSub32(Zero, TrustedImm32(1), regT2)); // regT2: argumentCount; Label copyLoop = label(); load32(BaseIndex(callFrameRegister, regT2, TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload) +(CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))), regT0); load32(BaseIndex(callFrameRegister, regT2, TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag) +(CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))), regT1); store32(regT0, BaseIndex(regT3, regT2, TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload) +(CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))))); store32(regT1, BaseIndex(regT3, regT2, TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag) +(CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))))); branchSub32(NonZero, TrustedImm32(1), regT2).linkTo(copyLoop, this); end.append(jump()); } if (canOptimize) slowCase.link(this); emitLoad(thisValue, regT1, regT0); emitLoad(arguments, regT3, regT2); callOperation(operationLoadVarargs, regT1, regT0, regT3, regT2, firstFreeRegister); move(returnValueRegister, regT3); if (canOptimize) end.link(this); }
SUPPRESS_ASAN void* prepareOSREntry(ExecState* exec, CodeBlock* codeBlock, unsigned bytecodeIndex) { ASSERT(JITCode::isOptimizingJIT(codeBlock->jitType())); ASSERT(codeBlock->alternative()); ASSERT(codeBlock->alternative()->jitType() == JITCode::BaselineJIT); ASSERT(!codeBlock->jitCodeMap()); if (!Options::useOSREntryToDFG()) return 0; if (Options::verboseOSR()) { dataLog( "DFG OSR in ", *codeBlock->alternative(), " -> ", *codeBlock, " from bc#", bytecodeIndex, "\n"); } VM* vm = &exec->vm(); sanitizeStackForVM(vm); if (bytecodeIndex) codeBlock->ownerScriptExecutable()->setDidTryToEnterInLoop(true); if (codeBlock->jitType() != JITCode::DFGJIT) { RELEASE_ASSERT(codeBlock->jitType() == JITCode::FTLJIT); // When will this happen? We could have: // // - An exit from the FTL JIT into the baseline JIT followed by an attempt // to reenter. We're fine with allowing this to fail. If it happens // enough we'll just reoptimize. It basically means that the OSR exit cost // us dearly and so reoptimizing is the right thing to do. // // - We have recursive code with hot loops. Consider that foo has a hot loop // that calls itself. We have two foo's on the stack, lets call them foo1 // and foo2, with foo1 having called foo2 from foo's hot loop. foo2 gets // optimized all the way into the FTL. Then it returns into foo1, and then // foo1 wants to get optimized. It might reach this conclusion from its // hot loop and attempt to OSR enter. And we'll tell it that it can't. It // might be worth addressing this case, but I just think this case will // be super rare. For now, if it does happen, it'll cause some compilation // thrashing. if (Options::verboseOSR()) dataLog(" OSR failed because the target code block is not DFG.\n"); return 0; } JITCode* jitCode = codeBlock->jitCode()->dfg(); OSREntryData* entry = jitCode->osrEntryDataForBytecodeIndex(bytecodeIndex); if (!entry) { if (Options::verboseOSR()) dataLogF(" OSR failed because the entrypoint was optimized out.\n"); return 0; } ASSERT(entry->m_bytecodeIndex == bytecodeIndex); // The code below checks if it is safe to perform OSR entry. It may find // that it is unsafe to do so, for any number of reasons, which are documented // below. If the code decides not to OSR then it returns 0, and it's the caller's // responsibility to patch up the state in such a way as to ensure that it's // both safe and efficient to continue executing baseline code for now. This // should almost certainly include calling either codeBlock->optimizeAfterWarmUp() // or codeBlock->dontOptimizeAnytimeSoon(). // 1) Verify predictions. If the predictions are inconsistent with the actual // values, then OSR entry is not possible at this time. It's tempting to // assume that we could somehow avoid this case. We can certainly avoid it // for first-time loop OSR - that is, OSR into a CodeBlock that we have just // compiled. Then we are almost guaranteed that all of the predictions will // check out. It would be pretty easy to make that a hard guarantee. But // then there would still be the case where two call frames with the same // baseline CodeBlock are on the stack at the same time. The top one // triggers compilation and OSR. In that case, we may no longer have // accurate value profiles for the one deeper in the stack. Hence, when we // pop into the CodeBlock that is deeper on the stack, we might OSR and // realize that the predictions are wrong. Probably, in most cases, this is // just an anomaly in the sense that the older CodeBlock simply went off // into a less-likely path. So, the wisest course of action is to simply not // OSR at this time. for (size_t argument = 0; argument < entry->m_expectedValues.numberOfArguments(); ++argument) { if (argument >= exec->argumentCountIncludingThis()) { if (Options::verboseOSR()) { dataLogF(" OSR failed because argument %zu was not passed, expected ", argument); entry->m_expectedValues.argument(argument).dump(WTF::dataFile()); dataLogF(".\n"); } return 0; } JSValue value; if (!argument) value = exec->thisValue(); else value = exec->argument(argument - 1); if (!entry->m_expectedValues.argument(argument).validate(value)) { if (Options::verboseOSR()) { dataLog( " OSR failed because argument ", argument, " is ", value, ", expected ", entry->m_expectedValues.argument(argument), ".\n"); } return 0; } } for (size_t local = 0; local < entry->m_expectedValues.numberOfLocals(); ++local) { int localOffset = virtualRegisterForLocal(local).offset(); if (entry->m_localsForcedDouble.get(local)) { if (!exec->registers()[localOffset].asanUnsafeJSValue().isNumber()) { if (Options::verboseOSR()) { dataLog( " OSR failed because variable ", localOffset, " is ", exec->registers()[localOffset].asanUnsafeJSValue(), ", expected number.\n"); } return 0; } continue; } if (entry->m_localsForcedAnyInt.get(local)) { if (!exec->registers()[localOffset].asanUnsafeJSValue().isAnyInt()) { if (Options::verboseOSR()) { dataLog( " OSR failed because variable ", localOffset, " is ", exec->registers()[localOffset].asanUnsafeJSValue(), ", expected ", "machine int.\n"); } return 0; } continue; } if (!entry->m_expectedValues.local(local).validate(exec->registers()[localOffset].asanUnsafeJSValue())) { if (Options::verboseOSR()) { dataLog( " OSR failed because variable ", localOffset, " is ", exec->registers()[localOffset].asanUnsafeJSValue(), ", expected ", entry->m_expectedValues.local(local), ".\n"); } return 0; } } // 2) Check the stack height. The DFG JIT may require a taller stack than the // baseline JIT, in some cases. If we can't grow the stack, then don't do // OSR right now. That's the only option we have unless we want basic block // boundaries to start throwing RangeErrors. Although that would be possible, // it seems silly: you'd be diverting the program to error handling when it // would have otherwise just kept running albeit less quickly. unsigned frameSizeForCheck = jitCode->common.requiredRegisterCountForExecutionAndExit(); if (!vm->interpreter->stack().ensureCapacityFor(&exec->registers()[virtualRegisterForLocal(frameSizeForCheck - 1).offset()])) { if (Options::verboseOSR()) dataLogF(" OSR failed because stack growth failed.\n"); return 0; } if (Options::verboseOSR()) dataLogF(" OSR should succeed.\n"); // At this point we're committed to entering. We will do some work to set things up, // but we also rely on our caller recognizing that when we return a non-null pointer, // that means that we're already past the point of no return and we must succeed at // entering. // 3) Set up the data in the scratch buffer and perform data format conversions. unsigned frameSize = jitCode->common.frameRegisterCount; unsigned baselineFrameSize = entry->m_expectedValues.numberOfLocals(); unsigned maxFrameSize = std::max(frameSize, baselineFrameSize); Register* scratch = bitwise_cast<Register*>(vm->scratchBufferForSize(sizeof(Register) * (2 + JSStack::CallFrameHeaderSize + maxFrameSize))->dataBuffer()); *bitwise_cast<size_t*>(scratch + 0) = frameSize; void* targetPC = codeBlock->jitCode()->executableAddressAtOffset(entry->m_machineCodeOffset); if (Options::verboseOSR()) dataLogF(" OSR using target PC %p.\n", targetPC); RELEASE_ASSERT(targetPC); *bitwise_cast<void**>(scratch + 1) = targetPC; Register* pivot = scratch + 2 + JSStack::CallFrameHeaderSize; for (int index = -JSStack::CallFrameHeaderSize; index < static_cast<int>(baselineFrameSize); ++index) { VirtualRegister reg(-1 - index); if (reg.isLocal()) { if (entry->m_localsForcedDouble.get(reg.toLocal())) { *bitwise_cast<double*>(pivot + index) = exec->registers()[reg.offset()].asanUnsafeJSValue().asNumber(); continue; } if (entry->m_localsForcedAnyInt.get(reg.toLocal())) { *bitwise_cast<int64_t*>(pivot + index) = exec->registers()[reg.offset()].asanUnsafeJSValue().asAnyInt() << JSValue::int52ShiftAmount; continue; } } pivot[index] = exec->registers()[reg.offset()].asanUnsafeJSValue(); } // 4) Reshuffle those registers that need reshuffling. Vector<JSValue> temporaryLocals(entry->m_reshufflings.size()); for (unsigned i = entry->m_reshufflings.size(); i--;) temporaryLocals[i] = pivot[VirtualRegister(entry->m_reshufflings[i].fromOffset).toLocal()].asanUnsafeJSValue(); for (unsigned i = entry->m_reshufflings.size(); i--;) pivot[VirtualRegister(entry->m_reshufflings[i].toOffset).toLocal()] = temporaryLocals[i]; // 5) Clear those parts of the call frame that the DFG ain't using. This helps GC on // some programs by eliminating some stale pointer pathologies. for (unsigned i = frameSize; i--;) { if (entry->m_machineStackUsed.get(i)) continue; pivot[i] = JSValue(); } // 6) Copy our callee saves to buffer. #if NUMBER_OF_CALLEE_SAVES_REGISTERS > 0 RegisterAtOffsetList* registerSaveLocations = codeBlock->calleeSaveRegisters(); RegisterAtOffsetList* allCalleeSaves = vm->getAllCalleeSaveRegisterOffsets(); RegisterSet dontSaveRegisters = RegisterSet(RegisterSet::stackRegisters(), RegisterSet::allFPRs()); unsigned registerCount = registerSaveLocations->size(); VMEntryRecord* record = vmEntryRecord(vm->topVMEntryFrame); for (unsigned i = 0; i < registerCount; i++) { RegisterAtOffset currentEntry = registerSaveLocations->at(i); if (dontSaveRegisters.get(currentEntry.reg())) continue; RegisterAtOffset* calleeSavesEntry = allCalleeSaves->find(currentEntry.reg()); *(bitwise_cast<intptr_t*>(pivot - 1) - currentEntry.offsetAsIndex()) = record->calleeSaveRegistersBuffer[calleeSavesEntry->offsetAsIndex()]; } #endif // 7) Fix the call frame to have the right code block. *bitwise_cast<CodeBlock**>(pivot - 1 - JSStack::CodeBlock) = codeBlock; if (Options::verboseOSR()) dataLogF(" OSR returning data buffer %p.\n", scratch); return scratch; }
static void computeUsesForBytecodeOffset(CodeBlock* codeBlock, unsigned bytecodeOffset, FastBitVector& uses) { Interpreter* interpreter = codeBlock->vm()->interpreter; Instruction* instructionsBegin = codeBlock->instructions().begin(); Instruction* instruction = &instructionsBegin[bytecodeOffset]; OpcodeID opcodeID = interpreter->getOpcodeID(instruction->u.opcode); switch (opcodeID) { // No uses. case op_new_regexp: case op_new_array_buffer: case op_throw_static_error: case op_debug: case op_resolve_scope: case op_pop_scope: case op_jneq_ptr: case op_new_func_exp: case op_loop_hint: case op_jmp: case op_new_object: case op_init_lazy_reg: case op_get_callee: case op_enter: case op_catch: return; // First argument. case op_new_func: case op_create_activation: case op_create_arguments: case op_to_this: case op_tear_off_activation: case op_profile_will_call: case op_profile_did_call: case op_throw: case op_push_with_scope: case op_end: case op_ret: case op_jtrue: case op_jfalse: case op_jeq_null: case op_jneq_null: case op_dec: case op_inc: { if (isValidRegisterForLiveness(codeBlock, instruction[1].u.operand)) setForOperand(codeBlock, uses, instruction[1].u.operand); return; } // First and second arguments. case op_del_by_id: case op_ret_object_or_this: case op_jlesseq: case op_jgreater: case op_jgreatereq: case op_jnless: case op_jnlesseq: case op_jngreater: case op_jngreatereq: case op_jless: { if (isValidRegisterForLiveness(codeBlock, instruction[1].u.operand)) setForOperand(codeBlock, uses, instruction[1].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[2].u.operand)) setForOperand(codeBlock, uses, instruction[2].u.operand); return; } // First, second, and third arguments. case op_del_by_val: case op_put_by_val_direct: case op_put_by_val: { if (isValidRegisterForLiveness(codeBlock, instruction[1].u.operand)) setForOperand(codeBlock, uses, instruction[1].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[2].u.operand)) setForOperand(codeBlock, uses, instruction[2].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[3].u.operand)) setForOperand(codeBlock, uses, instruction[3].u.operand); return; } // First and third arguments. case op_put_by_index: case op_put_by_id_replace: case op_put_by_id_transition: case op_put_by_id_transition_direct: case op_put_by_id_transition_direct_out_of_line: case op_put_by_id_transition_normal: case op_put_by_id_transition_normal_out_of_line: case op_put_by_id_generic: case op_put_by_id_out_of_line: case op_put_by_id: case op_put_to_scope: { if (isValidRegisterForLiveness(codeBlock, instruction[1].u.operand)) setForOperand(codeBlock, uses, instruction[1].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[3].u.operand)) setForOperand(codeBlock, uses, instruction[3].u.operand); return; } // First, third, and fourth arguments. case op_put_getter_setter: { if (isValidRegisterForLiveness(codeBlock, instruction[1].u.operand)) setForOperand(codeBlock, uses, instruction[1].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[3].u.operand)) setForOperand(codeBlock, uses, instruction[3].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[4].u.operand)) setForOperand(codeBlock, uses, instruction[4].u.operand); return; } // Second argument. case op_init_global_const_nop: case op_init_global_const: case op_push_name_scope: case op_get_from_scope: case op_to_primitive: case op_get_by_id: case op_get_by_id_out_of_line: case op_get_by_id_self: case op_get_by_id_proto: case op_get_by_id_chain: case op_get_by_id_getter_self: case op_get_by_id_getter_proto: case op_get_by_id_getter_chain: case op_get_by_id_custom_self: case op_get_by_id_custom_proto: case op_get_by_id_custom_chain: case op_get_by_id_generic: case op_get_array_length: case op_get_string_length: case op_get_arguments_length: case op_typeof: case op_is_undefined: case op_is_boolean: case op_is_number: case op_is_string: case op_is_object: case op_is_function: case op_to_number: case op_negate: case op_neq_null: case op_eq_null: case op_not: case op_mov: case op_new_array_with_size: case op_create_this: { if (isValidRegisterForLiveness(codeBlock, instruction[2].u.operand)) setForOperand(codeBlock, uses, instruction[2].u.operand); return; } // Second and third arguments. case op_get_by_val: case op_get_argument_by_val: case op_in: case op_instanceof: case op_check_has_instance: case op_add: case op_mul: case op_div: case op_mod: case op_sub: case op_lshift: case op_rshift: case op_urshift: case op_bitand: case op_bitxor: case op_bitor: case op_less: case op_lesseq: case op_greater: case op_greatereq: case op_nstricteq: case op_stricteq: case op_neq: case op_eq: { if (isValidRegisterForLiveness(codeBlock, instruction[2].u.operand)) setForOperand(codeBlock, uses, instruction[2].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[3].u.operand)) setForOperand(codeBlock, uses, instruction[3].u.operand); return; } // Second, third, and fourth arguments. case op_call_varargs: case op_get_pnames: { if (isValidRegisterForLiveness(codeBlock, instruction[2].u.operand)) setForOperand(codeBlock, uses, instruction[2].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[3].u.operand)) setForOperand(codeBlock, uses, instruction[3].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[4].u.operand)) setForOperand(codeBlock, uses, instruction[4].u.operand); return; } // Second, third, fourth, and fifth arguments. case op_next_pname: { if (isValidRegisterForLiveness(codeBlock, instruction[2].u.operand)) setForOperand(codeBlock, uses, instruction[2].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[3].u.operand)) setForOperand(codeBlock, uses, instruction[3].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[4].u.operand)) setForOperand(codeBlock, uses, instruction[4].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[5].u.operand)) setForOperand(codeBlock, uses, instruction[5].u.operand); return; } // Second, third, fourth, fifth, and sixth arguments. case op_get_by_pname: { if (isValidRegisterForLiveness(codeBlock, instruction[2].u.operand)) setForOperand(codeBlock, uses, instruction[2].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[3].u.operand)) setForOperand(codeBlock, uses, instruction[3].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[4].u.operand)) setForOperand(codeBlock, uses, instruction[4].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[5].u.operand)) setForOperand(codeBlock, uses, instruction[5].u.operand); if (isValidRegisterForLiveness(codeBlock, instruction[6].u.operand)) setForOperand(codeBlock, uses, instruction[6].u.operand); return; } // Third argument. case op_switch_string: case op_switch_char: case op_switch_imm: { if (isValidRegisterForLiveness(codeBlock, instruction[3].u.operand)) setForOperand(codeBlock, uses, instruction[3].u.operand); return; } // Variable number of arguments. case op_new_array: case op_strcat: { int base = instruction[2].u.operand; int count = instruction[3].u.operand; for (int i = 0; i < count; i++) { if (isValidRegisterForLiveness(codeBlock, base + i)) setForOperand(codeBlock, uses, base + i); } return; } // Crazy stuff for calls. case op_construct: case op_call_eval: case op_call: { if (isValidRegisterForLiveness(codeBlock, instruction[2].u.operand)) setForOperand(codeBlock, uses, instruction[2].u.operand); int argCount = instruction[3].u.operand; int registerOffset = instruction[4].u.operand; int lastArg = registerOffset + CallFrame::thisArgumentOffset(); for (int i = 0; i < argCount; i++) { if (isValidRegisterForLiveness(codeBlock, lastArg - i)) setForOperand(codeBlock, uses, lastArg - i); } return; } // Special stuff for tear off arguments. case op_tear_off_arguments: { if (isValidRegisterForLiveness(codeBlock, instruction[1].u.operand)) setForOperand(codeBlock, uses, instruction[1].u.operand); if (isValidRegisterForLiveness(codeBlock, unmodifiedArgumentsRegister(VirtualRegister(instruction[1].u.operand)).offset())) setForOperand(codeBlock, uses, unmodifiedArgumentsRegister(VirtualRegister(instruction[1].u.operand)).offset()); if (isValidRegisterForLiveness(codeBlock, instruction[2].u.operand)) setForOperand(codeBlock, uses, instruction[2].u.operand); return; } #define LLINT_HELPER_OPCODES(opcode, length) case opcode: FOR_EACH_LLINT_OPCODE_EXTENSION(LLINT_HELPER_OPCODES) return; #undef LLINT_HELPER_OPCODES } }
void reifyInlinedCallFrames(CCallHelpers& jit, const OSRExitBase& exit) { ASSERT(jit.baselineCodeBlock()->jitType() == JITCode::BaselineJIT); jit.storePtr(AssemblyHelpers::TrustedImmPtr(jit.baselineCodeBlock()), AssemblyHelpers::addressFor((VirtualRegister)JSStack::CodeBlock)); CodeOrigin codeOrigin; for (codeOrigin = exit.m_codeOrigin; codeOrigin.inlineCallFrame; codeOrigin = codeOrigin.inlineCallFrame->caller) { InlineCallFrame* inlineCallFrame = codeOrigin.inlineCallFrame; CodeBlock* baselineCodeBlock = jit.baselineCodeBlockFor(codeOrigin); CodeBlock* baselineCodeBlockForCaller = jit.baselineCodeBlockFor(inlineCallFrame->caller); unsigned callBytecodeIndex = inlineCallFrame->caller.bytecodeIndex; CallLinkInfo& callLinkInfo = baselineCodeBlockForCaller->getCallLinkInfo(callBytecodeIndex); void* jumpTarget = callLinkInfo.callReturnLocation.executableAddress(); GPRReg callerFrameGPR; if (inlineCallFrame->caller.inlineCallFrame) { jit.addPtr(AssemblyHelpers::TrustedImm32(inlineCallFrame->caller.inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister, GPRInfo::regT3); callerFrameGPR = GPRInfo::regT3; } else callerFrameGPR = GPRInfo::callFrameRegister; #if USE(JSVALUE64) jit.storePtr(AssemblyHelpers::TrustedImmPtr(baselineCodeBlock), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::CodeBlock))); if (!inlineCallFrame->isClosureCall) jit.store64(AssemblyHelpers::TrustedImm64(JSValue::encode(JSValue(inlineCallFrame->calleeConstant()->scope()))), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ScopeChain))); jit.store64(callerFrameGPR, AssemblyHelpers::addressForByteOffset(inlineCallFrame->callerFrameOffset())); jit.storePtr(AssemblyHelpers::TrustedImmPtr(jumpTarget), AssemblyHelpers::addressForByteOffset(inlineCallFrame->returnPCOffset())); uint32_t locationBits = CallFrame::Location::encodeAsBytecodeOffset(codeOrigin.bytecodeIndex); jit.store32(AssemblyHelpers::TrustedImm32(locationBits), AssemblyHelpers::tagFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ArgumentCount))); jit.store32(AssemblyHelpers::TrustedImm32(inlineCallFrame->arguments.size()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ArgumentCount))); if (!inlineCallFrame->isClosureCall) jit.store64(AssemblyHelpers::TrustedImm64(JSValue::encode(JSValue(inlineCallFrame->calleeConstant()))), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::Callee))); // Leave the captured arguments in regT3. if (baselineCodeBlock->usesArguments()) jit.loadPtr(AssemblyHelpers::addressFor(VirtualRegister(inlineCallFrame->stackOffset + unmodifiedArgumentsRegister(baselineCodeBlock->argumentsRegister()).offset())), GPRInfo::regT3); #else // USE(JSVALUE64) // so this is the 32-bit part jit.storePtr(AssemblyHelpers::TrustedImmPtr(baselineCodeBlock), AssemblyHelpers::addressFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::CodeBlock))); jit.store32(AssemblyHelpers::TrustedImm32(JSValue::CellTag), AssemblyHelpers::tagFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ScopeChain))); if (!inlineCallFrame->isClosureCall) jit.storePtr(AssemblyHelpers::TrustedImmPtr(inlineCallFrame->calleeConstant()->scope()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ScopeChain))); jit.storePtr(callerFrameGPR, AssemblyHelpers::addressForByteOffset(inlineCallFrame->callerFrameOffset())); jit.storePtr(AssemblyHelpers::TrustedImmPtr(jumpTarget), AssemblyHelpers::addressForByteOffset(inlineCallFrame->returnPCOffset())); Instruction* instruction = baselineCodeBlock->instructions().begin() + codeOrigin.bytecodeIndex; uint32_t locationBits = CallFrame::Location::encodeAsBytecodeInstruction(instruction); jit.store32(AssemblyHelpers::TrustedImm32(locationBits), AssemblyHelpers::tagFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ArgumentCount))); jit.store32(AssemblyHelpers::TrustedImm32(inlineCallFrame->arguments.size()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::ArgumentCount))); jit.store32(AssemblyHelpers::TrustedImm32(JSValue::CellTag), AssemblyHelpers::tagFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::Callee))); if (!inlineCallFrame->isClosureCall) jit.storePtr(AssemblyHelpers::TrustedImmPtr(inlineCallFrame->calleeConstant()), AssemblyHelpers::payloadFor((VirtualRegister)(inlineCallFrame->stackOffset + JSStack::Callee))); // Leave the captured arguments in regT3. if (baselineCodeBlock->usesArguments()) jit.loadPtr(AssemblyHelpers::payloadFor(VirtualRegister(inlineCallFrame->stackOffset + unmodifiedArgumentsRegister(baselineCodeBlock->argumentsRegister()).offset())), GPRInfo::regT3); #endif // USE(JSVALUE64) // ending the #else part, so directly above is the 32-bit part if (baselineCodeBlock->usesArguments()) { AssemblyHelpers::Jump noArguments = jit.branchTestPtr(AssemblyHelpers::Zero, GPRInfo::regT3); jit.addPtr(AssemblyHelpers::TrustedImm32(inlineCallFrame->stackOffset * sizeof(EncodedJSValue)), GPRInfo::callFrameRegister, GPRInfo::regT0); jit.storePtr(GPRInfo::regT0, AssemblyHelpers::Address(GPRInfo::regT3, Arguments::offsetOfRegisters())); noArguments.link(&jit); } } #if USE(JSVALUE64) uint32_t locationBits = CallFrame::Location::encodeAsBytecodeOffset(codeOrigin.bytecodeIndex); #else Instruction* instruction = jit.baselineCodeBlock()->instructions().begin() + codeOrigin.bytecodeIndex; uint32_t locationBits = CallFrame::Location::encodeAsBytecodeInstruction(instruction); #endif jit.store32(AssemblyHelpers::TrustedImm32(locationBits), AssemblyHelpers::tagFor((VirtualRegister)(JSStack::ArgumentCount))); }
void LocalOSRAvailabilityCalculator::executeNode(Node* node) { switch (node->op()) { case PutStack: { StackAccessData* data = node->stackAccessData(); m_availability.m_locals.operand(data->local).setFlush(data->flushedAt()); break; } case KillStack: { m_availability.m_locals.operand(node->unlinkedLocal()).setFlush(FlushedAt(ConflictingFlush)); break; } case GetStack: { StackAccessData* data = node->stackAccessData(); m_availability.m_locals.operand(data->local) = Availability(node, data->flushedAt()); break; } case MovHint: { m_availability.m_locals.operand(node->unlinkedLocal()).setNode(node->child1().node()); break; } case ZombieHint: { m_availability.m_locals.operand(node->unlinkedLocal()).setNodeUnavailable(); break; } case LoadVarargs: case ForwardVarargs: { LoadVarargsData* data = node->loadVarargsData(); m_availability.m_locals.operand(data->count) = Availability(FlushedAt(FlushedInt32, data->machineCount)); for (unsigned i = data->limit; i--;) { m_availability.m_locals.operand(VirtualRegister(data->start.offset() + i)) = Availability(FlushedAt(FlushedJSValue, VirtualRegister(data->machineStart.offset() + i))); } break; } case PhantomCreateRest: case PhantomDirectArguments: case PhantomClonedArguments: { InlineCallFrame* inlineCallFrame = node->origin.semantic.inlineCallFrame; if (!inlineCallFrame) { // We don't need to record anything about how the arguments are to be recovered. It's just a // given that we can read them from the stack. break; } unsigned numberOfArgumentsToSkip = 0; if (node->op() == PhantomCreateRest) numberOfArgumentsToSkip = node->numberOfArgumentsToSkip(); if (inlineCallFrame->isVarargs()) { // Record how to read each argument and the argument count. Availability argumentCount = m_availability.m_locals.operand(inlineCallFrame->stackOffset + CallFrameSlot::argumentCount); m_availability.m_heap.set(PromotedHeapLocation(ArgumentCountPLoc, node), argumentCount); } if (inlineCallFrame->isClosureCall) { Availability callee = m_availability.m_locals.operand( inlineCallFrame->stackOffset + CallFrameSlot::callee); m_availability.m_heap.set(PromotedHeapLocation(ArgumentsCalleePLoc, node), callee); } for (unsigned i = numberOfArgumentsToSkip; i < inlineCallFrame->arguments.size() - 1; ++i) { Availability argument = m_availability.m_locals.operand( inlineCallFrame->stackOffset + CallFrame::argumentOffset(i)); m_availability.m_heap.set(PromotedHeapLocation(ArgumentPLoc, node, i), argument); } break; } case PutHint: { m_availability.m_heap.set( PromotedHeapLocation(node->child1().node(), node->promotedLocationDescriptor()), Availability(node->child2().node())); break; } case PhantomSpread: m_availability.m_heap.set(PromotedHeapLocation(SpreadPLoc, node), Availability(node->child1().node())); break; case PhantomNewArrayWithSpread: for (unsigned i = 0; i < node->numChildren(); i++) { Node* child = m_graph.varArgChild(node, i).node(); m_availability.m_heap.set(PromotedHeapLocation(NewArrayWithSpreadArgumentPLoc, node, i), Availability(child)); } break; default: break; } }
static void fixFunctionBasedOnStackMaps( State& state, CodeBlock* codeBlock, JITCode* jitCode, GeneratedFunction generatedFunction, StackMaps::RecordMap& recordMap, bool didSeeUnwindInfo) { Graph& graph = state.graph; VM& vm = graph.m_vm; StackMaps stackmaps = jitCode->stackmaps; int localsOffset = offsetOfStackRegion(recordMap, state.capturedStackmapID) + graph.m_nextMachineLocal; int varargsSpillSlotsOffset; if (state.varargsSpillSlotsStackmapID != UINT_MAX) varargsSpillSlotsOffset = offsetOfStackRegion(recordMap, state.varargsSpillSlotsStackmapID); else varargsSpillSlotsOffset = 0; for (unsigned i = graph.m_inlineVariableData.size(); i--;) { InlineCallFrame* inlineCallFrame = graph.m_inlineVariableData[i].inlineCallFrame; if (inlineCallFrame->argumentsRegister.isValid()) inlineCallFrame->argumentsRegister += localsOffset; if (inlineCallFrame->argumentCountRegister.isValid()) inlineCallFrame->argumentCountRegister += localsOffset; for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) { inlineCallFrame->arguments[argument] = inlineCallFrame->arguments[argument].withLocalsOffset(localsOffset); } if (inlineCallFrame->isClosureCall) { inlineCallFrame->calleeRecovery = inlineCallFrame->calleeRecovery.withLocalsOffset(localsOffset); } } if (codeBlock->usesArguments()) { codeBlock->setArgumentsRegister( VirtualRegister(codeBlock->argumentsRegister().offset() + localsOffset)); } MacroAssembler::Label stackOverflowException; { CCallHelpers checkJIT(&vm, codeBlock); // At this point it's perfectly fair to just blow away all state and restore the // JS JIT view of the universe. checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0); checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); MacroAssembler::Call callLookupExceptionHandler = checkJIT.call(); checkJIT.jumpToExceptionHandler(); stackOverflowException = checkJIT.label(); checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0); checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); MacroAssembler::Call callLookupExceptionHandlerFromCallerFrame = checkJIT.call(); checkJIT.jumpToExceptionHandler(); auto linkBuffer = std::make_unique<LinkBuffer>( vm, checkJIT, codeBlock, JITCompilationMustSucceed); linkBuffer->link(callLookupExceptionHandler, FunctionPtr(lookupExceptionHandler)); linkBuffer->link(callLookupExceptionHandlerFromCallerFrame, FunctionPtr(lookupExceptionHandlerFromCallerFrame)); state.finalizer->handleExceptionsLinkBuffer = WTF::move(linkBuffer); } ExitThunkGenerator exitThunkGenerator(state); exitThunkGenerator.emitThunks(); if (exitThunkGenerator.didThings()) { RELEASE_ASSERT(state.finalizer->osrExit.size()); RELEASE_ASSERT(didSeeUnwindInfo); auto linkBuffer = std::make_unique<LinkBuffer>( vm, exitThunkGenerator, codeBlock, JITCompilationMustSucceed); RELEASE_ASSERT(state.finalizer->osrExit.size() == state.jitCode->osrExit.size()); for (unsigned i = 0; i < state.jitCode->osrExit.size(); ++i) { OSRExitCompilationInfo& info = state.finalizer->osrExit[i]; OSRExit& exit = jitCode->osrExit[i]; if (verboseCompilationEnabled()) dataLog("Handling OSR stackmap #", exit.m_stackmapID, " for ", exit.m_codeOrigin, "\n"); auto iter = recordMap.find(exit.m_stackmapID); if (iter == recordMap.end()) { // It was optimized out. continue; } info.m_thunkAddress = linkBuffer->locationOf(info.m_thunkLabel); exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump); for (unsigned j = exit.m_values.size(); j--;) { ExitValue value = exit.m_values[j]; if (!value.isInJSStackSomehow()) continue; if (!value.virtualRegister().isLocal()) continue; exit.m_values[j] = value.withVirtualRegister( VirtualRegister(value.virtualRegister().offset() + localsOffset)); } if (verboseCompilationEnabled()) { DumpContext context; dataLog(" Exit values: ", inContext(exit.m_values, &context), "\n"); } } state.finalizer->exitThunksLinkBuffer = WTF::move(linkBuffer); } if (!state.getByIds.isEmpty() || !state.putByIds.isEmpty() || !state.checkIns.isEmpty()) { CCallHelpers slowPathJIT(&vm, codeBlock); CCallHelpers::JumpList exceptionTarget; for (unsigned i = state.getByIds.size(); i--;) { GetByIdDescriptor& getById = state.getByIds[i]; if (verboseCompilationEnabled()) dataLog("Handling GetById stackmap #", getById.stackmapID(), "\n"); auto iter = recordMap.find(getById.stackmapID()); if (iter == recordMap.end()) { // It was optimized out. continue; } for (unsigned i = 0; i < iter->value.size(); ++i) { StackMaps::Record& record = iter->value[i]; RegisterSet usedRegisters = usedRegistersFor(record); GPRReg result = record.locations[0].directGPR(); GPRReg base = record.locations[1].directGPR(); JITGetByIdGenerator gen( codeBlock, getById.codeOrigin(), usedRegisters, JSValueRegs(base), JSValueRegs(result), NeedToSpill); MacroAssembler::Label begin = slowPathJIT.label(); MacroAssembler::Call call = callOperation( state, usedRegisters, slowPathJIT, getById.codeOrigin(), &exceptionTarget, operationGetByIdOptimize, result, gen.stubInfo(), base, getById.uid()); gen.reportSlowPathCall(begin, call); getById.m_slowPathDone.append(slowPathJIT.jump()); getById.m_generators.append(gen); } } for (unsigned i = state.putByIds.size(); i--;) { PutByIdDescriptor& putById = state.putByIds[i]; if (verboseCompilationEnabled()) dataLog("Handling PutById stackmap #", putById.stackmapID(), "\n"); auto iter = recordMap.find(putById.stackmapID()); if (iter == recordMap.end()) { // It was optimized out. continue; } for (unsigned i = 0; i < iter->value.size(); ++i) { StackMaps::Record& record = iter->value[i]; RegisterSet usedRegisters = usedRegistersFor(record); GPRReg base = record.locations[0].directGPR(); GPRReg value = record.locations[1].directGPR(); JITPutByIdGenerator gen( codeBlock, putById.codeOrigin(), usedRegisters, JSValueRegs(base), JSValueRegs(value), GPRInfo::patchpointScratchRegister, NeedToSpill, putById.ecmaMode(), putById.putKind()); MacroAssembler::Label begin = slowPathJIT.label(); MacroAssembler::Call call = callOperation( state, usedRegisters, slowPathJIT, putById.codeOrigin(), &exceptionTarget, gen.slowPathFunction(), gen.stubInfo(), value, base, putById.uid()); gen.reportSlowPathCall(begin, call); putById.m_slowPathDone.append(slowPathJIT.jump()); putById.m_generators.append(gen); } } for (unsigned i = state.checkIns.size(); i--;) { CheckInDescriptor& checkIn = state.checkIns[i]; if (verboseCompilationEnabled()) dataLog("Handling checkIn stackmap #", checkIn.stackmapID(), "\n"); auto iter = recordMap.find(checkIn.stackmapID()); if (iter == recordMap.end()) { // It was optimized out. continue; } for (unsigned i = 0; i < iter->value.size(); ++i) { StackMaps::Record& record = iter->value[i]; RegisterSet usedRegisters = usedRegistersFor(record); GPRReg result = record.locations[0].directGPR(); GPRReg obj = record.locations[1].directGPR(); StructureStubInfo* stubInfo = codeBlock->addStubInfo(); stubInfo->codeOrigin = checkIn.codeOrigin(); stubInfo->patch.baseGPR = static_cast<int8_t>(obj); stubInfo->patch.valueGPR = static_cast<int8_t>(result); stubInfo->patch.usedRegisters = usedRegisters; stubInfo->patch.spillMode = NeedToSpill; MacroAssembler::Label begin = slowPathJIT.label(); MacroAssembler::Call slowCall = callOperation( state, usedRegisters, slowPathJIT, checkIn.codeOrigin(), &exceptionTarget, operationInOptimize, result, stubInfo, obj, checkIn.m_id); checkIn.m_slowPathDone.append(slowPathJIT.jump()); checkIn.m_generators.append(CheckInGenerator(stubInfo, slowCall, begin)); } } exceptionTarget.link(&slowPathJIT); MacroAssembler::Jump exceptionJump = slowPathJIT.jump(); state.finalizer->sideCodeLinkBuffer = std::make_unique<LinkBuffer>(vm, slowPathJIT, codeBlock, JITCompilationMustSucceed); state.finalizer->sideCodeLinkBuffer->link( exceptionJump, state.finalizer->handleExceptionsLinkBuffer->entrypoint()); for (unsigned i = state.getByIds.size(); i--;) { generateICFastPath( state, codeBlock, generatedFunction, recordMap, state.getByIds[i], sizeOfGetById()); } for (unsigned i = state.putByIds.size(); i--;) { generateICFastPath( state, codeBlock, generatedFunction, recordMap, state.putByIds[i], sizeOfPutById()); } for (unsigned i = state.checkIns.size(); i--;) { generateCheckInICFastPath( state, codeBlock, generatedFunction, recordMap, state.checkIns[i], sizeOfIn()); } } adjustCallICsForStackmaps(state.jsCalls, recordMap); for (unsigned i = state.jsCalls.size(); i--;) { JSCall& call = state.jsCalls[i]; CCallHelpers fastPathJIT(&vm, codeBlock); call.emit(fastPathJIT); char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset; LinkBuffer linkBuffer(vm, fastPathJIT, startOfIC, sizeOfCall()); if (!linkBuffer.isValid()) { dataLog("Failed to insert inline cache for call because we thought the size would be ", sizeOfCall(), " but it ended up being ", fastPathJIT.m_assembler.codeSize(), " prior to compaction.\n"); RELEASE_ASSERT_NOT_REACHED(); } MacroAssembler::AssemblerType_T::fillNops( startOfIC + linkBuffer.size(), sizeOfCall() - linkBuffer.size()); call.link(vm, linkBuffer); } adjustCallICsForStackmaps(state.jsCallVarargses, recordMap); for (unsigned i = state.jsCallVarargses.size(); i--;) { JSCallVarargs& call = state.jsCallVarargses[i]; CCallHelpers fastPathJIT(&vm, codeBlock); call.emit(fastPathJIT, graph, varargsSpillSlotsOffset); char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset; size_t sizeOfIC = sizeOfICFor(call.node()); LinkBuffer linkBuffer(vm, fastPathJIT, startOfIC, sizeOfIC); if (!linkBuffer.isValid()) { dataLog("Failed to insert inline cache for varargs call (specifically, ", Graph::opName(call.node()->op()), ") because we thought the size would be ", sizeOfIC, " but it ended up being ", fastPathJIT.m_assembler.codeSize(), " prior to compaction.\n"); RELEASE_ASSERT_NOT_REACHED(); } MacroAssembler::AssemblerType_T::fillNops( startOfIC + linkBuffer.size(), sizeOfIC - linkBuffer.size()); call.link(vm, linkBuffer, state.finalizer->handleExceptionsLinkBuffer->entrypoint()); } RepatchBuffer repatchBuffer(codeBlock); auto iter = recordMap.find(state.handleStackOverflowExceptionStackmapID); // It's sort of remotely possible that we won't have an in-band exception handling // path, for some kinds of functions. if (iter != recordMap.end()) { for (unsigned i = iter->value.size(); i--;) { StackMaps::Record& record = iter->value[i]; CodeLocationLabel source = CodeLocationLabel( bitwise_cast<char*>(generatedFunction) + record.instructionOffset); RELEASE_ASSERT(stackOverflowException.isSet()); repatchBuffer.replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->locationOf(stackOverflowException)); } } iter = recordMap.find(state.handleExceptionStackmapID); // It's sort of remotely possible that we won't have an in-band exception handling // path, for some kinds of functions. if (iter != recordMap.end()) { for (unsigned i = iter->value.size(); i--;) { StackMaps::Record& record = iter->value[i]; CodeLocationLabel source = CodeLocationLabel( bitwise_cast<char*>(generatedFunction) + record.instructionOffset); repatchBuffer.replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->entrypoint()); } } for (unsigned exitIndex = 0; exitIndex < jitCode->osrExit.size(); ++exitIndex) { OSRExitCompilationInfo& info = state.finalizer->osrExit[exitIndex]; OSRExit& exit = jitCode->osrExit[exitIndex]; iter = recordMap.find(exit.m_stackmapID); Vector<const void*> codeAddresses; if (iter != recordMap.end()) { for (unsigned i = iter->value.size(); i--;) { StackMaps::Record& record = iter->value[i]; CodeLocationLabel source = CodeLocationLabel( bitwise_cast<char*>(generatedFunction) + record.instructionOffset); codeAddresses.append(bitwise_cast<char*>(generatedFunction) + record.instructionOffset + MacroAssembler::maxJumpReplacementSize()); if (info.m_isInvalidationPoint) jitCode->common.jumpReplacements.append(JumpReplacement(source, info.m_thunkAddress)); else repatchBuffer.replaceWithJump(source, info.m_thunkAddress); } } if (graph.compilation()) graph.compilation()->addOSRExitSite(codeAddresses); } }
bool run() { ScoreBoard scoreBoard(m_graph.m_nextMachineLocal); scoreBoard.assertClear(); #if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE) bool needsNewLine = false; #endif for (size_t blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) { BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; if (!block->isReachable) continue; for (size_t indexInBlock = 0; indexInBlock < block->size(); ++indexInBlock) { Node* node = block->at(indexInBlock); #if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE) if (needsNewLine) dataLogF("\n"); dataLogF(" @%u:", node->index()); needsNewLine = true; #endif if (!node->shouldGenerate()) continue; switch (node->op()) { case Phi: case Flush: case PhantomLocal: continue; case GetLocal: ASSERT(!node->child1()->hasResult()); break; default: break; } // First, call use on all of the current node's children, then // allocate a VirtualRegister for this node. We do so in this // order so that if a child is on its last use, and a // VirtualRegister is freed, then it may be reused for node. if (node->flags() & NodeHasVarArgs) { for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++) scoreBoard.useIfHasResult(m_graph.m_varArgChildren[childIdx]); } else { scoreBoard.useIfHasResult(node->child1()); scoreBoard.useIfHasResult(node->child2()); scoreBoard.useIfHasResult(node->child3()); } if (!node->hasResult()) continue; VirtualRegister virtualRegister = scoreBoard.allocate(); #if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE) dataLogF( " Assigning virtual register %u to node %u.", virtualRegister, node->index()); #endif node->setVirtualRegister(virtualRegister); // 'mustGenerate' nodes have their useCount artificially elevated, // call use now to account for this. if (node->mustGenerate()) scoreBoard.use(node); } scoreBoard.assertClear(); } #if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE) if (needsNewLine) dataLogF("\n"); #endif // Record the number of virtual registers we're using. This is used by calls // to figure out where to put the parameters. m_graph.m_nextMachineLocal = scoreBoard.highWatermark(); // 'm_numCalleeRegisters' is the number of locals and temporaries allocated // for the function (and checked for on entry). Since we perform a new and // different allocation of temporaries, more registers may now be required. // This also accounts for the number of temporaries that may be needed if we // OSR exit, due to inlining. Hence this computes the number of temporaries // that could be used by this code block even if it exits; it may be more // than what this code block needs if it never exits. unsigned calleeRegisters = scoreBoard.highWatermark() + m_graph.m_parameterSlots; for (InlineCallFrameSet::iterator iter = m_graph.m_inlineCallFrames->begin(); !!iter; ++iter) { InlineCallFrame* inlineCallFrame = *iter; CodeBlock* codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame); unsigned requiredCalleeRegisters = VirtualRegister(inlineCallFrame->stackOffset).toLocal() + 1 + codeBlock->m_numCalleeRegisters; if (requiredCalleeRegisters > calleeRegisters) calleeRegisters = requiredCalleeRegisters; } if ((unsigned)codeBlock()->m_numCalleeRegisters < calleeRegisters) codeBlock()->m_numCalleeRegisters = calleeRegisters; #if DFG_ENABLE(DEBUG_VERBOSE) dataLogF("Num callee registers: %u\n", calleeRegisters); #endif return true; }
void handleBlockForTryCatch(BasicBlock* block, InsertionSet& insertionSet) { HandlerInfo* currentExceptionHandler = nullptr; FastBitVector liveAtCatchHead; liveAtCatchHead.resize(m_graph.block(0)->variablesAtTail.numberOfLocals()); HandlerInfo* cachedHandlerResult; CodeOrigin cachedCodeOrigin; auto catchHandler = [&] (CodeOrigin origin) -> HandlerInfo* { ASSERT(origin); if (origin == cachedCodeOrigin) return cachedHandlerResult; unsigned bytecodeIndexToCheck = origin.bytecodeIndex; cachedCodeOrigin = origin; while (1) { InlineCallFrame* inlineCallFrame = origin.inlineCallFrame; CodeBlock* codeBlock = m_graph.baselineCodeBlockFor(inlineCallFrame); if (HandlerInfo* handler = codeBlock->handlerForBytecodeOffset(bytecodeIndexToCheck)) { liveAtCatchHead.clearAll(); unsigned catchBytecodeIndex = handler->target; m_graph.forAllLocalsLiveInBytecode(CodeOrigin(catchBytecodeIndex, inlineCallFrame), [&] (VirtualRegister operand) { liveAtCatchHead[operand.toLocal()] = true; }); cachedHandlerResult = handler; break; } if (!inlineCallFrame) { cachedHandlerResult = nullptr; break; } bytecodeIndexToCheck = inlineCallFrame->directCaller.bytecodeIndex; origin = inlineCallFrame->directCaller; } return cachedHandlerResult; }; Operands<VariableAccessData*> currentBlockAccessData(block->variablesAtTail.numberOfArguments(), block->variablesAtTail.numberOfLocals(), nullptr); HashSet<InlineCallFrame*> seenInlineCallFrames; auto flushEverything = [&] (NodeOrigin origin, unsigned index) { RELEASE_ASSERT(currentExceptionHandler); auto flush = [&] (VirtualRegister operand, bool alwaysInsert) { if ((operand.isLocal() && liveAtCatchHead[operand.toLocal()]) || operand.isArgument() || alwaysInsert) { ASSERT(isValidFlushLocation(block, index, operand)); VariableAccessData* accessData = currentBlockAccessData.operand(operand); if (!accessData) accessData = newVariableAccessData(operand); currentBlockAccessData.operand(operand) = accessData; insertionSet.insertNode(index, SpecNone, Flush, origin, OpInfo(accessData)); } }; for (unsigned local = 0; local < block->variablesAtTail.numberOfLocals(); local++) flush(virtualRegisterForLocal(local), false); for (InlineCallFrame* inlineCallFrame : seenInlineCallFrames) flush(VirtualRegister(inlineCallFrame->stackOffset + CallFrame::thisArgumentOffset()), true); flush(VirtualRegister(CallFrame::thisArgumentOffset()), true); seenInlineCallFrames.clear(); }; for (unsigned nodeIndex = 0; nodeIndex < block->size(); nodeIndex++) { Node* node = block->at(nodeIndex); { HandlerInfo* newHandler = catchHandler(node->origin.semantic); if (newHandler != currentExceptionHandler && currentExceptionHandler) flushEverything(node->origin, nodeIndex); currentExceptionHandler = newHandler; } if (currentExceptionHandler && (node->op() == SetLocal || node->op() == SetArgument)) { InlineCallFrame* inlineCallFrame = node->origin.semantic.inlineCallFrame; if (inlineCallFrame) seenInlineCallFrames.add(inlineCallFrame); VirtualRegister operand = node->local(); int stackOffset = inlineCallFrame ? inlineCallFrame->stackOffset : 0; if ((operand.isLocal() && liveAtCatchHead[operand.toLocal()]) || operand.isArgument() || (operand.offset() == stackOffset + CallFrame::thisArgumentOffset())) { ASSERT(isValidFlushLocation(block, nodeIndex, operand)); VariableAccessData* variableAccessData = currentBlockAccessData.operand(operand); if (!variableAccessData) variableAccessData = newVariableAccessData(operand); insertionSet.insertNode(nodeIndex, SpecNone, Flush, node->origin, OpInfo(variableAccessData)); } } if (node->accessesStack(m_graph)) currentBlockAccessData.operand(node->local()) = node->variableAccessData(); } if (currentExceptionHandler) { NodeOrigin origin = block->at(block->size() - 1)->origin; flushEverything(origin, block->size()); } }
void VariableEventStream::reconstruct( CodeBlock* codeBlock, CodeOrigin codeOrigin, MinifiedGraph& graph, unsigned index, Operands<ValueRecovery>& valueRecoveries) const { ASSERT(codeBlock->jitType() == JITCode::DFGJIT); CodeBlock* baselineCodeBlock = codeBlock->baselineVersion(); unsigned numVariables; if (codeOrigin.inlineCallFrame) numVariables = baselineCodeBlockForInlineCallFrame(codeOrigin.inlineCallFrame)->m_numCalleeRegisters + VirtualRegister(codeOrigin.inlineCallFrame->stackOffset).toLocal() + 1; else numVariables = baselineCodeBlock->m_numCalleeRegisters; // Crazy special case: if we're at index == 0 then this must be an argument check // failure, in which case all variables are already set up. The recoveries should // reflect this. if (!index) { valueRecoveries = Operands<ValueRecovery>(codeBlock->numParameters(), numVariables); for (size_t i = 0; i < valueRecoveries.size(); ++i) { valueRecoveries[i] = ValueRecovery::displacedInJSStack( VirtualRegister(valueRecoveries.operandForIndex(i)), DataFormatJS); } return; } // Step 1: Find the last checkpoint, and figure out the number of virtual registers as we go. unsigned startIndex = index - 1; while (at(startIndex).kind() != Reset) startIndex--; // Step 2: Create a mock-up of the DFG's state and execute the events. Operands<ValueSource> operandSources(codeBlock->numParameters(), numVariables); for (unsigned i = operandSources.size(); i--;) operandSources[i] = ValueSource(SourceIsDead); HashMap<MinifiedID, MinifiedGenerationInfo> generationInfos; for (unsigned i = startIndex; i < index; ++i) { const VariableEvent& event = at(i); switch (event.kind()) { case Reset: // nothing to do. break; case BirthToFill: case BirthToSpill: case Birth: { MinifiedGenerationInfo info; info.update(event); generationInfos.add(event.id(), info); break; } case Fill: case Spill: case Death: { HashMap<MinifiedID, MinifiedGenerationInfo>::iterator iter = generationInfos.find(event.id()); ASSERT(iter != generationInfos.end()); iter->value.update(event); break; } case MovHintEvent: if (operandSources.hasOperand(event.bytecodeRegister())) operandSources.setOperand(event.bytecodeRegister(), ValueSource(event.id())); break; case SetLocalEvent: if (operandSources.hasOperand(event.bytecodeRegister())) operandSources.setOperand(event.bytecodeRegister(), ValueSource::forDataFormat(event.machineRegister(), event.dataFormat())); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } } // Step 3: Compute value recoveries! valueRecoveries = Operands<ValueRecovery>(codeBlock->numParameters(), numVariables); for (unsigned i = 0; i < operandSources.size(); ++i) { ValueSource& source = operandSources[i]; if (source.isTriviallyRecoverable()) { valueRecoveries[i] = source.valueRecovery(); continue; } ASSERT(source.kind() == HaveNode); MinifiedNode* node = graph.at(source.id()); MinifiedGenerationInfo info = generationInfos.get(source.id()); if (!info.alive) { valueRecoveries[i] = ValueRecovery::constant(jsUndefined()); continue; } if (tryToSetConstantRecovery(valueRecoveries[i], node)) continue; ASSERT(info.format != DataFormatNone); if (info.filled) { if (info.format == DataFormatDouble) { valueRecoveries[i] = ValueRecovery::inFPR(info.u.fpr, DataFormatDouble); continue; } #if USE(JSVALUE32_64) if (info.format & DataFormatJS) { valueRecoveries[i] = ValueRecovery::inPair(info.u.pair.tagGPR, info.u.pair.payloadGPR); continue; } #endif valueRecoveries[i] = ValueRecovery::inGPR(info.u.gpr, info.format); continue; } valueRecoveries[i] = ValueRecovery::displacedInJSStack(static_cast<VirtualRegister>(info.u.virtualReg), info.format); } }
bool run() { SharedSymbolTable* symbolTable = codeBlock()->symbolTable(); // This enumerates the locals that we actually care about and packs them. So for example // if we use local 1, 3, 4, 5, 7, then we remap them: 1->0, 3->1, 4->2, 5->3, 7->4. We // treat a variable as being "used" if there exists an access to it (SetLocal, GetLocal, // Flush, PhantomLocal). BitVector usedLocals; // Collect those variables that are used from IR. bool hasGetLocalUnlinked = false; for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; for (unsigned nodeIndex = block->size(); nodeIndex--;) { Node* node = block->at(nodeIndex); switch (node->op()) { case GetLocal: case SetLocal: case Flush: case PhantomLocal: { VariableAccessData* variable = node->variableAccessData(); if (variable->local().isArgument()) break; usedLocals.set(variable->local().toLocal()); break; } case GetLocalUnlinked: { VirtualRegister operand = node->unlinkedLocal(); if (operand.isArgument()) break; usedLocals.set(operand.toLocal()); hasGetLocalUnlinked = true; break; } default: break; } } } // Ensure that captured variables and captured inline arguments are pinned down. // They should have been because of flushes, except that the flushes can be optimized // away. if (symbolTable) { for (int i = symbolTable->captureStart(); i > symbolTable->captureEnd(); i--) usedLocals.set(VirtualRegister(i).toLocal()); } if (codeBlock()->usesArguments()) { usedLocals.set(codeBlock()->argumentsRegister().toLocal()); usedLocals.set(unmodifiedArgumentsRegister(codeBlock()->argumentsRegister()).toLocal()); } if (codeBlock()->uncheckedActivationRegister().isValid()) usedLocals.set(codeBlock()->activationRegister().toLocal()); for (InlineCallFrameSet::iterator iter = m_graph.m_inlineCallFrames->begin(); !!iter; ++iter) { InlineCallFrame* inlineCallFrame = *iter; if (!inlineCallFrame->executable->usesArguments()) continue; VirtualRegister argumentsRegister = m_graph.argumentsRegisterFor(inlineCallFrame); usedLocals.set(argumentsRegister.toLocal()); usedLocals.set(unmodifiedArgumentsRegister(argumentsRegister).toLocal()); for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) { usedLocals.set(VirtualRegister( virtualRegisterForArgument(argument).offset() + inlineCallFrame->stackOffset).toLocal()); } } Vector<unsigned> allocation(usedLocals.size()); m_graph.m_nextMachineLocal = 0; for (unsigned i = 0; i < usedLocals.size(); ++i) { if (!usedLocals.get(i)) { allocation[i] = UINT_MAX; continue; } allocation[i] = m_graph.m_nextMachineLocal++; } for (unsigned i = m_graph.m_variableAccessData.size(); i--;) { VariableAccessData* variable = &m_graph.m_variableAccessData[i]; if (!variable->isRoot()) continue; if (variable->local().isArgument()) { variable->machineLocal() = variable->local(); continue; } size_t local = variable->local().toLocal(); if (local >= allocation.size()) continue; if (allocation[local] == UINT_MAX) continue; variable->machineLocal() = virtualRegisterForLocal( allocation[variable->local().toLocal()]); } if (codeBlock()->usesArguments()) { VirtualRegister argumentsRegister = virtualRegisterForLocal( allocation[codeBlock()->argumentsRegister().toLocal()]); RELEASE_ASSERT( virtualRegisterForLocal(allocation[ unmodifiedArgumentsRegister( codeBlock()->argumentsRegister()).toLocal()]) == unmodifiedArgumentsRegister(argumentsRegister)); codeBlock()->setArgumentsRegister(argumentsRegister); } if (codeBlock()->uncheckedActivationRegister().isValid()) { codeBlock()->setActivationRegister( virtualRegisterForLocal(allocation[codeBlock()->activationRegister().toLocal()])); } for (unsigned i = m_graph.m_inlineVariableData.size(); i--;) { InlineVariableData data = m_graph.m_inlineVariableData[i]; InlineCallFrame* inlineCallFrame = data.inlineCallFrame; if (inlineCallFrame->executable->usesArguments()) { inlineCallFrame->argumentsRegister = virtualRegisterForLocal( allocation[m_graph.argumentsRegisterFor(inlineCallFrame).toLocal()]); RELEASE_ASSERT( virtualRegisterForLocal(allocation[unmodifiedArgumentsRegister( m_graph.argumentsRegisterFor(inlineCallFrame)).toLocal()]) == unmodifiedArgumentsRegister(inlineCallFrame->argumentsRegister)); } for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) { ArgumentPosition& position = m_graph.m_argumentPositions[ data.argumentPositionStart + argument]; VariableAccessData* variable = position.someVariable(); ValueSource source; if (!variable) source = ValueSource(SourceIsDead); else { source = ValueSource::forFlushFormat( variable->machineLocal(), variable->flushFormat()); } inlineCallFrame->arguments[argument] = source.valueRecovery(); } RELEASE_ASSERT(inlineCallFrame->isClosureCall == !!data.calleeVariable); if (inlineCallFrame->isClosureCall) { ValueSource source = ValueSource::forFlushFormat( data.calleeVariable->machineLocal(), data.calleeVariable->flushFormat()); inlineCallFrame->calleeRecovery = source.valueRecovery(); } else RELEASE_ASSERT(inlineCallFrame->calleeRecovery.isConstant()); } if (symbolTable) { if (symbolTable->captureCount()) { unsigned captureStartLocal = allocation[ VirtualRegister(codeBlock()->symbolTable()->captureStart()).toLocal()]; ASSERT(captureStartLocal != UINT_MAX); m_graph.m_machineCaptureStart = virtualRegisterForLocal(captureStartLocal).offset(); } else m_graph.m_machineCaptureStart = virtualRegisterForLocal(0).offset(); // This is an abomination. If we had captured an argument then the argument ends // up being "slow", meaning that loads of the argument go through an extra lookup // table. if (const SlowArgument* slowArguments = symbolTable->slowArguments()) { auto newSlowArguments = std::make_unique<SlowArgument[]>( symbolTable->parameterCount()); for (size_t i = symbolTable->parameterCount(); i--;) { newSlowArguments[i] = slowArguments[i]; VirtualRegister reg = VirtualRegister(slowArguments[i].index); if (reg.isLocal()) newSlowArguments[i].index = virtualRegisterForLocal(allocation[reg.toLocal()]).offset(); } m_graph.m_slowArguments = std::move(newSlowArguments); } } // Fix GetLocalUnlinked's variable references. if (hasGetLocalUnlinked) { for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; for (unsigned nodeIndex = block->size(); nodeIndex--;) { Node* node = block->at(nodeIndex); switch (node->op()) { case GetLocalUnlinked: { VirtualRegister operand = node->unlinkedLocal(); if (operand.isLocal()) operand = virtualRegisterForLocal(allocation[operand.toLocal()]); node->setUnlinkedMachineLocal(operand); break; } default: break; } } } } return true; }
bool run() { // This enumerates the locals that we actually care about and packs them. So for example // if we use local 1, 3, 4, 5, 7, then we remap them: 1->0, 3->1, 4->2, 5->3, 7->4. We // treat a variable as being "used" if there exists an access to it (SetLocal, GetLocal, // Flush, PhantomLocal). BitVector usedLocals; // Collect those variables that are used from IR. bool hasNodesThatNeedFixup = false; for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; for (unsigned nodeIndex = block->size(); nodeIndex--;) { Node* node = block->at(nodeIndex); switch (node->op()) { case GetLocal: case SetLocal: case Flush: case PhantomLocal: { VariableAccessData* variable = node->variableAccessData(); if (variable->local().isArgument()) break; usedLocals.set(variable->local().toLocal()); break; } case GetLocalUnlinked: { VirtualRegister operand = node->unlinkedLocal(); if (operand.isArgument()) break; usedLocals.set(operand.toLocal()); hasNodesThatNeedFixup = true; break; } case LoadVarargs: case ForwardVarargs: { LoadVarargsData* data = node->loadVarargsData(); if (data->count.isLocal()) usedLocals.set(data->count.toLocal()); if (data->start.isLocal()) { // This part really relies on the contiguity of stack layout // assignments. ASSERT(VirtualRegister(data->start.offset() + data->limit - 1).isLocal()); for (unsigned i = data->limit; i--;) usedLocals.set(VirtualRegister(data->start.offset() + i).toLocal()); } // the else case shouldn't happen. hasNodesThatNeedFixup = true; break; } case PutStack: case GetStack: { StackAccessData* stack = node->stackAccessData(); if (stack->local.isArgument()) break; usedLocals.set(stack->local.toLocal()); break; } default: break; } } } for (InlineCallFrameSet::iterator iter = m_graph.m_plan.inlineCallFrames->begin(); !!iter; ++iter) { InlineCallFrame* inlineCallFrame = *iter; if (inlineCallFrame->isVarargs()) { usedLocals.set(VirtualRegister( JSStack::ArgumentCount + inlineCallFrame->stackOffset).toLocal()); } for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) { usedLocals.set(VirtualRegister( virtualRegisterForArgument(argument).offset() + inlineCallFrame->stackOffset).toLocal()); } } Vector<unsigned> allocation(usedLocals.size()); m_graph.m_nextMachineLocal = 0; for (unsigned i = 0; i < usedLocals.size(); ++i) { if (!usedLocals.get(i)) { allocation[i] = UINT_MAX; continue; } allocation[i] = m_graph.m_nextMachineLocal++; } for (unsigned i = m_graph.m_variableAccessData.size(); i--;) { VariableAccessData* variable = &m_graph.m_variableAccessData[i]; if (!variable->isRoot()) continue; if (variable->local().isArgument()) { variable->machineLocal() = variable->local(); continue; } size_t local = variable->local().toLocal(); if (local >= allocation.size()) continue; if (allocation[local] == UINT_MAX) continue; variable->machineLocal() = assign(allocation, variable->local()); } for (StackAccessData* data : m_graph.m_stackAccessData) { if (!data->local.isLocal()) { data->machineLocal = data->local; continue; } if (static_cast<size_t>(data->local.toLocal()) >= allocation.size()) continue; if (allocation[data->local.toLocal()] == UINT_MAX) continue; data->machineLocal = assign(allocation, data->local); } // This register is never valid for DFG code blocks. codeBlock()->setActivationRegister(VirtualRegister()); if (LIKELY(!m_graph.hasDebuggerEnabled())) codeBlock()->setScopeRegister(VirtualRegister()); else codeBlock()->setScopeRegister(assign(allocation, codeBlock()->scopeRegister())); for (unsigned i = m_graph.m_inlineVariableData.size(); i--;) { InlineVariableData data = m_graph.m_inlineVariableData[i]; InlineCallFrame* inlineCallFrame = data.inlineCallFrame; if (inlineCallFrame->isVarargs()) { inlineCallFrame->argumentCountRegister = assign( allocation, VirtualRegister(inlineCallFrame->stackOffset + JSStack::ArgumentCount)); } for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) { ArgumentPosition& position = m_graph.m_argumentPositions[ data.argumentPositionStart + argument]; VariableAccessData* variable = position.someVariable(); ValueSource source; if (!variable) source = ValueSource(SourceIsDead); else { source = ValueSource::forFlushFormat( variable->machineLocal(), variable->flushFormat()); } inlineCallFrame->arguments[argument] = source.valueRecovery(); } RELEASE_ASSERT(inlineCallFrame->isClosureCall == !!data.calleeVariable); if (inlineCallFrame->isClosureCall) { VariableAccessData* variable = data.calleeVariable->find(); ValueSource source = ValueSource::forFlushFormat( variable->machineLocal(), variable->flushFormat()); inlineCallFrame->calleeRecovery = source.valueRecovery(); } else RELEASE_ASSERT(inlineCallFrame->calleeRecovery.isConstant()); } // Fix GetLocalUnlinked's variable references. if (hasNodesThatNeedFixup) { for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; for (unsigned nodeIndex = block->size(); nodeIndex--;) { Node* node = block->at(nodeIndex); switch (node->op()) { case GetLocalUnlinked: { node->setUnlinkedMachineLocal(assign(allocation, node->unlinkedLocal())); break; } case LoadVarargs: case ForwardVarargs: { LoadVarargsData* data = node->loadVarargsData(); data->machineCount = assign(allocation, data->count); data->machineStart = assign(allocation, data->start); break; } default: break; } } } } return true; }
bool foldConstants(BasicBlock* block) { bool changed = false; m_state.beginBasicBlock(block); for (unsigned indexInBlock = 0; indexInBlock < block->size(); ++indexInBlock) { if (!m_state.isValid()) break; Node* node = block->at(indexInBlock); bool alreadyHandled = false; bool eliminated = false; switch (node->op()) { case BooleanToNumber: { if (node->child1().useKind() == UntypedUse && !m_interpreter.needsTypeCheck(node->child1(), SpecBoolean)) node->child1().setUseKind(BooleanUse); break; } case CheckArgumentsNotCreated: { if (!isEmptySpeculation( m_state.variables().operand( m_graph.argumentsRegisterFor(node->origin.semantic)).m_type)) break; node->convertToPhantom(); eliminated = true; break; } case CheckStructure: case ArrayifyToStructure: { AbstractValue& value = m_state.forNode(node->child1()); StructureSet set; if (node->op() == ArrayifyToStructure) set = node->structure(); else set = node->structureSet(); if (value.m_structure.isSubsetOf(set)) { m_interpreter.execute(indexInBlock); // Catch the fact that we may filter on cell. node->convertToPhantom(); eliminated = true; break; } break; } case CheckArray: case Arrayify: { if (!node->arrayMode().alreadyChecked(m_graph, node, m_state.forNode(node->child1()))) break; node->convertToPhantom(); eliminated = true; break; } case PutStructure: { if (m_state.forNode(node->child1()).m_structure.onlyStructure() != node->transition()->next) break; node->convertToPhantom(); eliminated = true; break; } case CheckFunction: { if (m_state.forNode(node->child1()).value() != node->function()->value()) break; node->convertToPhantom(); eliminated = true; break; } case CheckInBounds: { JSValue left = m_state.forNode(node->child1()).value(); JSValue right = m_state.forNode(node->child2()).value(); if (left && right && left.isInt32() && right.isInt32() && static_cast<uint32_t>(left.asInt32()) < static_cast<uint32_t>(right.asInt32())) { node->convertToPhantom(); eliminated = true; break; } break; } case MultiGetByOffset: { Edge baseEdge = node->child1(); Node* base = baseEdge.node(); MultiGetByOffsetData& data = node->multiGetByOffsetData(); // First prune the variants, then check if the MultiGetByOffset can be // strength-reduced to a GetByOffset. AbstractValue baseValue = m_state.forNode(base); m_interpreter.execute(indexInBlock); // Push CFA over this node after we get the state before. alreadyHandled = true; // Don't allow the default constant folder to do things to this. for (unsigned i = 0; i < data.variants.size(); ++i) { GetByIdVariant& variant = data.variants[i]; variant.structureSet().filter(baseValue); if (variant.structureSet().isEmpty()) { data.variants[i--] = data.variants.last(); data.variants.removeLast(); changed = true; } } if (data.variants.size() != 1) break; emitGetByOffset( indexInBlock, node, baseValue, data.variants[0], data.identifierNumber); changed = true; break; } case MultiPutByOffset: { Edge baseEdge = node->child1(); Node* base = baseEdge.node(); MultiPutByOffsetData& data = node->multiPutByOffsetData(); AbstractValue baseValue = m_state.forNode(base); m_interpreter.execute(indexInBlock); // Push CFA over this node after we get the state before. alreadyHandled = true; // Don't allow the default constant folder to do things to this. for (unsigned i = 0; i < data.variants.size(); ++i) { PutByIdVariant& variant = data.variants[i]; variant.oldStructure().filter(baseValue); if (variant.oldStructure().isEmpty()) { data.variants[i--] = data.variants.last(); data.variants.removeLast(); changed = true; continue; } if (variant.kind() == PutByIdVariant::Transition && variant.oldStructure().onlyStructure() == variant.newStructure()) { variant = PutByIdVariant::replace( variant.oldStructure(), variant.offset()); changed = true; } } if (data.variants.size() != 1) break; emitPutByOffset( indexInBlock, node, baseValue, data.variants[0], data.identifierNumber); changed = true; break; } case GetById: case GetByIdFlush: { Edge childEdge = node->child1(); Node* child = childEdge.node(); unsigned identifierNumber = node->identifierNumber(); AbstractValue baseValue = m_state.forNode(child); m_interpreter.execute(indexInBlock); // Push CFA over this node after we get the state before. alreadyHandled = true; // Don't allow the default constant folder to do things to this. if (baseValue.m_structure.isTop() || baseValue.m_structure.isClobbered() || (node->child1().useKind() == UntypedUse || (baseValue.m_type & ~SpecCell))) break; GetByIdStatus status = GetByIdStatus::computeFor( vm(), baseValue.m_structure.set(), m_graph.identifiers()[identifierNumber]); if (!status.isSimple()) break; for (unsigned i = status.numVariants(); i--;) { if (!status[i].constantChecks().isEmpty() || status[i].alternateBase()) { // FIXME: We could handle prototype cases. // https://bugs.webkit.org/show_bug.cgi?id=110386 break; } } if (status.numVariants() == 1) { emitGetByOffset(indexInBlock, node, baseValue, status[0], identifierNumber); changed = true; break; } if (!isFTL(m_graph.m_plan.mode)) break; MultiGetByOffsetData* data = m_graph.m_multiGetByOffsetData.add(); data->variants = status.variants(); data->identifierNumber = identifierNumber; node->convertToMultiGetByOffset(data); changed = true; break; } case PutById: case PutByIdDirect: case PutByIdFlush: { NodeOrigin origin = node->origin; Edge childEdge = node->child1(); Node* child = childEdge.node(); unsigned identifierNumber = node->identifierNumber(); ASSERT(childEdge.useKind() == CellUse); AbstractValue baseValue = m_state.forNode(child); m_interpreter.execute(indexInBlock); // Push CFA over this node after we get the state before. alreadyHandled = true; // Don't allow the default constant folder to do things to this. if (baseValue.m_structure.isTop() || baseValue.m_structure.isClobbered()) break; PutByIdStatus status = PutByIdStatus::computeFor( vm(), m_graph.globalObjectFor(origin.semantic), baseValue.m_structure.set(), m_graph.identifiers()[identifierNumber], node->op() == PutByIdDirect); if (!status.isSimple()) break; ASSERT(status.numVariants()); if (status.numVariants() > 1 && !isFTL(m_graph.m_plan.mode)) break; changed = true; for (unsigned i = status.numVariants(); i--;) addChecks(origin, indexInBlock, status[i].constantChecks()); if (status.numVariants() == 1) { emitPutByOffset(indexInBlock, node, baseValue, status[0], identifierNumber); break; } ASSERT(isFTL(m_graph.m_plan.mode)); MultiPutByOffsetData* data = m_graph.m_multiPutByOffsetData.add(); data->variants = status.variants(); data->identifierNumber = identifierNumber; node->convertToMultiPutByOffset(data); break; } case ToPrimitive: { if (m_state.forNode(node->child1()).m_type & ~(SpecFullNumber | SpecBoolean | SpecString)) break; node->convertToIdentity(); changed = true; break; } case GetMyArgumentByVal: { InlineCallFrame* inlineCallFrame = node->origin.semantic.inlineCallFrame; JSValue value = m_state.forNode(node->child1()).m_value; if (inlineCallFrame && value && value.isInt32()) { int32_t index = value.asInt32(); if (index >= 0 && static_cast<size_t>(index + 1) < inlineCallFrame->arguments.size()) { // Roll the interpreter over this. m_interpreter.execute(indexInBlock); eliminated = true; int operand = inlineCallFrame->stackOffset + m_graph.baselineCodeBlockFor(inlineCallFrame)->argumentIndexAfterCapture(index); m_insertionSet.insertNode( indexInBlock, SpecNone, CheckArgumentsNotCreated, node->origin); m_insertionSet.insertNode( indexInBlock, SpecNone, Phantom, node->origin, node->children); node->convertToGetLocalUnlinked(VirtualRegister(operand)); break; } } break; } case Check: { alreadyHandled = true; m_interpreter.execute(indexInBlock); for (unsigned i = 0; i < AdjacencyList::Size; ++i) { Edge edge = node->children.child(i); if (!edge) break; if (edge.isProved() || edge.willNotHaveCheck()) { node->children.removeEdge(i--); changed = true; } } break; } default: break; } if (eliminated) { changed = true; continue; } if (alreadyHandled) continue; m_interpreter.execute(indexInBlock); if (!m_state.isValid()) { // If we invalidated then we shouldn't attempt to constant-fold. Here's an // example: // // c: JSConstant(4.2) // x: ValueToInt32(Check:Int32:@const) // // It would be correct for an analysis to assume that execution cannot // proceed past @x. Therefore, constant-folding @x could be rather bad. But, // the CFA may report that it found a constant even though it also reported // that everything has been invalidated. This will only happen in a couple of // the constant folding cases; most of them are also separately defensive // about such things. break; } if (!node->shouldGenerate() || m_state.didClobber() || node->hasConstant()) continue; // Interesting fact: this freezing that we do right here may turn an fragile value into // a weak value. See DFGValueStrength.h. FrozenValue* value = m_graph.freeze(m_state.forNode(node).value()); if (!*value) continue; NodeOrigin origin = node->origin; AdjacencyList children = node->children; m_graph.convertToConstant(node, value); if (!children.isEmpty()) { m_insertionSet.insertNode( indexInBlock, SpecNone, Phantom, origin, children); } changed = true; } m_state.reset(); m_insertionSet.execute(block); return changed; }
bool run() { ASSERT(m_graph.m_form == ThreadedCPS || m_graph.m_form == SSA); ASSERT(m_graph.m_unificationState == GloballyUnified); ASSERT(m_graph.m_refCountState == EverythingIsLive); m_count = 0; if (m_verbose && !shouldDumpGraphAtEachPhase(m_graph.m_plan.mode)) { dataLog("Graph before CFA:\n"); m_graph.dump(); } // This implements a pseudo-worklist-based forward CFA, except that the visit order // of blocks is the bytecode program order (which is nearly topological), and // instead of a worklist we just walk all basic blocks checking if cfaShouldRevisit // is set to true. This is likely to balance the efficiency properties of both // worklist-based and forward fixpoint-based approaches. Like a worklist-based // approach, it won't visit code if it's meaningless to do so (nothing changed at // the head of the block or the predecessors have not been visited). Like a forward // fixpoint-based approach, it has a high probability of only visiting a block // after all predecessors have been visited. Only loops will cause this analysis to // revisit blocks, and the amount of revisiting is proportional to loop depth. m_state.initialize(); do { m_changed = false; performForwardCFA(); } while (m_changed); if (m_graph.m_form != SSA) { if (m_verbose) dataLog(" Widening state at OSR entry block.\n"); ASSERT(!m_changed); // Widen the abstract values at the block that serves as the must-handle OSR entry. for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; if (!block->isOSRTarget) continue; if (block->bytecodeBegin != m_graph.m_plan.osrEntryBytecodeIndex) continue; if (m_verbose) dataLog(" Found must-handle block: ", *block, "\n"); bool changed = false; for (size_t i = m_graph.m_plan.mustHandleValues.size(); i--;) { int operand = m_graph.m_plan.mustHandleValues.operandForIndex(i); JSValue value = m_graph.m_plan.mustHandleValues[i]; Node* node = block->variablesAtHead.operand(operand); if (!node) { if (m_verbose) dataLog(" Not live: ", VirtualRegister(operand), "\n"); continue; } if (m_verbose) dataLog(" Widening ", VirtualRegister(operand), " with ", value, "\n"); AbstractValue& target = block->valuesAtHead.operand(operand); changed |= target.mergeOSREntryValue(m_graph, value); target.fixTypeForRepresentation( m_graph, resultFor(node->variableAccessData()->flushFormat())); } if (changed || !block->cfaHasVisited) { m_changed = true; block->cfaShouldRevisit = true; } } // Propagate any of the changes we just introduced. while (m_changed) { m_changed = false; performForwardCFA(); } // Make sure we record the intersection of all proofs that we ever allowed the // compiler to rely upon. for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) { BasicBlock* block = m_graph.block(blockIndex); if (!block) continue; block->intersectionOfCFAHasVisited &= block->cfaHasVisited; for (unsigned i = block->intersectionOfPastValuesAtHead.size(); i--;) block->intersectionOfPastValuesAtHead[i].filter(block->valuesAtHead[i]); } } return true; }