void LocalOSRAvailabilityCalculator::executeNode(Node* node)
{
switch (node->op()) {
case SetLocal: {
VariableAccessData* variable = node->variableAccessData();
m_availability.operand(variable->local()) =
Availability(node->child1().node(), variable->flushedAt());
break;
}
case GetArgument: {
VariableAccessData* variable = node->variableAccessData();
m_availability.operand(variable->local()) =
Availability(node, variable->flushedAt());
break;
}
case MovHint: {
m_availability.operand(node->unlinkedLocal()) =
Availability(node->child1().node());
break;
}
case ZombieHint: {
m_availability.operand(node->unlinkedLocal()) =
Availability::unavailable();
break;
}
default:
break;
}
}
void LocalOSRAvailabilityCalculator::executeNode(Node* node)
{
switch (node->op()) {
case PutLocal: {
VariableAccessData* variable = node->variableAccessData();
m_availability.m_locals.operand(variable->local()).setFlush(variable->flushedAt());
break;
}
case KillLocal: {
m_availability.m_locals.operand(node->unlinkedLocal()).setFlush(FlushedAt(ConflictingFlush));
break;
}
case GetLocal: {
VariableAccessData* variable = node->variableAccessData();
m_availability.m_locals.operand(variable->local()) =
Availability(node, variable->flushedAt());
break;
}
case MovHint: {
m_availability.m_locals.operand(node->unlinkedLocal()).setNode(node->child1().node());
break;
}
case ZombieHint: {
m_availability.m_locals.operand(node->unlinkedLocal()).setNodeUnavailable();
break;
}
case LoadVarargs: {
LoadVarargsData* data = node->loadVarargsData();
m_availability.m_locals.operand(data->count) =
Availability(FlushedAt(FlushedInt32, data->machineCount));
for (unsigned i = data->limit; i--;) {
m_availability.m_locals.operand(VirtualRegister(data->start.offset() + i)) =
Availability(FlushedAt(FlushedJSValue, VirtualRegister(data->machineStart.offset() + i)));
}
break;
}
default:
break;
}
promoteHeapAccess(
node,
[&] (PromotedHeapLocation location, Edge value) {
m_availability.m_heap.set(location, Availability(value.node()));
},
[&] (PromotedHeapLocation) { });
}
void JITCompiler::noticeOSREntry(BasicBlock& basicBlock, JITCompiler::Label blockHead, LinkBuffer& linkBuffer)
{
// OSR entry is not allowed into blocks deemed unreachable by control flow analysis.
if (!basicBlock.intersectionOfCFAHasVisited)
return;
OSREntryData* entry = m_jitCode->appendOSREntryData(basicBlock.bytecodeBegin, linkBuffer.offsetOf(blockHead));
entry->m_expectedValues = basicBlock.intersectionOfPastValuesAtHead;
// Fix the expected values: in our protocol, a dead variable will have an expected
// value of (None, []). But the old JIT may stash some values there. So we really
// need (Top, TOP).
for (size_t argument = 0; argument < basicBlock.variablesAtHead.numberOfArguments(); ++argument) {
Node* node = basicBlock.variablesAtHead.argument(argument);
if (!node || !node->shouldGenerate())
entry->m_expectedValues.argument(argument).makeHeapTop();
}
for (size_t local = 0; local < basicBlock.variablesAtHead.numberOfLocals(); ++local) {
Node* node = basicBlock.variablesAtHead.local(local);
if (!node || !node->shouldGenerate())
entry->m_expectedValues.local(local).makeHeapTop();
else {
VariableAccessData* variable = node->variableAccessData();
entry->m_machineStackUsed.set(variable->machineLocal().toLocal());
switch (variable->flushFormat()) {
case FlushedDouble:
entry->m_localsForcedDouble.set(local);
break;
case FlushedInt52:
entry->m_localsForcedMachineInt.set(local);
break;
default:
break;
}
if (variable->local() != variable->machineLocal()) {
entry->m_reshufflings.append(
OSREntryReshuffling(
variable->local().offset(), variable->machineLocal().offset()));
}
}
}
entry->m_reshufflings.shrinkToFit();
}
void LocalOSRAvailabilityCalculator::executeNode(Node* node)
{
switch (node->op()) {
case SetLocal: {
VariableAccessData* variable = node->variableAccessData();
m_availability.m_locals.operand(variable->local()) =
Availability(node->child1().node(), variable->flushedAt());
break;
}
case GetArgument: {
VariableAccessData* variable = node->variableAccessData();
m_availability.m_locals.operand(variable->local()) =
Availability(node, variable->flushedAt());
break;
}
case MovHint: {
m_availability.m_locals.operand(node->unlinkedLocal()) =
Availability(node->child1().node());
break;
}
case ZombieHint: {
m_availability.m_locals.operand(node->unlinkedLocal()) =
Availability::unavailable();
break;
}
default:
break;
}
promoteHeapAccess(
node,
[&] (PromotedHeapLocation location, Edge value) {
m_availability.m_heap.set(location, Availability(value.node()));
},
[&] (PromotedHeapLocation) { });
}
void propagate(Node* node)
{
NodeFlags flags = node->flags() & NodeBytecodeBackPropMask;
switch (node->op()) {
case GetLocal: {
VariableAccessData* variableAccessData = node->variableAccessData();
flags &= ~NodeBytecodeUsesAsInt; // We don't care about cross-block uses-as-int.
m_changed |= variableAccessData->mergeFlags(flags);
break;
}
case SetLocal: {
VariableAccessData* variableAccessData = node->variableAccessData();
if (!variableAccessData->isLoadedFrom())
break;
flags = variableAccessData->flags();
RELEASE_ASSERT(!(flags & ~NodeBytecodeBackPropMask));
flags |= NodeBytecodeUsesAsNumber; // Account for the fact that control flow may cause overflows that our modeling can't handle.
node->child1()->mergeFlags(flags);
break;
}
case Flush: {
VariableAccessData* variableAccessData = node->variableAccessData();
m_changed |= variableAccessData->mergeFlags(NodeBytecodeUsesAsValue);
break;
}
case MovHint:
case Check:
break;
case BitAnd:
case BitOr:
case BitXor:
case BitRShift:
case BitLShift:
case BitURShift:
case ArithIMul: {
flags |= NodeBytecodeUsesAsInt;
flags &= ~(NodeBytecodeUsesAsNumber | NodeBytecodeNeedsNegZero | NodeBytecodeUsesAsOther);
flags &= ~NodeBytecodeUsesAsArrayIndex;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case StringCharCodeAt: {
node->child1()->mergeFlags(NodeBytecodeUsesAsValue);
node->child2()->mergeFlags(NodeBytecodeUsesAsValue | NodeBytecodeUsesAsInt | NodeBytecodeUsesAsArrayIndex);
break;
}
case UInt32ToNumber: {
node->child1()->mergeFlags(flags);
break;
}
case ValueAdd: {
if (isNotNegZero(node->child1().node()) || isNotNegZero(node->child2().node()))
flags &= ~NodeBytecodeNeedsNegZero;
if (node->child1()->hasNumberResult() || node->child2()->hasNumberResult())
flags &= ~NodeBytecodeUsesAsOther;
if (!isWithinPowerOfTwo<32>(node->child1()) && !isWithinPowerOfTwo<32>(node->child2()))
flags |= NodeBytecodeUsesAsNumber;
if (!m_allowNestedOverflowingAdditions)
flags |= NodeBytecodeUsesAsNumber;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithAdd: {
flags &= ~NodeBytecodeUsesAsOther;
if (isNotNegZero(node->child1().node()) || isNotNegZero(node->child2().node()))
flags &= ~NodeBytecodeNeedsNegZero;
if (!isWithinPowerOfTwo<32>(node->child1()) && !isWithinPowerOfTwo<32>(node->child2()))
flags |= NodeBytecodeUsesAsNumber;
if (!m_allowNestedOverflowingAdditions)
flags |= NodeBytecodeUsesAsNumber;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithClz32: {
flags &= ~(NodeBytecodeUsesAsNumber | NodeBytecodeNeedsNegZero | NodeBytecodeUsesAsOther | ~NodeBytecodeUsesAsArrayIndex);
flags |= NodeBytecodeUsesAsInt;
node->child1()->mergeFlags(flags);
break;
}
case ArithSub: {
flags &= ~NodeBytecodeUsesAsOther;
if (isNotNegZero(node->child1().node()) || isNotPosZero(node->child2().node()))
flags &= ~NodeBytecodeNeedsNegZero;
if (!isWithinPowerOfTwo<32>(node->child1()) && !isWithinPowerOfTwo<32>(node->child2()))
flags |= NodeBytecodeUsesAsNumber;
if (!m_allowNestedOverflowingAdditions)
flags |= NodeBytecodeUsesAsNumber;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithNegate: {
flags &= ~NodeBytecodeUsesAsOther;
node->child1()->mergeFlags(flags);
break;
}
case ArithMul: {
// As soon as a multiply happens, we can easily end up in the part
// of the double domain where the point at which you do truncation
// can change the outcome. So, ArithMul always forces its inputs to
// check for overflow. Additionally, it will have to check for overflow
// itself unless we can prove that there is no way for the values
// produced to cause double rounding.
if (!isWithinPowerOfTwo<22>(node->child1().node())
&& !isWithinPowerOfTwo<22>(node->child2().node()))
flags |= NodeBytecodeUsesAsNumber;
node->mergeFlags(flags);
flags |= NodeBytecodeUsesAsNumber | NodeBytecodeNeedsNegZero;
flags &= ~NodeBytecodeUsesAsOther;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithDiv: {
flags |= NodeBytecodeUsesAsNumber | NodeBytecodeNeedsNegZero;
flags &= ~NodeBytecodeUsesAsOther;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithMod: {
flags |= NodeBytecodeUsesAsNumber;
flags &= ~NodeBytecodeUsesAsOther;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags & ~NodeBytecodeNeedsNegZero);
break;
}
case GetByVal: {
node->child1()->mergeFlags(NodeBytecodeUsesAsValue);
node->child2()->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther | NodeBytecodeUsesAsInt | NodeBytecodeUsesAsArrayIndex);
break;
}
case NewArrayWithSize: {
node->child1()->mergeFlags(NodeBytecodeUsesAsValue | NodeBytecodeUsesAsInt | NodeBytecodeUsesAsArrayIndex);
break;
}
case NewTypedArray: {
// Negative zero is not observable. NaN versus undefined are only observable
// in that you would get a different exception message. So, like, whatever: we
// claim here that NaN v. undefined is observable.
node->child1()->mergeFlags(NodeBytecodeUsesAsInt | NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther | NodeBytecodeUsesAsArrayIndex);
break;
}
case StringCharAt: {
node->child1()->mergeFlags(NodeBytecodeUsesAsValue);
node->child2()->mergeFlags(NodeBytecodeUsesAsValue | NodeBytecodeUsesAsInt | NodeBytecodeUsesAsArrayIndex);
break;
}
case ToString:
case CallStringConstructor: {
node->child1()->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther);
break;
}
case ToPrimitive:
case ToNumber: {
node->child1()->mergeFlags(flags);
break;
}
case PutByValDirect:
case PutByVal: {
m_graph.varArgChild(node, 0)->mergeFlags(NodeBytecodeUsesAsValue);
m_graph.varArgChild(node, 1)->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther | NodeBytecodeUsesAsInt | NodeBytecodeUsesAsArrayIndex);
m_graph.varArgChild(node, 2)->mergeFlags(NodeBytecodeUsesAsValue);
break;
}
case Switch: {
SwitchData* data = node->switchData();
switch (data->kind) {
case SwitchImm:
// We don't need NodeBytecodeNeedsNegZero because if the cases are all integers
// then -0 and 0 are treated the same. We don't need NodeBytecodeUsesAsOther
// because if all of the cases are integers then NaN and undefined are
// treated the same (i.e. they will take default).
node->child1()->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsInt);
break;
case SwitchChar: {
// We don't need NodeBytecodeNeedsNegZero because if the cases are all strings
// then -0 and 0 are treated the same. We don't need NodeBytecodeUsesAsOther
// because if all of the cases are single-character strings then NaN
// and undefined are treated the same (i.e. they will take default).
node->child1()->mergeFlags(NodeBytecodeUsesAsNumber);
break;
}
case SwitchString:
// We don't need NodeBytecodeNeedsNegZero because if the cases are all strings
// then -0 and 0 are treated the same.
node->child1()->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther);
break;
case SwitchCell:
// There is currently no point to being clever here since this is used for switching
// on objects.
mergeDefaultFlags(node);
break;
}
break;
}
case Identity:
// This would be trivial to handle but we just assert that we cannot see these yet.
RELEASE_ASSERT_NOT_REACHED();
break;
// Note: ArithSqrt, ArithUnary and other math intrinsics don't have special
// rules in here because they are always followed by Phantoms to signify that if the
// method call speculation fails, the bytecode may use the arguments in arbitrary ways.
// This corresponds to that possibility of someone doing something like:
// Math.sin = function(x) { doArbitraryThingsTo(x); }
default:
mergeDefaultFlags(node);
break;
}
}
bool run()
{
RELEASE_ASSERT(m_graph.m_plan.mode == FTLForOSREntryMode);
RELEASE_ASSERT(m_graph.m_form == ThreadedCPS);
unsigned bytecodeIndex = m_graph.m_plan.osrEntryBytecodeIndex;
RELEASE_ASSERT(bytecodeIndex);
RELEASE_ASSERT(bytecodeIndex != UINT_MAX);
// Needed by createPreHeader().
m_graph.ensureDominators();
CodeBlock* baseline = m_graph.m_profiledBlock;
BasicBlock* target = 0;
for (unsigned blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
unsigned nodeIndex = 0;
Node* firstNode = block->at(0);
while (firstNode->isSemanticallySkippable())
firstNode = block->at(++nodeIndex);
if (firstNode->op() == LoopHint
&& firstNode->origin.semantic == CodeOrigin(bytecodeIndex)) {
target = block;
break;
}
}
if (!target) {
// This is a terrible outcome. It shouldn't often happen but it might
// happen and so we should defend against it. If it happens, then this
// compilation is a failure.
return false;
}
BlockInsertionSet insertionSet(m_graph);
// We say that the execution count of the entry block is 1, because we know for sure
// that this must be the case. Under our definition of executionCount, "1" means "once
// per invocation". We could have said NaN here, since that would ask any clients of
// executionCount to use best judgement - but that seems unnecessary since we know for
// sure what the executionCount should be in this case.
BasicBlock* newRoot = insertionSet.insert(0, 1);
// We'd really like to use an unset origin, but ThreadedCPS won't allow that.
NodeOrigin origin = NodeOrigin(CodeOrigin(0), CodeOrigin(0), false);
Vector<Node*> locals(baseline->m_numCalleeLocals);
for (int local = 0; local < baseline->m_numCalleeLocals; ++local) {
Node* previousHead = target->variablesAtHead.local(local);
if (!previousHead)
continue;
VariableAccessData* variable = previousHead->variableAccessData();
locals[local] = newRoot->appendNode(
m_graph, variable->prediction(), ExtractOSREntryLocal, origin,
OpInfo(variable->local().offset()));
newRoot->appendNode(
m_graph, SpecNone, MovHint, origin, OpInfo(variable->local().offset()),
Edge(locals[local]));
}
// Now use the origin of the target, since it's not OK to exit, and we will probably hoist
// type checks to here.
origin = target->at(0)->origin;
for (int argument = 0; argument < baseline->numParameters(); ++argument) {
Node* oldNode = target->variablesAtHead.argument(argument);
if (!oldNode) {
// Just for sanity, always have a SetArgument even if it's not needed.
oldNode = m_graph.m_arguments[argument];
}
Node* node = newRoot->appendNode(
m_graph, SpecNone, SetArgument, origin,
OpInfo(oldNode->variableAccessData()));
m_graph.m_arguments[argument] = node;
}
for (int local = 0; local < baseline->m_numCalleeLocals; ++local) {
Node* previousHead = target->variablesAtHead.local(local);
if (!previousHead)
continue;
VariableAccessData* variable = previousHead->variableAccessData();
Node* node = locals[local];
newRoot->appendNode(
m_graph, SpecNone, SetLocal, origin, OpInfo(variable), Edge(node));
}
newRoot->appendNode(
m_graph, SpecNone, Jump, origin,
OpInfo(createPreHeader(m_graph, insertionSet, target)));
insertionSet.execute();
m_graph.resetReachability();
m_graph.killUnreachableBlocks();
return true;
}
bool run()
{
for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
VariableAccessData* variable = &m_graph.m_variableAccessData[i];
if (!variable->isRoot())
continue;
variable->clearVotes();
}
// Identify the set of variables that are always subject to the same structure
// checks. For now, only consider monomorphic structure checks (one structure).
for (BlockIndex blockIndex = 0; blockIndex < m_graph.m_blocks.size(); ++blockIndex) {
BasicBlock* block = m_graph.m_blocks[blockIndex].get();
if (!block)
continue;
for (unsigned indexInBlock = 0; indexInBlock < block->size(); ++indexInBlock) {
NodeIndex nodeIndex = block->at(indexInBlock);
Node& node = m_graph[nodeIndex];
if (!node.shouldGenerate())
continue;
switch (node.op()) {
case CheckStructure: {
Node& child = m_graph[node.child1()];
if (child.op() != GetLocal)
break;
VariableAccessData* variable = child.variableAccessData();
variable->vote(VoteStructureCheck);
if (variable->isCaptured() || variable->structureCheckHoistingFailed())
break;
if (!isCellSpeculation(variable->prediction()))
break;
noticeStructureCheck(variable, node.structureSet());
break;
}
case ForwardCheckStructure:
case ForwardStructureTransitionWatchpoint:
// We currently rely on the fact that we're the only ones who would
// insert this node.
ASSERT_NOT_REACHED();
break;
case GetByOffset:
case PutByOffset:
case PutStructure:
case StructureTransitionWatchpoint:
case AllocatePropertyStorage:
case ReallocatePropertyStorage:
case GetPropertyStorage:
case GetByVal:
case PutByVal:
case PutByValAlias:
case GetArrayLength:
case CheckArray:
case GetIndexedPropertyStorage:
case Phantom:
// Don't count these uses.
break;
default:
m_graph.vote(node, VoteOther);
break;
}
}
}
// Disable structure hoisting on variables that appear to mostly be used in
// contexts where it doesn't make sense.
for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
VariableAccessData* variable = &m_graph.m_variableAccessData[i];
if (!variable->isRoot())
continue;
if (variable->voteRatio() >= Options::structureCheckVoteRatioForHoisting())
continue;
HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
if (iter == m_map.end())
continue;
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog("Zeroing the structure to hoist for %s because the ratio is %lf.\n",
m_graph.nameOfVariableAccessData(variable), variable->voteRatio());
#endif
iter->second.m_structure = 0;
}
// Identify the set of variables that are live across a structure clobber.
Operands<VariableAccessData*> live(
m_graph.m_blocks[0]->variablesAtTail.numberOfArguments(),
m_graph.m_blocks[0]->variablesAtTail.numberOfLocals());
for (BlockIndex blockIndex = 0; blockIndex < m_graph.m_blocks.size(); ++blockIndex) {
BasicBlock* block = m_graph.m_blocks[blockIndex].get();
if (!block)
continue;
ASSERT(live.numberOfArguments() == block->variablesAtTail.numberOfArguments());
ASSERT(live.numberOfLocals() == block->variablesAtTail.numberOfLocals());
for (unsigned i = live.size(); i--;) {
NodeIndex indexAtTail = block->variablesAtTail[i];
VariableAccessData* variable;
if (indexAtTail == NoNode)
variable = 0;
else
variable = m_graph[indexAtTail].variableAccessData();
live[i] = variable;
}
for (unsigned indexInBlock = block->size(); indexInBlock--;) {
NodeIndex nodeIndex = block->at(indexInBlock);
Node& node = m_graph[nodeIndex];
if (!node.shouldGenerate())
continue;
switch (node.op()) {
case GetLocal:
case Flush:
// This is a birth.
live.operand(node.local()) = node.variableAccessData();
break;
case SetLocal:
case SetArgument:
ASSERT(live.operand(node.local())); // Must be live.
ASSERT(live.operand(node.local()) == node.variableAccessData()); // Must have the variable we expected.
// This is a death.
live.operand(node.local()) = 0;
break;
// Use the CFA's notion of what clobbers the world.
case ValueAdd:
if (m_graph.addShouldSpeculateInteger(node))
break;
if (Node::shouldSpeculateNumber(m_graph[node.child1()], m_graph[node.child2()]))
break;
clobber(live);
break;
case CompareLess:
case CompareLessEq:
case CompareGreater:
case CompareGreaterEq:
case CompareEq: {
Node& left = m_graph[node.child1()];
Node& right = m_graph[node.child2()];
if (Node::shouldSpeculateInteger(left, right))
break;
if (Node::shouldSpeculateNumber(left, right))
break;
if (node.op() == CompareEq) {
if ((m_graph.isConstant(node.child1().index())
&& m_graph.valueOfJSConstant(node.child1().index()).isNull())
|| (m_graph.isConstant(node.child2().index())
&& m_graph.valueOfJSConstant(node.child2().index()).isNull()))
break;
if (Node::shouldSpeculateFinalObject(left, right))
break;
if (Node::shouldSpeculateArray(left, right))
break;
if (left.shouldSpeculateFinalObject() && right.shouldSpeculateFinalObjectOrOther())
break;
if (right.shouldSpeculateFinalObject() && left.shouldSpeculateFinalObjectOrOther())
break;
if (left.shouldSpeculateArray() && right.shouldSpeculateArrayOrOther())
break;
if (right.shouldSpeculateArray() && left.shouldSpeculateArrayOrOther())
break;
}
clobber(live);
break;
}
case GetByVal:
case PutByVal:
case PutByValAlias:
if (m_graph.byValIsPure(node))
break;
clobber(live);
break;
case GetMyArgumentsLengthSafe:
case GetMyArgumentByValSafe:
case GetById:
case GetByIdFlush:
case PutStructure:
case PhantomPutStructure:
case PutById:
case PutByIdDirect:
case Call:
case Construct:
case Resolve:
case ResolveBase:
case ResolveBaseStrictPut:
case ResolveGlobal:
clobber(live);
break;
default:
ASSERT(node.op() != Phi);
break;
}
}
}
bool changed = false;
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
for (HashMap<VariableAccessData*, CheckData>::iterator it = m_map.begin();
it != m_map.end(); ++it) {
if (!it->second.m_structure) {
dataLog("Not hoisting checks for %s because of heuristics.\n", m_graph.nameOfVariableAccessData(it->first));
continue;
}
if (it->second.m_isClobbered && !it->second.m_structure->transitionWatchpointSetIsStillValid()) {
dataLog("Not hoisting checks for %s because the structure is clobbered and has an invalid watchpoint set.\n", m_graph.nameOfVariableAccessData(it->first));
continue;
}
dataLog("Hoisting checks for %s\n", m_graph.nameOfVariableAccessData(it->first));
}
#endif // DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
// Make changes:
// 1) If a variable's live range does not span a clobber, then inject structure
// checks before the SetLocal.
// 2) If a variable's live range spans a clobber but is watchpointable, then
// inject structure checks before the SetLocal and replace all other structure
// checks on that variable with structure transition watchpoints.
InsertionSet<NodeIndex> insertionSet;
for (BlockIndex blockIndex = 0; blockIndex < m_graph.m_blocks.size(); ++blockIndex) {
BasicBlock* block = m_graph.m_blocks[blockIndex].get();
if (!block)
continue;
for (unsigned indexInBlock = 0; indexInBlock < block->size(); ++indexInBlock) {
NodeIndex nodeIndex = block->at(indexInBlock);
Node& node = m_graph[nodeIndex];
// Be careful not to use 'node' after appending to the graph. In those switch
// cases where we need to append, we first carefully extract everything we need
// from the node, before doing any appending.
if (!node.shouldGenerate())
continue;
switch (node.op()) {
case SetArgument: {
ASSERT(!blockIndex);
// Insert a GetLocal and a CheckStructure immediately following this
// SetArgument, if the variable was a candidate for structure hoisting.
// If the basic block previously only had the SetArgument as its
// variable-at-tail, then replace it with this GetLocal.
VariableAccessData* variable = node.variableAccessData();
HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
if (iter == m_map.end())
break;
if (!iter->second.m_structure)
break;
if (iter->second.m_isClobbered && !iter->second.m_structure->transitionWatchpointSetIsStillValid())
break;
node.ref();
CodeOrigin codeOrigin = node.codeOrigin;
Node getLocal(GetLocal, codeOrigin, OpInfo(variable), nodeIndex);
getLocal.predict(variable->prediction());
getLocal.ref();
NodeIndex getLocalIndex = m_graph.size();
m_graph.append(getLocal);
insertionSet.append(indexInBlock + 1, getLocalIndex);
Node checkStructure(CheckStructure, codeOrigin, OpInfo(m_graph.addStructureSet(iter->second.m_structure)), getLocalIndex);
checkStructure.ref();
NodeIndex checkStructureIndex = m_graph.size();
m_graph.append(checkStructure);
insertionSet.append(indexInBlock + 1, checkStructureIndex);
if (block->variablesAtTail.operand(variable->local()) == nodeIndex)
block->variablesAtTail.operand(variable->local()) = getLocalIndex;
m_graph.substituteGetLocal(*block, indexInBlock, variable, getLocalIndex);
changed = true;
break;
}
case SetLocal: {
VariableAccessData* variable = node.variableAccessData();
HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
if (iter == m_map.end())
break;
if (!iter->second.m_structure)
break;
if (iter->second.m_isClobbered && !iter->second.m_structure->transitionWatchpointSetIsStillValid())
break;
// First insert a dead SetLocal to tell OSR that the child's value should
// be dropped into this bytecode variable if the CheckStructure decides
// to exit.
CodeOrigin codeOrigin = node.codeOrigin;
NodeIndex child1 = node.child1().index();
Node setLocal(SetLocal, codeOrigin, OpInfo(variable), child1);
NodeIndex setLocalIndex = m_graph.size();
m_graph.append(setLocal);
insertionSet.append(indexInBlock, setLocalIndex);
m_graph[child1].ref();
// Use a ForwardCheckStructure to indicate that we should exit to the
// next bytecode instruction rather than reexecuting the current one.
Node checkStructure(ForwardCheckStructure, codeOrigin, OpInfo(m_graph.addStructureSet(iter->second.m_structure)), child1);
checkStructure.ref();
NodeIndex checkStructureIndex = m_graph.size();
m_graph.append(checkStructure);
insertionSet.append(indexInBlock, checkStructureIndex);
changed = true;
break;
}
case CheckStructure: {
Node& child = m_graph[node.child1()];
if (child.op() != GetLocal)
break;
HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(child.variableAccessData());
if (iter == m_map.end())
break;
if (!iter->second.m_structure)
break;
if (!iter->second.m_isClobbered) {
node.setOpAndDefaultFlags(Phantom);
ASSERT(node.refCount() == 1);
break;
}
if (!iter->second.m_structure->transitionWatchpointSetIsStillValid())
break;
ASSERT(iter->second.m_structure == node.structureSet().singletonStructure());
node.convertToStructureTransitionWatchpoint();
changed = true;
break;
}
default:
break;
}
}
insertionSet.execute(*block);
}
return changed;
}
void Graph::dump(const char* prefix, NodeIndex nodeIndex)
{
Node& node = at(nodeIndex);
NodeType op = node.op();
unsigned refCount = node.refCount();
bool skipped = !refCount;
bool mustGenerate = node.mustGenerate();
if (mustGenerate)
--refCount;
dataLog("%s", prefix);
printNodeWhiteSpace(node);
// Example/explanation of dataflow dump output
//
// 14: <!2:7> GetByVal(@3, @13)
// ^1 ^2 ^3 ^4 ^5
//
// (1) The nodeIndex of this operation.
// (2) The reference count. The number printed is the 'real' count,
// not including the 'mustGenerate' ref. If the node is
// 'mustGenerate' then the count it prefixed with '!'.
// (3) The virtual register slot assigned to this node.
// (4) The name of the operation.
// (5) The arguments to the operation. The may be of the form:
// @# - a NodeIndex referencing a prior node in the graph.
// arg# - an argument number.
// $# - the index in the CodeBlock of a constant { for numeric constants the value is displayed | for integers, in both decimal and hex }.
// id# - the index in the CodeBlock of an identifier { if codeBlock is passed to dump(), the string representation is displayed }.
// var# - the index of a var on the global object, used by GetGlobalVar/PutGlobalVar operations.
dataLog("% 4d:%s<%c%u:", (int)nodeIndex, skipped ? " skipped " : " ", mustGenerate ? '!' : ' ', refCount);
if (node.hasResult() && !skipped && node.hasVirtualRegister())
dataLog("%u", node.virtualRegister());
else
dataLog("-");
dataLog(">\t%s(", opName(op));
bool hasPrinted = false;
if (node.flags() & NodeHasVarArgs) {
for (unsigned childIdx = node.firstChild(); childIdx < node.firstChild() + node.numChildren(); childIdx++) {
if (hasPrinted)
dataLog(", ");
else
hasPrinted = true;
dataLog("%s@%u%s",
useKindToString(m_varArgChildren[childIdx].useKind()),
m_varArgChildren[childIdx].index(),
speculationToAbbreviatedString(
at(m_varArgChildren[childIdx]).prediction()));
}
} else {
if (!!node.child1()) {
dataLog("%s@%u%s",
useKindToString(node.child1().useKind()),
node.child1().index(),
speculationToAbbreviatedString(at(node.child1()).prediction()));
}
if (!!node.child2()) {
dataLog(", %s@%u%s",
useKindToString(node.child2().useKind()),
node.child2().index(),
speculationToAbbreviatedString(at(node.child2()).prediction()));
}
if (!!node.child3()) {
dataLog(", %s@%u%s",
useKindToString(node.child3().useKind()),
node.child3().index(),
speculationToAbbreviatedString(at(node.child3()).prediction()));
}
hasPrinted = !!node.child1();
}
if (strlen(nodeFlagsAsString(node.flags()))) {
dataLog("%s%s", hasPrinted ? ", " : "", nodeFlagsAsString(node.flags()));
hasPrinted = true;
}
if (node.hasArrayMode()) {
dataLog("%s%s", hasPrinted ? ", " : "", modeToString(node.arrayMode()));
hasPrinted = true;
}
if (node.hasVarNumber()) {
dataLog("%svar%u", hasPrinted ? ", " : "", node.varNumber());
hasPrinted = true;
}
if (node.hasRegisterPointer()) {
dataLog(
"%sglobal%u(%p)", hasPrinted ? ", " : "",
globalObjectFor(node.codeOrigin)->findRegisterIndex(node.registerPointer()),
node.registerPointer());
hasPrinted = true;
}
if (node.hasIdentifier()) {
dataLog("%sid%u{%s}", hasPrinted ? ", " : "", node.identifierNumber(), m_codeBlock->identifier(node.identifierNumber()).ustring().utf8().data());
hasPrinted = true;
}
if (node.hasStructureSet()) {
for (size_t i = 0; i < node.structureSet().size(); ++i) {
dataLog("%sstruct(%p)", hasPrinted ? ", " : "", node.structureSet()[i]);
hasPrinted = true;
}
}
if (node.hasStructure()) {
dataLog("%sstruct(%p)", hasPrinted ? ", " : "", node.structure());
hasPrinted = true;
}
if (node.hasStructureTransitionData()) {
dataLog("%sstruct(%p -> %p)", hasPrinted ? ", " : "", node.structureTransitionData().previousStructure, node.structureTransitionData().newStructure);
hasPrinted = true;
}
if (node.hasStorageAccessData()) {
StorageAccessData& storageAccessData = m_storageAccessData[node.storageAccessDataIndex()];
dataLog("%sid%u{%s}", hasPrinted ? ", " : "", storageAccessData.identifierNumber, m_codeBlock->identifier(storageAccessData.identifierNumber).ustring().utf8().data());
dataLog(", %lu", static_cast<unsigned long>(storageAccessData.offset));
hasPrinted = true;
}
ASSERT(node.hasVariableAccessData() == node.hasLocal());
if (node.hasVariableAccessData()) {
VariableAccessData* variableAccessData = node.variableAccessData();
int operand = variableAccessData->operand();
if (operandIsArgument(operand))
dataLog("%sarg%u(%s)", hasPrinted ? ", " : "", operandToArgument(operand), nameOfVariableAccessData(variableAccessData));
else
dataLog("%sr%u(%s)", hasPrinted ? ", " : "", operand, nameOfVariableAccessData(variableAccessData));
hasPrinted = true;
}
if (node.hasConstantBuffer()) {
if (hasPrinted)
dataLog(", ");
dataLog("%u:[", node.startConstant());
for (unsigned i = 0; i < node.numConstants(); ++i) {
if (i)
dataLog(", ");
dataLog("%s", m_codeBlock->constantBuffer(node.startConstant())[i].description());
}
dataLog("]");
hasPrinted = true;
}
if (op == JSConstant) {
dataLog("%s$%u", hasPrinted ? ", " : "", node.constantNumber());
JSValue value = valueOfJSConstant(nodeIndex);
dataLog(" = %s", value.description());
hasPrinted = true;
}
if (op == WeakJSConstant) {
dataLog("%s%p", hasPrinted ? ", " : "", node.weakConstant());
hasPrinted = true;
}
if (node.isBranch() || node.isJump()) {
dataLog("%sT:#%u", hasPrinted ? ", " : "", node.takenBlockIndex());
hasPrinted = true;
}
if (node.isBranch()) {
dataLog("%sF:#%u", hasPrinted ? ", " : "", node.notTakenBlockIndex());
hasPrinted = true;
}
dataLog("%sbc#%u", hasPrinted ? ", " : "", node.codeOrigin.bytecodeIndex);
hasPrinted = true;
(void)hasPrinted;
dataLog(")");
if (!skipped) {
if (node.hasVariableAccessData())
dataLog(" predicting %s%s", speculationToString(node.variableAccessData()->prediction()), node.variableAccessData()->shouldUseDoubleFormat() ? ", forcing double" : "");
else if (node.hasHeapPrediction())
dataLog(" predicting %s", speculationToString(node.getHeapPrediction()));
}
dataLog("\n");
}
bool run()
{
ASSERT(m_graph.m_form == SSA);
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
block->ssa->availabilityAtHead.fill(Availability());
block->ssa->availabilityAtTail.fill(Availability());
}
BasicBlock* root = m_graph.block(0);
for (unsigned argument = root->ssa->availabilityAtHead.numberOfArguments(); argument--;) {
root->ssa->availabilityAtHead.argument(argument) =
Availability::unavailable().withFlush(
FlushedAt(FlushedJSValue, virtualRegisterForArgument(argument)));
}
for (unsigned local = root->ssa->availabilityAtHead.numberOfLocals(); local--;)
root->ssa->availabilityAtHead.local(local) = Availability::unavailable();
if (m_graph.m_plan.mode == FTLForOSREntryMode) {
for (unsigned local = m_graph.m_profiledBlock->m_numCalleeRegisters; local--;) {
root->ssa->availabilityAtHead.local(local) =
Availability::unavailable().withFlush(
FlushedAt(FlushedJSValue, virtualRegisterForLocal(local)));
}
}
// This could be made more efficient by processing blocks in reverse postorder.
Operands<Availability> availability;
bool changed;
do {
changed = false;
for (BlockIndex blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
availability = block->ssa->availabilityAtHead;
for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case SetLocal: {
VariableAccessData* variable = node->variableAccessData();
availability.operand(variable->local()) =
Availability(node->child1().node(), variable->flushedAt());
break;
}
case GetArgument: {
VariableAccessData* variable = node->variableAccessData();
availability.operand(variable->local()) =
Availability(node, variable->flushedAt());
break;
}
case MovHint:
case MovHintAndCheck: {
VariableAccessData* variable = node->variableAccessData();
availability.operand(variable->local()) =
Availability(node->child1().node());
break;
}
case ZombieHint: {
VariableAccessData* variable = node->variableAccessData();
availability.operand(variable->local()) = Availability::unavailable();
break;
}
default:
break;
}
}
if (availability == block->ssa->availabilityAtTail)
continue;
block->ssa->availabilityAtTail = availability;
changed = true;
for (unsigned successorIndex = block->numSuccessors(); successorIndex--;) {
BasicBlock* successor = block->successor(successorIndex);
for (unsigned i = availability.size(); i--;) {
successor->ssa->availabilityAtHead[i] = availability[i].merge(
successor->ssa->availabilityAtHead[i]);
}
}
}
} while (changed);
return true;
}
bool run()
{
ASSERT(m_graph.m_form == ThreadedCPS);
for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
VariableAccessData* variable = &m_graph.m_variableAccessData[i];
if (!variable->isRoot())
continue;
variable->clearVotes();
}
// Identify the set of variables that are always subject to the same structure
// checks. For now, only consider monomorphic structure checks (one structure).
for (BlockIndex blockIndex = 0; blockIndex < m_graph.m_blocks.size(); ++blockIndex) {
BasicBlock* block = m_graph.m_blocks[blockIndex].get();
if (!block)
continue;
for (unsigned indexInBlock = 0; indexInBlock < block->size(); ++indexInBlock) {
Node* node = block->at(indexInBlock);
switch (node->op()) {
case CheckStructure:
case StructureTransitionWatchpoint: {
Node* child = node->child1().node();
if (child->op() != GetLocal)
break;
VariableAccessData* variable = child->variableAccessData();
variable->vote(VoteStructureCheck);
if (!shouldConsiderForHoisting(variable))
break;
noticeStructureCheck(variable, node->structureSet());
break;
}
case ForwardCheckStructure:
case ForwardStructureTransitionWatchpoint:
// We currently rely on the fact that we're the only ones who would
// insert this node.
RELEASE_ASSERT_NOT_REACHED();
break;
case GetByOffset:
case PutByOffset:
case PutStructure:
case AllocatePropertyStorage:
case ReallocatePropertyStorage:
case GetButterfly:
case GetByVal:
case PutByVal:
case PutByValAlias:
case GetArrayLength:
case CheckArray:
case GetIndexedPropertyStorage:
case Phantom:
// Don't count these uses.
break;
case ArrayifyToStructure:
case Arrayify:
if (node->arrayMode().conversion() == Array::RageConvert) {
// Rage conversion changes structures. We should avoid tying to do
// any kind of hoisting when rage conversion is in play.
Node* child = node->child1().node();
if (child->op() != GetLocal)
break;
VariableAccessData* variable = child->variableAccessData();
variable->vote(VoteOther);
if (!shouldConsiderForHoisting(variable))
break;
noticeStructureCheck(variable, 0);
}
break;
case SetLocal: {
// Find all uses of the source of the SetLocal. If any of them are a
// kind of CheckStructure, then we should notice them to ensure that
// we're not hoisting a check that would contravene checks that are
// already being performed.
VariableAccessData* variable = node->variableAccessData();
if (!shouldConsiderForHoisting(variable))
break;
Node* source = node->child1().node();
for (unsigned subIndexInBlock = 0; subIndexInBlock < block->size(); ++subIndexInBlock) {
Node* subNode = block->at(subIndexInBlock);
switch (subNode->op()) {
case CheckStructure: {
if (subNode->child1() != source)
break;
noticeStructureCheck(variable, subNode->structureSet());
break;
}
case StructureTransitionWatchpoint: {
if (subNode->child1() != source)
break;
noticeStructureCheck(variable, subNode->structure());
break;
}
default:
break;
}
}
m_graph.voteChildren(node, VoteOther);
break;
}
case GarbageValue:
break;
default:
m_graph.voteChildren(node, VoteOther);
break;
}
}
}
// Disable structure hoisting on variables that appear to mostly be used in
// contexts where it doesn't make sense.
for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
VariableAccessData* variable = &m_graph.m_variableAccessData[i];
if (!variable->isRoot())
continue;
if (variable->voteRatio() >= Options::structureCheckVoteRatioForHoisting())
continue;
HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
if (iter == m_map.end())
continue;
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog(
"Zeroing the structure to hoist for ", VariableAccessDataDump(m_graph, variable),
" because the ratio is ", variable->voteRatio(), ".\n");
#endif
iter->value.m_structure = 0;
}
// Disable structure check hoisting for variables that cross the OSR entry that
// we're currently taking, and where the value currently does not have the
// structure we want.
for (BlockIndex blockIndex = 0; blockIndex < m_graph.m_blocks.size(); ++blockIndex) {
BasicBlock* block = m_graph.m_blocks[blockIndex].get();
if (!block)
continue;
ASSERT(block->isReachable);
if (!block->isOSRTarget)
continue;
if (block->bytecodeBegin != m_graph.m_osrEntryBytecodeIndex)
continue;
for (size_t i = 0; i < m_graph.m_mustHandleValues.size(); ++i) {
int operand = m_graph.m_mustHandleValues.operandForIndex(i);
Node* node = block->variablesAtHead.operand(operand);
if (!node)
continue;
VariableAccessData* variable = node->variableAccessData();
HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
if (iter == m_map.end())
continue;
if (!iter->value.m_structure)
continue;
JSValue value = m_graph.m_mustHandleValues[i];
if (!value || !value.isCell()) {
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog(
"Zeroing the structure to hoist for ", VariableAccessDataDump(m_graph, variable),
" because the OSR entry value is not a cell: ", value, ".\n");
#endif
iter->value.m_structure = 0;
continue;
}
if (value.asCell()->structure() != iter->value.m_structure) {
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog(
"Zeroing the structure to hoist for ", VariableAccessDataDump(m_graph, variable),
" because the OSR entry value has structure ",
RawPointer(value.asCell()->structure()), " and we wanted ",
RawPointer(iter->value.m_structure), ".\n");
#endif
iter->value.m_structure = 0;
continue;
}
}
}
bool changed = false;
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
for (HashMap<VariableAccessData*, CheckData>::iterator it = m_map.begin();
it != m_map.end(); ++it) {
if (!it->value.m_structure) {
dataLog(
"Not hoisting checks for ", VariableAccessDataDump(m_graph, it->key),
" because of heuristics.\n");
continue;
}
dataLog("Hoisting checks for ", VariableAccessDataDump(m_graph, it->key), "\n");
}
#endif // DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
// Place CheckStructure's at SetLocal sites.
InsertionSet insertionSet(m_graph);
for (BlockIndex blockIndex = 0; blockIndex < m_graph.m_blocks.size(); ++blockIndex) {
BasicBlock* block = m_graph.m_blocks[blockIndex].get();
if (!block)
continue;
for (unsigned indexInBlock = 0; indexInBlock < block->size(); ++indexInBlock) {
Node* node = block->at(indexInBlock);
// Be careful not to use 'node' after appending to the graph. In those switch
// cases where we need to append, we first carefully extract everything we need
// from the node, before doing any appending.
switch (node->op()) {
case SetArgument: {
ASSERT(!blockIndex);
// Insert a GetLocal and a CheckStructure immediately following this
// SetArgument, if the variable was a candidate for structure hoisting.
// If the basic block previously only had the SetArgument as its
// variable-at-tail, then replace it with this GetLocal.
VariableAccessData* variable = node->variableAccessData();
HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
if (iter == m_map.end())
break;
if (!iter->value.m_structure)
break;
CodeOrigin codeOrigin = node->codeOrigin;
Node* getLocal = insertionSet.insertNode(
indexInBlock + 1, variable->prediction(), GetLocal, codeOrigin,
OpInfo(variable), Edge(node));
insertionSet.insertNode(
indexInBlock + 1, SpecNone, CheckStructure, codeOrigin,
OpInfo(m_graph.addStructureSet(iter->value.m_structure)),
Edge(getLocal, CellUse));
if (block->variablesAtTail.operand(variable->local()) == node)
block->variablesAtTail.operand(variable->local()) = getLocal;
m_graph.substituteGetLocal(*block, indexInBlock, variable, getLocal);
changed = true;
break;
}
case SetLocal: {
VariableAccessData* variable = node->variableAccessData();
HashMap<VariableAccessData*, CheckData>::iterator iter = m_map.find(variable);
if (iter == m_map.end())
break;
if (!iter->value.m_structure)
break;
// First insert a dead SetLocal to tell OSR that the child's value should
// be dropped into this bytecode variable if the CheckStructure decides
// to exit.
CodeOrigin codeOrigin = node->codeOrigin;
Edge child1 = node->child1();
insertionSet.insertNode(
indexInBlock, SpecNone, SetLocal, codeOrigin, OpInfo(variable), child1);
// Use a ForwardCheckStructure to indicate that we should exit to the
// next bytecode instruction rather than reexecuting the current one.
insertionSet.insertNode(
indexInBlock, SpecNone, ForwardCheckStructure, codeOrigin,
OpInfo(m_graph.addStructureSet(iter->value.m_structure)),
Edge(child1.node(), CellUse));
changed = true;
break;
}
default:
break;
}
}
insertionSet.execute(block);
}
return changed;
}
void propagate(Node* node)
{
NodeFlags flags = node->flags() & NodeBytecodeBackPropMask;
switch (node->op()) {
case GetLocal: {
VariableAccessData* variableAccessData = node->variableAccessData();
variableAccessData->mergeFlags(flags);
break;
}
case SetLocal: {
VariableAccessData* variableAccessData = node->variableAccessData();
if (!variableAccessData->isLoadedFrom())
break;
node->child1()->mergeFlags(NodeBytecodeUsesAsValue);
break;
}
case BitAnd:
case BitOr:
case BitXor:
case BitRShift:
case BitLShift:
case BitURShift:
case ArithIMul: {
flags |= NodeBytecodeUsesAsInt;
flags &= ~(NodeBytecodeUsesAsNumber | NodeBytecodeNeedsNegZero | NodeBytecodeUsesAsOther);
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ValueToInt32: {
flags |= NodeBytecodeUsesAsInt;
flags &= ~(NodeBytecodeUsesAsNumber | NodeBytecodeNeedsNegZero | NodeBytecodeUsesAsOther);
node->child1()->mergeFlags(flags);
break;
}
case StringCharCodeAt: {
node->child1()->mergeFlags(NodeBytecodeUsesAsValue);
node->child2()->mergeFlags(NodeBytecodeUsesAsValue | NodeBytecodeUsesAsInt);
break;
}
case Identity:
case UInt32ToNumber: {
node->child1()->mergeFlags(flags);
break;
}
case ValueAdd: {
if (isNotNegZero(node->child1().node()) || isNotNegZero(node->child2().node()))
flags &= ~NodeBytecodeNeedsNegZero;
if (node->child1()->hasNumberResult() || node->child2()->hasNumberResult())
flags &= ~NodeBytecodeUsesAsOther;
if (!isWithinPowerOfTwo<32>(node->child1()) && !isWithinPowerOfTwo<32>(node->child2()))
flags |= NodeBytecodeUsesAsNumber;
if (!m_allowNestedOverflowingAdditions)
flags |= NodeBytecodeUsesAsNumber;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithAdd: {
if (isNotNegZero(node->child1().node()) || isNotNegZero(node->child2().node()))
flags &= ~NodeBytecodeNeedsNegZero;
if (!isWithinPowerOfTwo<32>(node->child1()) && !isWithinPowerOfTwo<32>(node->child2()))
flags |= NodeBytecodeUsesAsNumber;
if (!m_allowNestedOverflowingAdditions)
flags |= NodeBytecodeUsesAsNumber;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithSub: {
if (isNotNegZero(node->child1().node()) || isNotPosZero(node->child2().node()))
flags &= ~NodeBytecodeNeedsNegZero;
if (!isWithinPowerOfTwo<32>(node->child1()) && !isWithinPowerOfTwo<32>(node->child2()))
flags |= NodeBytecodeUsesAsNumber;
if (!m_allowNestedOverflowingAdditions)
flags |= NodeBytecodeUsesAsNumber;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithNegate: {
flags &= ~NodeBytecodeUsesAsOther;
node->child1()->mergeFlags(flags);
break;
}
case ArithMul: {
// As soon as a multiply happens, we can easily end up in the part
// of the double domain where the point at which you do truncation
// can change the outcome. So, ArithMul always forces its inputs to
// check for overflow. Additionally, it will have to check for overflow
// itself unless we can prove that there is no way for the values
// produced to cause double rounding.
if (!isWithinPowerOfTwo<22>(node->child1().node())
&& !isWithinPowerOfTwo<22>(node->child2().node()))
flags |= NodeBytecodeUsesAsNumber;
node->mergeFlags(flags);
flags |= NodeBytecodeUsesAsNumber | NodeBytecodeNeedsNegZero;
flags &= ~NodeBytecodeUsesAsOther;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithDiv: {
flags |= NodeBytecodeUsesAsNumber | NodeBytecodeNeedsNegZero;
flags &= ~NodeBytecodeUsesAsOther;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case ArithMod: {
flags |= NodeBytecodeUsesAsNumber | NodeBytecodeNeedsNegZero;
flags &= ~NodeBytecodeUsesAsOther;
node->child1()->mergeFlags(flags);
node->child2()->mergeFlags(flags);
break;
}
case GetByVal: {
node->child1()->mergeFlags(NodeBytecodeUsesAsValue);
node->child2()->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther | NodeBytecodeUsesAsInt);
break;
}
case GetMyArgumentByValSafe: {
node->child1()->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther | NodeBytecodeUsesAsInt);
break;
}
case NewArrayWithSize: {
node->child1()->mergeFlags(NodeBytecodeUsesAsValue | NodeBytecodeUsesAsInt);
break;
}
case NewTypedArray: {
// Negative zero is not observable. NaN versus undefined are only observable
// in that you would get a different exception message. So, like, whatever: we
// claim here that NaN v. undefined is observable.
node->child1()->mergeFlags(NodeBytecodeUsesAsInt | NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther);
break;
}
case StringCharAt: {
node->child1()->mergeFlags(NodeBytecodeUsesAsValue);
node->child2()->mergeFlags(NodeBytecodeUsesAsValue | NodeBytecodeUsesAsInt);
break;
}
case ToString: {
node->child1()->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther);
break;
}
case ToPrimitive: {
node->child1()->mergeFlags(flags);
break;
}
case PutByVal: {
m_graph.varArgChild(node, 0)->mergeFlags(NodeBytecodeUsesAsValue);
m_graph.varArgChild(node, 1)->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther | NodeBytecodeUsesAsInt);
m_graph.varArgChild(node, 2)->mergeFlags(NodeBytecodeUsesAsValue);
break;
}
case Switch: {
SwitchData* data = node->switchData();
switch (data->kind) {
case SwitchImm:
// We don't need NodeBytecodeNeedsNegZero because if the cases are all integers
// then -0 and 0 are treated the same. We don't need NodeBytecodeUsesAsOther
// because if all of the cases are integers then NaN and undefined are
// treated the same (i.e. they will take default).
node->child1()->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsInt);
break;
case SwitchChar: {
// We don't need NodeBytecodeNeedsNegZero because if the cases are all strings
// then -0 and 0 are treated the same. We don't need NodeBytecodeUsesAsOther
// because if all of the cases are single-character strings then NaN
// and undefined are treated the same (i.e. they will take default).
node->child1()->mergeFlags(NodeBytecodeUsesAsNumber);
break;
}
case SwitchString:
// We don't need NodeBytecodeNeedsNegZero because if the cases are all strings
// then -0 and 0 are treated the same.
node->child1()->mergeFlags(NodeBytecodeUsesAsNumber | NodeBytecodeUsesAsOther);
break;
}
break;
}
default:
mergeDefaultFlags(node);
break;
}
}
bool run()
{
SharedSymbolTable* symbolTable = codeBlock()->symbolTable();
// This enumerates the locals that we actually care about and packs them. So for example
// if we use local 1, 3, 4, 5, 7, then we remap them: 1->0, 3->1, 4->2, 5->3, 7->4. We
// treat a variable as being "used" if there exists an access to it (SetLocal, GetLocal,
// Flush, PhantomLocal).
BitVector usedLocals;
// Collect those variables that are used from IR.
bool hasGetLocalUnlinked = false;
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned nodeIndex = block->size(); nodeIndex--;) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case GetLocal:
case SetLocal:
case Flush:
case PhantomLocal: {
VariableAccessData* variable = node->variableAccessData();
if (variable->local().isArgument())
break;
usedLocals.set(variable->local().toLocal());
break;
}
case GetLocalUnlinked: {
VirtualRegister operand = node->unlinkedLocal();
if (operand.isArgument())
break;
usedLocals.set(operand.toLocal());
hasGetLocalUnlinked = true;
break;
}
default:
break;
}
}
}
// Ensure that captured variables and captured inline arguments are pinned down.
// They should have been because of flushes, except that the flushes can be optimized
// away.
if (symbolTable) {
for (int i = symbolTable->captureStart(); i > symbolTable->captureEnd(); i--)
usedLocals.set(VirtualRegister(i).toLocal());
}
if (codeBlock()->usesArguments()) {
usedLocals.set(codeBlock()->argumentsRegister().toLocal());
usedLocals.set(unmodifiedArgumentsRegister(codeBlock()->argumentsRegister()).toLocal());
}
if (codeBlock()->uncheckedActivationRegister().isValid())
usedLocals.set(codeBlock()->activationRegister().toLocal());
for (InlineCallFrameSet::iterator iter = m_graph.m_inlineCallFrames->begin(); !!iter; ++iter) {
InlineCallFrame* inlineCallFrame = *iter;
if (!inlineCallFrame->executable->usesArguments())
continue;
VirtualRegister argumentsRegister = m_graph.argumentsRegisterFor(inlineCallFrame);
usedLocals.set(argumentsRegister.toLocal());
usedLocals.set(unmodifiedArgumentsRegister(argumentsRegister).toLocal());
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
usedLocals.set(VirtualRegister(
virtualRegisterForArgument(argument).offset() +
inlineCallFrame->stackOffset).toLocal());
}
}
Vector<unsigned> allocation(usedLocals.size());
m_graph.m_nextMachineLocal = 0;
for (unsigned i = 0; i < usedLocals.size(); ++i) {
if (!usedLocals.get(i)) {
allocation[i] = UINT_MAX;
continue;
}
allocation[i] = m_graph.m_nextMachineLocal++;
}
for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
VariableAccessData* variable = &m_graph.m_variableAccessData[i];
if (!variable->isRoot())
continue;
if (variable->local().isArgument()) {
variable->machineLocal() = variable->local();
continue;
}
size_t local = variable->local().toLocal();
if (local >= allocation.size())
continue;
if (allocation[local] == UINT_MAX)
continue;
variable->machineLocal() = virtualRegisterForLocal(
allocation[variable->local().toLocal()]);
}
if (codeBlock()->usesArguments()) {
VirtualRegister argumentsRegister = virtualRegisterForLocal(
allocation[codeBlock()->argumentsRegister().toLocal()]);
RELEASE_ASSERT(
virtualRegisterForLocal(allocation[
unmodifiedArgumentsRegister(
codeBlock()->argumentsRegister()).toLocal()])
== unmodifiedArgumentsRegister(argumentsRegister));
codeBlock()->setArgumentsRegister(argumentsRegister);
}
if (codeBlock()->uncheckedActivationRegister().isValid()) {
codeBlock()->setActivationRegister(
virtualRegisterForLocal(allocation[codeBlock()->activationRegister().toLocal()]));
}
for (unsigned i = m_graph.m_inlineVariableData.size(); i--;) {
InlineVariableData data = m_graph.m_inlineVariableData[i];
InlineCallFrame* inlineCallFrame = data.inlineCallFrame;
if (inlineCallFrame->executable->usesArguments()) {
inlineCallFrame->argumentsRegister = virtualRegisterForLocal(
allocation[m_graph.argumentsRegisterFor(inlineCallFrame).toLocal()]);
RELEASE_ASSERT(
virtualRegisterForLocal(allocation[unmodifiedArgumentsRegister(
m_graph.argumentsRegisterFor(inlineCallFrame)).toLocal()])
== unmodifiedArgumentsRegister(inlineCallFrame->argumentsRegister));
}
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
ArgumentPosition& position = m_graph.m_argumentPositions[
data.argumentPositionStart + argument];
VariableAccessData* variable = position.someVariable();
ValueSource source;
if (!variable)
source = ValueSource(SourceIsDead);
else {
source = ValueSource::forFlushFormat(
variable->machineLocal(), variable->flushFormat());
}
inlineCallFrame->arguments[argument] = source.valueRecovery();
}
RELEASE_ASSERT(inlineCallFrame->isClosureCall == !!data.calleeVariable);
if (inlineCallFrame->isClosureCall) {
ValueSource source = ValueSource::forFlushFormat(
data.calleeVariable->machineLocal(),
data.calleeVariable->flushFormat());
inlineCallFrame->calleeRecovery = source.valueRecovery();
} else
RELEASE_ASSERT(inlineCallFrame->calleeRecovery.isConstant());
}
if (symbolTable) {
if (symbolTable->captureCount()) {
unsigned captureStartLocal = allocation[
VirtualRegister(codeBlock()->symbolTable()->captureStart()).toLocal()];
ASSERT(captureStartLocal != UINT_MAX);
m_graph.m_machineCaptureStart = virtualRegisterForLocal(captureStartLocal).offset();
} else
m_graph.m_machineCaptureStart = virtualRegisterForLocal(0).offset();
// This is an abomination. If we had captured an argument then the argument ends
// up being "slow", meaning that loads of the argument go through an extra lookup
// table.
if (const SlowArgument* slowArguments = symbolTable->slowArguments()) {
auto newSlowArguments = std::make_unique<SlowArgument[]>(
symbolTable->parameterCount());
for (size_t i = symbolTable->parameterCount(); i--;) {
newSlowArguments[i] = slowArguments[i];
VirtualRegister reg = VirtualRegister(slowArguments[i].index);
if (reg.isLocal())
newSlowArguments[i].index = virtualRegisterForLocal(allocation[reg.toLocal()]).offset();
}
m_graph.m_slowArguments = std::move(newSlowArguments);
}
}
// Fix GetLocalUnlinked's variable references.
if (hasGetLocalUnlinked) {
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned nodeIndex = block->size(); nodeIndex--;) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case GetLocalUnlinked: {
VirtualRegister operand = node->unlinkedLocal();
if (operand.isLocal())
operand = virtualRegisterForLocal(allocation[operand.toLocal()]);
node->setUnlinkedMachineLocal(operand);
break;
}
default:
break;
}
}
}
}
return true;
}
bool run()
{
RELEASE_ASSERT(m_graph.m_plan.mode == FTLForOSREntryMode);
RELEASE_ASSERT(m_graph.m_form == ThreadedCPS);
unsigned bytecodeIndex = m_graph.m_plan.osrEntryBytecodeIndex;
RELEASE_ASSERT(bytecodeIndex);
RELEASE_ASSERT(bytecodeIndex != UINT_MAX);
// Needed by createPreHeader().
m_graph.m_dominators.computeIfNecessary(m_graph);
CodeBlock* baseline = m_graph.m_profiledBlock;
BasicBlock* target = 0;
for (unsigned blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
unsigned nodeIndex = 0;
Node* firstNode = block->at(0);
while (firstNode->isSemanticallySkippable())
firstNode = block->at(++nodeIndex);
if (firstNode->op() == LoopHint
&& firstNode->origin.semantic == CodeOrigin(bytecodeIndex)) {
target = block;
break;
}
}
if (!target) {
// This is a terrible outcome. It shouldn't often happen but it might
// happen and so we should defend against it. If it happens, then this
// compilation is a failure.
return false;
}
BlockInsertionSet insertionSet(m_graph);
BasicBlock* newRoot = insertionSet.insert(0, QNaN);
NodeOrigin origin = target->at(0)->origin;
Vector<Node*> locals(baseline->m_numCalleeRegisters);
for (int local = 0; local < baseline->m_numCalleeRegisters; ++local) {
Node* previousHead = target->variablesAtHead.local(local);
if (!previousHead)
continue;
VariableAccessData* variable = previousHead->variableAccessData();
locals[local] = newRoot->appendNode(
m_graph, variable->prediction(), ExtractOSREntryLocal, origin,
OpInfo(variable->local().offset()));
newRoot->appendNode(
m_graph, SpecNone, MovHint, origin, OpInfo(variable->local().offset()),
Edge(locals[local]));
}
for (int argument = 0; argument < baseline->numParameters(); ++argument) {
Node* oldNode = target->variablesAtHead.argument(argument);
if (!oldNode) {
// Just for sanity, always have a SetArgument even if it's not needed.
oldNode = m_graph.m_arguments[argument];
}
Node* node = newRoot->appendNode(
m_graph, SpecNone, SetArgument, origin,
OpInfo(oldNode->variableAccessData()));
m_graph.m_arguments[argument] = node;
}
for (int local = 0; local < baseline->m_numCalleeRegisters; ++local) {
Node* previousHead = target->variablesAtHead.local(local);
if (!previousHead)
continue;
VariableAccessData* variable = previousHead->variableAccessData();
Node* node = locals[local];
newRoot->appendNode(
m_graph, SpecNone, SetLocal, origin, OpInfo(variable), Edge(node));
}
newRoot->appendNode(
m_graph, SpecNone, Jump, origin,
OpInfo(createPreHeader(m_graph, insertionSet, target)));
insertionSet.execute();
m_graph.resetReachability();
m_graph.killUnreachableBlocks();
return true;
}
bool run()
{
ASSERT(m_graph.m_form == ThreadedCPS);
ASSERT(m_graph.m_unificationState == LocallyUnified);
// Ensure that all Phi functions are unified.
for (BlockIndex blockIndex = m_graph.m_blocks.size(); blockIndex--;) {
BasicBlock* block = m_graph.m_blocks[blockIndex].get();
if (!block)
continue;
ASSERT(block->isReachable);
for (unsigned phiIndex = block->phis.size(); phiIndex--;) {
Node* phi = block->phis[phiIndex];
for (unsigned childIdx = 0; childIdx < AdjacencyList::Size; ++childIdx) {
if (!phi->children.child(childIdx))
break;
phi->variableAccessData()->unify(
phi->children.child(childIdx)->variableAccessData());
}
}
}
// Ensure that all predictions are fixed up based on the unification.
for (unsigned i = 0; i < m_graph.m_variableAccessData.size(); ++i) {
VariableAccessData* data = &m_graph.m_variableAccessData[i];
data->find()->predict(data->nonUnifiedPrediction());
data->find()->mergeIsCaptured(data->isCaptured());
data->find()->mergeStructureCheckHoistingFailed(data->structureCheckHoistingFailed());
data->find()->mergeShouldNeverUnbox(data->shouldNeverUnbox());
data->find()->mergeIsLoadedFrom(data->isLoadedFrom());
}
m_graph.m_unificationState = GloballyUnified;
return true;
}
bool run()
{
// This enumerates the locals that we actually care about and packs them. So for example
// if we use local 1, 3, 4, 5, 7, then we remap them: 1->0, 3->1, 4->2, 5->3, 7->4. We
// treat a variable as being "used" if there exists an access to it (SetLocal, GetLocal,
// Flush, PhantomLocal).
BitVector usedLocals;
// Collect those variables that are used from IR.
bool hasNodesThatNeedFixup = false;
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned nodeIndex = block->size(); nodeIndex--;) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case GetLocal:
case SetLocal:
case Flush:
case PhantomLocal: {
VariableAccessData* variable = node->variableAccessData();
if (variable->local().isArgument())
break;
usedLocals.set(variable->local().toLocal());
break;
}
case GetLocalUnlinked: {
VirtualRegister operand = node->unlinkedLocal();
if (operand.isArgument())
break;
usedLocals.set(operand.toLocal());
hasNodesThatNeedFixup = true;
break;
}
case LoadVarargs:
case ForwardVarargs: {
LoadVarargsData* data = node->loadVarargsData();
if (data->count.isLocal())
usedLocals.set(data->count.toLocal());
if (data->start.isLocal()) {
// This part really relies on the contiguity of stack layout
// assignments.
ASSERT(VirtualRegister(data->start.offset() + data->limit - 1).isLocal());
for (unsigned i = data->limit; i--;)
usedLocals.set(VirtualRegister(data->start.offset() + i).toLocal());
} // the else case shouldn't happen.
hasNodesThatNeedFixup = true;
break;
}
case PutStack:
case GetStack: {
StackAccessData* stack = node->stackAccessData();
if (stack->local.isArgument())
break;
usedLocals.set(stack->local.toLocal());
break;
}
default:
break;
}
}
}
for (InlineCallFrameSet::iterator iter = m_graph.m_plan.inlineCallFrames->begin(); !!iter; ++iter) {
InlineCallFrame* inlineCallFrame = *iter;
if (inlineCallFrame->isVarargs()) {
usedLocals.set(VirtualRegister(
JSStack::ArgumentCount + inlineCallFrame->stackOffset).toLocal());
}
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
usedLocals.set(VirtualRegister(
virtualRegisterForArgument(argument).offset() +
inlineCallFrame->stackOffset).toLocal());
}
}
Vector<unsigned> allocation(usedLocals.size());
m_graph.m_nextMachineLocal = 0;
for (unsigned i = 0; i < usedLocals.size(); ++i) {
if (!usedLocals.get(i)) {
allocation[i] = UINT_MAX;
continue;
}
allocation[i] = m_graph.m_nextMachineLocal++;
}
for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
VariableAccessData* variable = &m_graph.m_variableAccessData[i];
if (!variable->isRoot())
continue;
if (variable->local().isArgument()) {
variable->machineLocal() = variable->local();
continue;
}
size_t local = variable->local().toLocal();
if (local >= allocation.size())
continue;
if (allocation[local] == UINT_MAX)
continue;
variable->machineLocal() = assign(allocation, variable->local());
}
for (StackAccessData* data : m_graph.m_stackAccessData) {
if (!data->local.isLocal()) {
data->machineLocal = data->local;
continue;
}
if (static_cast<size_t>(data->local.toLocal()) >= allocation.size())
continue;
if (allocation[data->local.toLocal()] == UINT_MAX)
continue;
data->machineLocal = assign(allocation, data->local);
}
// This register is never valid for DFG code blocks.
codeBlock()->setActivationRegister(VirtualRegister());
if (LIKELY(!m_graph.hasDebuggerEnabled()))
codeBlock()->setScopeRegister(VirtualRegister());
else
codeBlock()->setScopeRegister(assign(allocation, codeBlock()->scopeRegister()));
for (unsigned i = m_graph.m_inlineVariableData.size(); i--;) {
InlineVariableData data = m_graph.m_inlineVariableData[i];
InlineCallFrame* inlineCallFrame = data.inlineCallFrame;
if (inlineCallFrame->isVarargs()) {
inlineCallFrame->argumentCountRegister = assign(
allocation, VirtualRegister(inlineCallFrame->stackOffset + JSStack::ArgumentCount));
}
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
ArgumentPosition& position = m_graph.m_argumentPositions[
data.argumentPositionStart + argument];
VariableAccessData* variable = position.someVariable();
ValueSource source;
if (!variable)
source = ValueSource(SourceIsDead);
else {
source = ValueSource::forFlushFormat(
variable->machineLocal(), variable->flushFormat());
}
inlineCallFrame->arguments[argument] = source.valueRecovery();
}
RELEASE_ASSERT(inlineCallFrame->isClosureCall == !!data.calleeVariable);
if (inlineCallFrame->isClosureCall) {
VariableAccessData* variable = data.calleeVariable->find();
ValueSource source = ValueSource::forFlushFormat(
variable->machineLocal(),
variable->flushFormat());
inlineCallFrame->calleeRecovery = source.valueRecovery();
} else
RELEASE_ASSERT(inlineCallFrame->calleeRecovery.isConstant());
}
// Fix GetLocalUnlinked's variable references.
if (hasNodesThatNeedFixup) {
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned nodeIndex = block->size(); nodeIndex--;) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case GetLocalUnlinked: {
node->setUnlinkedMachineLocal(assign(allocation, node->unlinkedLocal()));
break;
}
case LoadVarargs:
case ForwardVarargs: {
LoadVarargsData* data = node->loadVarargsData();
data->machineCount = assign(allocation, data->count);
data->machineStart = assign(allocation, data->start);
break;
}
default:
break;
}
}
}
}
return true;
}
bool run()
{
RELEASE_ASSERT(m_graph.m_form == ThreadedCPS);
m_graph.clearReplacements();
m_graph.m_dominators.computeIfNecessary(m_graph);
if (verbose) {
dataLog("Graph before SSA transformation:\n");
m_graph.dump();
}
// Create a SSACalculator::Variable for every root VariableAccessData.
for (VariableAccessData& variable : m_graph.m_variableAccessData) {
if (!variable.isRoot() || variable.isCaptured())
continue;
SSACalculator::Variable* ssaVariable = m_calculator.newVariable();
ASSERT(ssaVariable->index() == m_variableForSSAIndex.size());
m_variableForSSAIndex.append(&variable);
m_ssaVariableForVariable.add(&variable, ssaVariable);
}
// Find all SetLocals and create Defs for them. We handle SetArgument by creating a
// GetLocal, and recording the flush format.
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
// Must process the block in forward direction because we want to see the last
// assignment for every local.
for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
Node* node = block->at(nodeIndex);
if (node->op() != SetLocal && node->op() != SetArgument)
continue;
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured())
continue;
Node* childNode;
if (node->op() == SetLocal)
childNode = node->child1().node();
else {
ASSERT(node->op() == SetArgument);
childNode = m_insertionSet.insertNode(
nodeIndex, node->variableAccessData()->prediction(),
GetStack, node->origin,
OpInfo(m_graph.m_stackAccessData.add(variable->local(), variable->flushFormat())));
if (!ASSERT_DISABLED)
m_argumentGetters.add(childNode);
m_argumentMapping.add(node, childNode);
}
m_calculator.newDef(
m_ssaVariableForVariable.get(variable), block, childNode);
}
m_insertionSet.execute(block);
}
// Decide where Phis are to be inserted. This creates the Phi's but doesn't insert them
// yet. We will later know where to insert them because SSACalculator is such a bro.
m_calculator.computePhis(
[&] (SSACalculator::Variable* ssaVariable, BasicBlock* block) -> Node* {
VariableAccessData* variable = m_variableForSSAIndex[ssaVariable->index()];
// Prune by liveness. This doesn't buy us much other than compile times.
Node* headNode = block->variablesAtHead.operand(variable->local());
if (!headNode)
return nullptr;
// There is the possibiltiy of "rebirths". The SSA calculator will already prune
// rebirths for the same VariableAccessData. But it will not be able to prune
// rebirths that arose from the same local variable number but a different
// VariableAccessData. We do that pruning here.
//
// Here's an example of a rebirth that this would catch:
//
// var x;
// if (foo) {
// if (bar) {
// x = 42;
// } else {
// x = 43;
// }
// print(x);
// x = 44;
// } else {
// x = 45;
// }
// print(x); // Without this check, we'd have a Phi for x = 42|43 here.
//
// FIXME: Consider feeding local variable numbers, not VariableAccessData*'s, as
// the "variables" for SSACalculator. That would allow us to eliminate this
// special case.
// https://bugs.webkit.org/show_bug.cgi?id=136641
if (headNode->variableAccessData() != variable)
return nullptr;
Node* phiNode = m_graph.addNode(
variable->prediction(), Phi, NodeOrigin());
FlushFormat format = variable->flushFormat();
NodeFlags result = resultFor(format);
phiNode->mergeFlags(result);
return phiNode;
});
if (verbose) {
dataLog("Computed Phis, about to transform the graph.\n");
dataLog("\n");
dataLog("Graph:\n");
m_graph.dump();
dataLog("\n");
dataLog("Mappings:\n");
for (unsigned i = 0; i < m_variableForSSAIndex.size(); ++i)
dataLog(" ", i, ": ", VariableAccessDataDump(m_graph, m_variableForSSAIndex[i]), "\n");
dataLog("\n");
dataLog("SSA calculator: ", m_calculator, "\n");
}
// Do the bulk of the SSA conversion. For each block, this tracks the operand->Node
// mapping based on a combination of what the SSACalculator tells us, and us walking over
// the block in forward order. We use our own data structure, valueForOperand, for
// determining the local mapping, but we rely on SSACalculator for the non-local mapping.
//
// This does three things at once:
//
// - Inserts the Phis in all of the places where they need to go. We've already created
// them and they are accounted for in the SSACalculator's data structures, but we
// haven't inserted them yet, mostly because we want to insert all of a block's Phis in
// one go to amortize the cost of node insertion.
//
// - Create and insert Upsilons.
//
// - Convert all of the preexisting SSA nodes (other than the old CPS Phi nodes) into SSA
// form by replacing as follows:
//
// - MovHint has KillLocal prepended to it.
//
// - GetLocal over captured variables lose their phis and become GetStack.
//
// - GetLocal over uncaptured variables die and get replaced with references to the node
// specified by valueForOperand.
//
// - SetLocal turns into PutStack if it's flushed, or turns into a Check otherwise.
//
// - Flush loses its children and turns into a Phantom.
//
// - PhantomLocal becomes Phantom, and its child is whatever is specified by
// valueForOperand.
//
// - SetArgument is removed. Note that GetStack nodes have already been inserted.
Operands<Node*> valueForOperand(OperandsLike, m_graph.block(0)->variablesAtHead);
for (BasicBlock* block : m_graph.blocksInPreOrder()) {
valueForOperand.clear();
// CPS will claim that the root block has all arguments live. But we have already done
// the first step of SSA conversion: argument locals are no longer live at head;
// instead we have GetStack nodes for extracting the values of arguments. So, we
// skip the at-head available value calculation for the root block.
if (block != m_graph.block(0)) {
for (size_t i = valueForOperand.size(); i--;) {
Node* nodeAtHead = block->variablesAtHead[i];
if (!nodeAtHead)
continue;
VariableAccessData* variable = nodeAtHead->variableAccessData();
if (variable->isCaptured())
continue;
if (verbose)
dataLog("Considering live variable ", VariableAccessDataDump(m_graph, variable), " at head of block ", *block, "\n");
SSACalculator::Variable* ssaVariable = m_ssaVariableForVariable.get(variable);
SSACalculator::Def* def = m_calculator.reachingDefAtHead(block, ssaVariable);
if (!def) {
// If we are required to insert a Phi, then we won't have a reaching def
// at head.
continue;
}
Node* node = def->value();
if (node->replacement) {
// This will occur when a SetLocal had a GetLocal as its source. The
// GetLocal would get replaced with an actual SSA value by the time we get
// here. Note that the SSA value with which the GetLocal got replaced
// would not in turn have a replacement.
node = node->replacement;
ASSERT(!node->replacement);
}
if (verbose)
dataLog("Mapping: ", VirtualRegister(valueForOperand.operandForIndex(i)), " -> ", node, "\n");
valueForOperand[i] = node;
}
}
// Insert Phis by asking the calculator what phis there are in this block. Also update
// valueForOperand with those Phis. For Phis associated with variables that are not
// flushed, we also insert a MovHint.
size_t phiInsertionPoint = 0;
for (SSACalculator::Def* phiDef : m_calculator.phisForBlock(block)) {
VariableAccessData* variable = m_variableForSSAIndex[phiDef->variable()->index()];
m_insertionSet.insert(phiInsertionPoint, phiDef->value());
valueForOperand.operand(variable->local()) = phiDef->value();
m_insertionSet.insertNode(
phiInsertionPoint, SpecNone, MovHint, NodeOrigin(),
OpInfo(variable->local().offset()), phiDef->value()->defaultEdge());
}
for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
Node* node = block->at(nodeIndex);
if (verbose) {
dataLog("Processing node ", node, ":\n");
m_graph.dump(WTF::dataFile(), " ", node);
}
m_graph.performSubstitution(node);
switch (node->op()) {
case MovHint: {
m_insertionSet.insertNode(
nodeIndex, SpecNone, KillStack, node->origin,
OpInfo(node->unlinkedLocal().offset()));
break;
}
case SetLocal: {
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured() || !!(node->flags() & NodeIsFlushed)) {
node->convertToPutStack(
m_graph.m_stackAccessData.add(
variable->local(), variable->flushFormat()));
} else
node->setOpAndDefaultFlags(Check);
if (!variable->isCaptured()) {
if (verbose)
dataLog("Mapping: ", variable->local(), " -> ", node->child1().node(), "\n");
valueForOperand.operand(variable->local()) = node->child1().node();
}
break;
}
case GetStack: {
ASSERT(m_argumentGetters.contains(node));
valueForOperand.operand(node->stackAccessData()->local) = node;
break;
}
case GetLocal: {
VariableAccessData* variable = node->variableAccessData();
node->children.reset();
if (variable->isCaptured()) {
node->convertToGetStack(m_graph.m_stackAccessData.add(variable->local(), variable->flushFormat()));
break;
}
node->convertToPhantom();
if (verbose)
dataLog("Replacing node ", node, " with ", valueForOperand.operand(variable->local()), "\n");
node->replacement = valueForOperand.operand(variable->local());
break;
}
case Flush: {
node->children.reset();
node->convertToPhantom();
break;
}
case PhantomLocal: {
ASSERT(node->child1().useKind() == UntypedUse);
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured()) {
// This is a fun case. We could have a captured variable that had some
// or all of its uses strength reduced to phantoms rather than flushes.
// SSA conversion will currently still treat it as flushed, in the sense
// that it will just keep the SetLocal. Therefore, there is nothing that
// needs to be done here: we don't need to also keep the source value
// alive. And even if we did want to keep the source value alive, we
// wouldn't be able to, because the variablesAtHead value for a captured
// local wouldn't have been computed by the Phi reduction algorithm
// above.
node->children.reset();
} else
node->child1() = valueForOperand.operand(variable->local())->defaultEdge();
node->convertToPhantom();
break;
}
case SetArgument: {
node->convertToPhantom();
break;
}
default:
break;
}
}
// We want to insert Upsilons just before the end of the block. On the surface this
// seems dangerous because the Upsilon will have a checking UseKind. But, we will not
// actually be performing the check at the point of the Upsilon; the check will
// already have been performed at the point where the original SetLocal was.
size_t upsilonInsertionPoint = block->size() - 1;
NodeOrigin upsilonOrigin = block->last()->origin;
for (unsigned successorIndex = block->numSuccessors(); successorIndex--;) {
BasicBlock* successorBlock = block->successor(successorIndex);
for (SSACalculator::Def* phiDef : m_calculator.phisForBlock(successorBlock)) {
Node* phiNode = phiDef->value();
SSACalculator::Variable* ssaVariable = phiDef->variable();
VariableAccessData* variable = m_variableForSSAIndex[ssaVariable->index()];
FlushFormat format = variable->flushFormat();
UseKind useKind = useKindFor(format);
m_insertionSet.insertNode(
upsilonInsertionPoint, SpecNone, Upsilon, upsilonOrigin,
OpInfo(phiNode), Edge(
valueForOperand.operand(variable->local()),
useKind));
}
}
m_insertionSet.execute(block);
}
// Free all CPS phis and reset variables vectors.
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned phiIndex = block->phis.size(); phiIndex--;)
m_graph.m_allocator.free(block->phis[phiIndex]);
block->phis.clear();
block->variablesAtHead.clear();
block->variablesAtTail.clear();
block->valuesAtHead.clear();
block->valuesAtHead.clear();
block->ssa = std::make_unique<BasicBlock::SSAData>(block);
}
m_graph.m_argumentFormats.resize(m_graph.m_arguments.size());
for (unsigned i = m_graph.m_arguments.size(); i--;) {
FlushFormat format = FlushedJSValue;
Node* node = m_argumentMapping.get(m_graph.m_arguments[i]);
// m_argumentMapping.get could return null for a captured local. That's fine. We only
// track the argument loads of those arguments for which we speculate type. We don't
// speculate type for captured arguments.
if (node)
format = node->stackAccessData()->format;
m_graph.m_argumentFormats[i] = format;
m_graph.m_arguments[i] = node; // Record the load that loads the arguments for the benefit of exit profiling.
}
m_graph.m_form = SSA;
if (verbose) {
dataLog("Graph after SSA transformation:\n");
m_graph.dump();
}
return true;
}
void propagate(Node& node)
{
if (!node.shouldGenerate())
return;
NodeType op = node.op();
NodeFlags flags = node.flags() & NodeBackPropMask;
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog(" %s @%u: %s ", Graph::opName(op), m_compileIndex, nodeFlagsAsString(flags));
#endif
bool changed = false;
switch (op) {
case JSConstant:
case WeakJSConstant: {
changed |= setPrediction(speculationFromValue(m_graph.valueOfJSConstant(m_compileIndex)));
break;
}
case GetLocal: {
VariableAccessData* variableAccessData = node.variableAccessData();
SpeculatedType prediction = variableAccessData->prediction();
if (prediction)
changed |= mergePrediction(prediction);
changed |= variableAccessData->mergeFlags(flags);
break;
}
case SetLocal: {
VariableAccessData* variableAccessData = node.variableAccessData();
changed |= variableAccessData->predict(m_graph[node.child1()].prediction());
changed |= m_graph[node.child1()].mergeFlags(variableAccessData->flags());
break;
}
case Flush: {
// Make sure that the analysis knows that flushed locals escape.
VariableAccessData* variableAccessData = node.variableAccessData();
changed |= variableAccessData->mergeFlags(NodeUsedAsValue);
break;
}
case BitAnd:
case BitOr:
case BitXor:
case BitRShift:
case BitLShift:
case BitURShift: {
changed |= setPrediction(SpecInt32);
flags |= NodeUsedAsInt;
flags &= ~(NodeUsedAsNumber | NodeNeedsNegZero);
changed |= m_graph[node.child1()].mergeFlags(flags);
changed |= m_graph[node.child2()].mergeFlags(flags);
break;
}
case ValueToInt32: {
changed |= setPrediction(SpecInt32);
flags |= NodeUsedAsInt;
flags &= ~(NodeUsedAsNumber | NodeNeedsNegZero);
changed |= m_graph[node.child1()].mergeFlags(flags);
break;
}
case ArrayPop: {
changed |= mergePrediction(node.getHeapPrediction());
changed |= mergeDefaultFlags(node);
break;
}
case ArrayPush: {
changed |= mergePrediction(node.getHeapPrediction());
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsValue);
changed |= m_graph[node.child2()].mergeFlags(NodeUsedAsValue);
break;
}
case RegExpExec:
case RegExpTest: {
changed |= mergePrediction(node.getHeapPrediction());
changed |= mergeDefaultFlags(node);
break;
}
case StringCharCodeAt: {
changed |= mergePrediction(SpecInt32);
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsValue);
changed |= m_graph[node.child2()].mergeFlags(NodeUsedAsNumber | NodeUsedAsInt);
break;
}
case ArithMod: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
if (left && right) {
if (isInt32Speculation(mergeSpeculations(left, right))
&& nodeCanSpeculateInteger(node.arithNodeFlags()))
changed |= mergePrediction(SpecInt32);
else
changed |= mergePrediction(SpecDouble);
}
flags |= NodeUsedAsValue;
changed |= m_graph[node.child1()].mergeFlags(flags);
changed |= m_graph[node.child2()].mergeFlags(flags);
break;
}
case UInt32ToNumber: {
if (nodeCanSpeculateInteger(node.arithNodeFlags()))
changed |= mergePrediction(SpecInt32);
else
changed |= mergePrediction(SpecNumber);
changed |= m_graph[node.child1()].mergeFlags(flags);
break;
}
case ValueAdd: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
if (left && right) {
if (isNumberSpeculation(left) && isNumberSpeculation(right)) {
if (m_graph.addShouldSpeculateInteger(node))
changed |= mergePrediction(SpecInt32);
else
changed |= mergePrediction(speculatedDoubleTypeForPredictions(left, right));
} else if (!(left & SpecNumber) || !(right & SpecNumber)) {
// left or right is definitely something other than a number.
changed |= mergePrediction(SpecString);
} else
changed |= mergePrediction(SpecString | SpecInt32 | SpecDouble);
}
if (isNotNegZero(node.child1().index()) || isNotNegZero(node.child2().index()))
flags &= ~NodeNeedsNegZero;
changed |= m_graph[node.child1()].mergeFlags(flags);
changed |= m_graph[node.child2()].mergeFlags(flags);
break;
}
case ArithAdd: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
if (left && right) {
if (m_graph.addShouldSpeculateInteger(node))
changed |= mergePrediction(SpecInt32);
else
changed |= mergePrediction(speculatedDoubleTypeForPredictions(left, right));
}
if (isNotNegZero(node.child1().index()) || isNotNegZero(node.child2().index()))
flags &= ~NodeNeedsNegZero;
changed |= m_graph[node.child1()].mergeFlags(flags);
changed |= m_graph[node.child2()].mergeFlags(flags);
break;
}
case ArithSub: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
if (left && right) {
if (m_graph.addShouldSpeculateInteger(node))
changed |= mergePrediction(SpecInt32);
else
changed |= mergePrediction(speculatedDoubleTypeForPredictions(left, right));
}
if (isNotZero(node.child1().index()) || isNotZero(node.child2().index()))
flags &= ~NodeNeedsNegZero;
changed |= m_graph[node.child1()].mergeFlags(flags);
changed |= m_graph[node.child2()].mergeFlags(flags);
break;
}
case ArithNegate:
if (m_graph[node.child1()].prediction()) {
if (m_graph.negateShouldSpeculateInteger(node))
changed |= mergePrediction(SpecInt32);
else
changed |= mergePrediction(speculatedDoubleTypeForPrediction(m_graph[node.child1()].prediction()));
}
changed |= m_graph[node.child1()].mergeFlags(flags);
break;
case ArithMin:
case ArithMax: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
if (left && right) {
if (isInt32Speculation(mergeSpeculations(left, right))
&& nodeCanSpeculateInteger(node.arithNodeFlags()))
changed |= mergePrediction(SpecInt32);
else
changed |= mergePrediction(speculatedDoubleTypeForPredictions(left, right));
}
flags |= NodeUsedAsNumber;
changed |= m_graph[node.child1()].mergeFlags(flags);
changed |= m_graph[node.child2()].mergeFlags(flags);
break;
}
case ArithMul: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
if (left && right) {
if (m_graph.mulShouldSpeculateInteger(node))
changed |= mergePrediction(SpecInt32);
else
changed |= mergePrediction(speculatedDoubleTypeForPredictions(left, right));
}
// As soon as a multiply happens, we can easily end up in the part
// of the double domain where the point at which you do truncation
// can change the outcome. So, ArithMul always checks for overflow
// no matter what, and always forces its inputs to check as well.
flags |= NodeUsedAsNumber | NodeNeedsNegZero;
changed |= m_graph[node.child1()].mergeFlags(flags);
changed |= m_graph[node.child2()].mergeFlags(flags);
break;
}
case ArithDiv: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
if (left && right) {
if (isInt32Speculation(mergeSpeculations(left, right))
&& nodeCanSpeculateInteger(node.arithNodeFlags()))
changed |= mergePrediction(SpecInt32);
else
changed |= mergePrediction(SpecDouble);
}
// As soon as a multiply happens, we can easily end up in the part
// of the double domain where the point at which you do truncation
// can change the outcome. So, ArithMul always checks for overflow
// no matter what, and always forces its inputs to check as well.
flags |= NodeUsedAsNumber | NodeNeedsNegZero;
changed |= m_graph[node.child1()].mergeFlags(flags);
changed |= m_graph[node.child2()].mergeFlags(flags);
break;
}
case ArithSqrt: {
changed |= setPrediction(SpecDouble);
changed |= m_graph[node.child1()].mergeFlags(flags | NodeUsedAsValue);
break;
}
case ArithAbs: {
SpeculatedType child = m_graph[node.child1()].prediction();
if (nodeCanSpeculateInteger(node.arithNodeFlags()))
changed |= mergePrediction(child);
else
changed |= setPrediction(speculatedDoubleTypeForPrediction(child));
flags &= ~NodeNeedsNegZero;
changed |= m_graph[node.child1()].mergeFlags(flags);
break;
}
case LogicalNot:
case CompareLess:
case CompareLessEq:
case CompareGreater:
case CompareGreaterEq:
case CompareEq:
case CompareStrictEq:
case InstanceOf:
case IsUndefined:
case IsBoolean:
case IsNumber:
case IsString:
case IsObject:
case IsFunction: {
changed |= setPrediction(SpecBoolean);
changed |= mergeDefaultFlags(node);
break;
}
case GetById: {
changed |= mergePrediction(node.getHeapPrediction());
changed |= mergeDefaultFlags(node);
break;
}
case GetByIdFlush:
changed |= mergePrediction(node.getHeapPrediction());
changed |= mergeDefaultFlags(node);
break;
case GetByVal: {
if (m_graph[node.child1()].shouldSpeculateFloat32Array()
|| m_graph[node.child1()].shouldSpeculateFloat64Array())
changed |= mergePrediction(SpecDouble);
else
changed |= mergePrediction(node.getHeapPrediction());
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsValue);
changed |= m_graph[node.child2()].mergeFlags(NodeUsedAsNumber | NodeUsedAsInt);
break;
}
case GetMyArgumentByValSafe: {
changed |= mergePrediction(node.getHeapPrediction());
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsNumber | NodeUsedAsInt);
break;
}
case GetMyArgumentsLengthSafe: {
changed |= setPrediction(SpecInt32);
break;
}
case GetScopeRegisters:
case GetButterfly:
case GetIndexedPropertyStorage:
case AllocatePropertyStorage:
case ReallocatePropertyStorage: {
changed |= setPrediction(SpecOther);
changed |= mergeDefaultFlags(node);
break;
}
case GetByOffset: {
changed |= mergePrediction(node.getHeapPrediction());
changed |= mergeDefaultFlags(node);
break;
}
case Call:
case Construct: {
changed |= mergePrediction(node.getHeapPrediction());
for (unsigned childIdx = node.firstChild();
childIdx < node.firstChild() + node.numChildren();
++childIdx) {
Edge edge = m_graph.m_varArgChildren[childIdx];
changed |= m_graph[edge].mergeFlags(NodeUsedAsValue);
}
break;
}
case ConvertThis: {
SpeculatedType prediction = m_graph[node.child1()].prediction();
if (prediction) {
if (prediction & ~SpecObjectMask) {
prediction &= SpecObjectMask;
prediction = mergeSpeculations(prediction, SpecObjectOther);
}
changed |= mergePrediction(prediction);
}
changed |= mergeDefaultFlags(node);
break;
}
case GetGlobalVar: {
changed |= mergePrediction(node.getHeapPrediction());
break;
}
case PutGlobalVar:
case PutGlobalVarCheck: {
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsValue);
break;
}
case GetScopedVar:
case Resolve:
case ResolveBase:
case ResolveBaseStrictPut:
case ResolveGlobal: {
SpeculatedType prediction = node.getHeapPrediction();
changed |= mergePrediction(prediction);
break;
}
case GetScope: {
changed |= setPrediction(SpecCellOther);
break;
}
case GetCallee: {
changed |= setPrediction(SpecFunction);
break;
}
case CreateThis:
case NewObject: {
changed |= setPrediction(SpecFinalObject);
changed |= mergeDefaultFlags(node);
break;
}
case NewArray: {
changed |= setPrediction(SpecArray);
for (unsigned childIdx = node.firstChild();
childIdx < node.firstChild() + node.numChildren();
++childIdx) {
Edge edge = m_graph.m_varArgChildren[childIdx];
changed |= m_graph[edge].mergeFlags(NodeUsedAsValue);
}
break;
}
case NewArrayWithSize: {
changed |= setPrediction(SpecArray);
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsNumber | NodeUsedAsInt);
break;
}
case NewArrayBuffer: {
changed |= setPrediction(SpecArray);
break;
}
case NewRegexp: {
changed |= setPrediction(SpecObjectOther);
break;
}
case StringCharAt: {
changed |= setPrediction(SpecString);
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsValue);
changed |= m_graph[node.child2()].mergeFlags(NodeUsedAsNumber | NodeUsedAsInt);
break;
}
case StrCat: {
changed |= setPrediction(SpecString);
for (unsigned childIdx = node.firstChild();
childIdx < node.firstChild() + node.numChildren();
++childIdx)
changed |= m_graph[m_graph.m_varArgChildren[childIdx]].mergeFlags(NodeUsedAsNumber);
break;
}
case ToPrimitive: {
SpeculatedType child = m_graph[node.child1()].prediction();
if (child) {
if (isObjectSpeculation(child)) {
// I'd love to fold this case into the case below, but I can't, because
// removing SpecObjectMask from something that only has an object
// prediction and nothing else means we have an ill-formed SpeculatedType
// (strong predict-none). This should be killed once we remove all traces
// of static (aka weak) predictions.
changed |= mergePrediction(SpecString);
} else if (child & SpecObjectMask) {
// Objects get turned into strings. So if the input has hints of objectness,
// the output will have hinsts of stringiness.
changed |= mergePrediction(
mergeSpeculations(child & ~SpecObjectMask, SpecString));
} else
changed |= mergePrediction(child);
}
changed |= m_graph[node.child1()].mergeFlags(flags);
break;
}
case CreateActivation: {
changed |= setPrediction(SpecObjectOther);
break;
}
case CreateArguments: {
// At this stage we don't try to predict whether the arguments are ours or
// someone else's. We could, but we don't, yet.
changed |= setPrediction(SpecArguments);
break;
}
case NewFunction:
case NewFunctionNoCheck:
case NewFunctionExpression: {
changed |= setPrediction(SpecFunction);
break;
}
case PutByValAlias:
case GetArrayLength:
case Int32ToDouble:
case DoubleAsInt32:
case GetLocalUnlinked:
case GetMyArgumentsLength:
case GetMyArgumentByVal:
case PhantomPutStructure:
case PhantomArguments:
case CheckArray:
case Arrayify: {
// This node should never be visible at this stage of compilation. It is
// inserted by fixup(), which follows this phase.
ASSERT_NOT_REACHED();
break;
}
case PutByVal:
changed |= m_graph[m_graph.varArgChild(node, 0)].mergeFlags(NodeUsedAsValue);
changed |= m_graph[m_graph.varArgChild(node, 1)].mergeFlags(NodeUsedAsNumber | NodeUsedAsInt);
changed |= m_graph[m_graph.varArgChild(node, 2)].mergeFlags(NodeUsedAsValue);
break;
case PutScopedVar:
case Return:
case Throw:
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsValue);
break;
case PutById:
case PutByIdDirect:
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsValue);
changed |= m_graph[node.child2()].mergeFlags(NodeUsedAsValue);
break;
case PutByOffset:
changed |= m_graph[node.child1()].mergeFlags(NodeUsedAsValue);
changed |= m_graph[node.child3()].mergeFlags(NodeUsedAsValue);
break;
case Phi:
break;
#ifndef NDEBUG
// These get ignored because they don't return anything.
case DFG::Jump:
case Branch:
case Breakpoint:
case CheckHasInstance:
case ThrowReferenceError:
case ForceOSRExit:
case SetArgument:
case CheckStructure:
case ForwardCheckStructure:
case StructureTransitionWatchpoint:
case ForwardStructureTransitionWatchpoint:
case CheckFunction:
case PutStructure:
case TearOffActivation:
case TearOffArguments:
case CheckNumber:
case CheckArgumentsNotCreated:
case GlobalVarWatchpoint:
case GarbageValue:
changed |= mergeDefaultFlags(node);
break;
// These gets ignored because it doesn't do anything.
case Phantom:
case InlineStart:
case Nop:
break;
case LastNodeType:
ASSERT_NOT_REACHED();
break;
#else
default:
changed |= mergeDefaultFlags(node);
break;
#endif
}
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog("%s\n", speculationToString(m_graph[m_compileIndex].prediction()));
#endif
m_changed |= changed;
}
bool run()
{
RELEASE_ASSERT(m_graph.m_form == ThreadedCPS);
if (dumpGraph) {
dataLog("Graph dump at top of SSA conversion:\n");
m_graph.dump();
}
// Eliminate all duplicate or self-pointing Phi edges. This means that
// we transform:
//
// p: Phi(@n1, @n2, @n3)
//
// into:
//
// p: Phi(@x)
//
// if each @ni in {@n1, @n2, @n3} is either equal to @p to is equal
// to @x, for exactly one other @x. Additionally, trivial Phis (i.e.
// p: Phi(@x)) are forwarded, so that if have an edge to such @p, we
// replace it with @x. This loop does this for Phis only; later we do
// such forwarding for Phi references found in other nodes.
//
// See Aycock and Horspool in CC'00 for a better description of what
// we're doing here.
do {
m_changed = false;
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned phiIndex = block->phis.size(); phiIndex--;) {
Node* phi = block->phis[phiIndex];
if (phi->variableAccessData()->isCaptured())
continue;
forwardPhiChildren(phi);
deduplicateChildren(phi);
}
}
} while (m_changed);
// For each basic block, for each local live at the head of that block,
// figure out what node we should be referring to instead of that local.
// If it turns out to be a non-trivial Phi, make sure that we create an
// SSA Phi and Upsilons in predecessor blocks. We reuse
// BasicBlock::variablesAtHead for tracking which nodes to refer to.
Operands<bool> nonTrivialPhis(OperandsLike, m_graph.block(0)->variablesAtHead);
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
nonTrivialPhis.fill(false);
for (unsigned i = block->phis.size(); i--;) {
Node* phi = block->phis[i];
if (!phi->children.justOneChild())
nonTrivialPhis.operand(phi->local()) = true;
}
for (unsigned i = block->variablesAtHead.size(); i--;) {
Node* node = block->variablesAtHead[i];
if (!node)
continue;
if (verbose)
dataLog("At block #", blockIndex, " for operand r", block->variablesAtHead.operandForIndex(i), " have node ", node, "\n");
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured()) {
// Poison this entry in variablesAtHead because we don't
// want anyone to try to refer to it, if the variable is
// captured.
block->variablesAtHead[i] = 0;
continue;
}
switch (node->op()) {
case Phi:
case SetArgument:
break;
case Flush:
case GetLocal:
case PhantomLocal:
node = node->child1().node();
break;
default:
RELEASE_ASSERT_NOT_REACHED();
}
RELEASE_ASSERT(node->op() == Phi || node->op() == SetArgument);
bool isFlushed = !!(node->flags() & NodeIsFlushed);
if (node->op() == Phi) {
if (!nonTrivialPhis.operand(node->local())) {
Edge edge = node->children.justOneChild();
ASSERT(edge);
if (verbose)
dataLog(" One child: ", edge, ", ", RawPointer(edge.node()), "\n");
node = edge.node(); // It's something from a different basic block.
} else {
if (verbose)
dataLog(" Non-trivial.\n");
// It's a non-trivial Phi.
FlushFormat format = variable->flushFormat();
NodeFlags result = resultFor(format);
UseKind useKind = useKindFor(format);
node = m_insertionSet.insertNode(0, SpecNone, Phi, NodeOrigin());
if (verbose)
dataLog(" Inserted new node: ", node, "\n");
node->mergeFlags(result);
RELEASE_ASSERT((node->flags() & NodeResultMask) == result);
for (unsigned j = block->predecessors.size(); j--;) {
BasicBlock* predecessor = block->predecessors[j];
predecessor->appendNonTerminal(
m_graph, SpecNone, Upsilon, predecessor->last()->origin,
OpInfo(node), Edge(predecessor->variablesAtTail[i], useKind));
}
if (isFlushed) {
// Do nothing. For multiple reasons.
// Reason #1: If the local is flushed then we don't need to bother
// with a MovHint since every path to this point in the code will
// have flushed the bytecode variable using a SetLocal and hence
// the Availability::flushedAt() will agree, and that will be
// sufficient for figuring out how to recover the variable's value.
// Reason #2: If we had inserted a MovHint and the Phi function had
// died (because the only user of the value was the "flush" - i.e.
// some asynchronous runtime thingy) then the MovHint would turn
// into a ZombieHint, which would fool us into thinking that the
// variable is dead.
// Reason #3: If we had inserted a MovHint then even if the Phi
// stayed alive, we would still end up generating inefficient code
// since we would be telling the OSR exit compiler to use some SSA
// value for the bytecode variable rather than just telling it that
// the value was already on the stack.
} else {
m_insertionSet.insertNode(
0, SpecNone, MovHint, NodeOrigin(),
OpInfo(variable->local().offset()), Edge(node));
}
}
}
block->variablesAtHead[i] = node;
}
m_insertionSet.execute(block);
}
if (verbose) {
dataLog("Variables at head after SSA Phi insertion:\n");
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
dataLog(" ", *block, ": ", block->variablesAtHead, "\n");
}
}
// At this point variablesAtHead in each block refers to either:
//
// 1) A new SSA phi in the current block.
// 2) A SetArgument, which will soon get converted into a GetArgument.
// 3) An old CPS phi in a different block.
//
// We don't have to do anything for (1) and (2), but we do need to
// do a replacement for (3).
// Clear all replacements, since other phases may have used them.
m_graph.clearReplacements();
if (dumpGraph) {
dataLog("Graph just before identifying replacements:\n");
m_graph.dump();
}
// For all of the old CPS Phis, figure out what they correspond to in SSA.
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
if (verbose)
dataLog("Dealing with block #", blockIndex, "\n");
for (unsigned phiIndex = block->phis.size(); phiIndex--;) {
Node* phi = block->phis[phiIndex];
if (verbose) {
dataLog(
"Considering ", phi, " (", RawPointer(phi), "), for r",
phi->local(), ", and its replacement in ", *block, ", ",
block->variablesAtHead.operand(phi->local()), "\n");
}
ASSERT(phi != block->variablesAtHead.operand(phi->local()));
phi->misc.replacement = block->variablesAtHead.operand(phi->local());
}
}
// Now make sure that all variablesAtHead in each block points to the
// canonical SSA value. Prior to this, variablesAtHead[local] may point to
// an old CPS Phi in a different block.
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (size_t i = block->variablesAtHead.size(); i--;) {
Node* node = block->variablesAtHead[i];
if (!node)
continue;
while (node->misc.replacement) {
ASSERT(node != node->misc.replacement);
node = node->misc.replacement;
}
block->variablesAtHead[i] = node;
}
}
if (verbose) {
dataLog("Variables at head after convergence:\n");
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
dataLog(" ", *block, ": ", block->variablesAtHead, "\n");
}
}
// Convert operations over locals into operations over SSA nodes.
// - GetLocal over captured variables lose their phis.
// - GetLocal over uncaptured variables die and get replaced with references
// to the node specified by variablesAtHead.
// - SetLocal gets NodeMustGenerate if it's flushed, or turns into a
// Check otherwise.
// - Flush loses its children and turns into a Phantom.
// - PhantomLocal becomes Phantom, and its child is whatever is specified
// by variablesAtHead.
// - SetArgument turns into GetArgument unless it's a captured variable.
// - Upsilons get their children fixed to refer to the true value of that local
// at the end of the block. Prior to this loop, Upsilons will refer to
// variableAtTail[operand], which may be any of Flush, PhantomLocal, GetLocal,
// SetLocal, SetArgument, or Phi. We accomplish this by setting the
// replacement pointers of all of those nodes to refer to either
// variablesAtHead[operand], or the child of the SetLocal.
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned phiIndex = block->phis.size(); phiIndex--;) {
block->phis[phiIndex]->misc.replacement =
block->variablesAtHead.operand(block->phis[phiIndex]->local());
}
for (unsigned nodeIndex = block->size(); nodeIndex--;)
ASSERT(!block->at(nodeIndex)->misc.replacement);
for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
Node* node = block->at(nodeIndex);
m_graph.performSubstitution(node);
switch (node->op()) {
case SetLocal: {
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured() || !!(node->flags() & NodeIsFlushed))
node->mergeFlags(NodeMustGenerate);
else
node->setOpAndDefaultFlags(Check);
node->misc.replacement = node->child1().node(); // Only for Upsilons.
break;
}
case GetLocal: {
// It seems tempting to just do forwardPhi(GetLocal), except that we
// could have created a new (SSA) Phi, and the GetLocal could still be
// referring to an old (CPS) Phi. Uses variablesAtHead to tell us what
// to refer to.
node->children.reset();
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured())
break;
node->convertToPhantom();
node->misc.replacement = block->variablesAtHead.operand(variable->local());
break;
}
case Flush: {
node->children.reset();
node->convertToPhantom();
// This is only for Upsilons. An Upsilon will only refer to a Flush if
// there were no SetLocals or GetLocals in the block.
node->misc.replacement = block->variablesAtHead.operand(node->local());
break;
}
case PhantomLocal: {
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured())
break;
node->child1().setNode(block->variablesAtHead.operand(variable->local()));
node->convertToPhantom();
// This is only for Upsilons. An Upsilon will only refer to a
// PhantomLocal if there were no SetLocals or GetLocals in the block.
node->misc.replacement = block->variablesAtHead.operand(variable->local());
break;
}
case SetArgument: {
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured())
break;
node->setOpAndDefaultFlags(GetArgument);
node->mergeFlags(resultFor(node->variableAccessData()->flushFormat()));
break;
}
default:
break;
}
}
}
// Free all CPS phis and reset variables vectors.
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned phiIndex = block->phis.size(); phiIndex--;)
m_graph.m_allocator.free(block->phis[phiIndex]);
block->phis.clear();
block->variablesAtHead.clear();
block->variablesAtTail.clear();
block->valuesAtHead.clear();
block->valuesAtHead.clear();
block->ssa = adoptPtr(new BasicBlock::SSAData(block));
}
m_graph.m_arguments.clear();
m_graph.m_form = SSA;
return true;
}
void doRoundOfDoubleVoting()
{
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog("Voting on double uses of locals [%u]\n", m_count);
#endif
for (unsigned i = 0; i < m_graph.m_variableAccessData.size(); ++i)
m_graph.m_variableAccessData[i].find()->clearVotes();
for (m_compileIndex = 0; m_compileIndex < m_graph.size(); ++m_compileIndex) {
Node& node = m_graph[m_compileIndex];
switch (node.op()) {
case ValueAdd:
case ArithAdd:
case ArithSub: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
DoubleBallot ballot;
if (isNumberSpeculation(left) && isNumberSpeculation(right)
&& !m_graph.addShouldSpeculateInteger(node))
ballot = VoteDouble;
else
ballot = VoteValue;
m_graph.vote(node.child1(), ballot);
m_graph.vote(node.child2(), ballot);
break;
}
case ArithMul: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
DoubleBallot ballot;
if (isNumberSpeculation(left) && isNumberSpeculation(right)
&& !m_graph.mulShouldSpeculateInteger(node))
ballot = VoteDouble;
else
ballot = VoteValue;
m_graph.vote(node.child1(), ballot);
m_graph.vote(node.child2(), ballot);
break;
}
case ArithMin:
case ArithMax:
case ArithMod:
case ArithDiv: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
DoubleBallot ballot;
if (isNumberSpeculation(left) && isNumberSpeculation(right)
&& !(Node::shouldSpeculateInteger(m_graph[node.child1()], m_graph[node.child1()])
&& node.canSpeculateInteger()))
ballot = VoteDouble;
else
ballot = VoteValue;
m_graph.vote(node.child1(), ballot);
m_graph.vote(node.child2(), ballot);
break;
}
case ArithAbs:
DoubleBallot ballot;
if (!(m_graph[node.child1()].shouldSpeculateInteger()
&& node.canSpeculateInteger()))
ballot = VoteDouble;
else
ballot = VoteValue;
m_graph.vote(node.child1(), ballot);
break;
case ArithSqrt:
m_graph.vote(node.child1(), VoteDouble);
break;
case SetLocal: {
SpeculatedType prediction = m_graph[node.child1()].prediction();
if (isDoubleSpeculation(prediction))
node.variableAccessData()->vote(VoteDouble);
else if (!isNumberSpeculation(prediction) || isInt32Speculation(prediction))
node.variableAccessData()->vote(VoteValue);
break;
}
default:
m_graph.vote(node, VoteValue);
break;
}
}
for (unsigned i = 0; i < m_graph.m_variableAccessData.size(); ++i) {
VariableAccessData* variableAccessData = &m_graph.m_variableAccessData[i];
if (!variableAccessData->isRoot())
continue;
if (operandIsArgument(variableAccessData->local())
|| variableAccessData->isCaptured())
continue;
m_changed |= variableAccessData->tallyVotesForShouldUseDoubleFormat();
}
for (unsigned i = 0; i < m_graph.m_argumentPositions.size(); ++i)
m_changed |= m_graph.m_argumentPositions[i].mergeArgumentAwareness();
for (unsigned i = 0; i < m_graph.m_variableAccessData.size(); ++i) {
VariableAccessData* variableAccessData = &m_graph.m_variableAccessData[i];
if (!variableAccessData->isRoot())
continue;
if (operandIsArgument(variableAccessData->local())
|| variableAccessData->isCaptured())
continue;
m_changed |= variableAccessData->makePredictionForDoubleFormat();
}
}
void process(BlockIndex blockIndex)
{
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
return;
m_live = block->ssa->flushFormatAtTail;
for (unsigned nodeIndex = block->size(); nodeIndex--;) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case SetLocal: {
VariableAccessData* variable = node->variableAccessData();
setForNode(node, variable->local(), variable->flushFormat(), DeadFlush);
break;
}
case GetArgument: {
VariableAccessData* variable = node->variableAccessData();
setForNode(node, variable->local(), variable->flushFormat(), FlushedJSValue);
break;
}
case Flush:
case GetLocal: {
VariableAccessData* variable = node->variableAccessData();
FlushFormat format = variable->flushFormat();
setForNode(node, variable->local(), format, format);
break;
}
default:
break;
}
}
if (m_live == block->ssa->flushFormatAtHead)
return;
m_changed = true;
block->ssa->flushFormatAtHead = m_live;
for (unsigned i = block->predecessors.size(); i--;) {
BasicBlock* predecessor = block->predecessors[i];
for (unsigned j = m_live.size(); j--;) {
FlushFormat& predecessorFormat = predecessor->ssa->flushFormatAtTail[j];
FlushFormat myFormat = m_live[j];
// Three possibilities:
// 1) Predecessor format is Dead, in which case it acquires our format.
// 2) Predecessor format is identical to our format, in which case we
// do nothing.
// 3) Predecessor format is different from our format and it's not Dead,
// in which case we have an erroneous set of Flushes and SetLocals.
// FIXME: What if the predecessor was already processed by the fixpoint
// and says "not Dead" and the current block says "Dead"? We may want to
// revisit this, and say that this is is acceptable.
if (predecessorFormat == DeadFlush) {
predecessorFormat = myFormat;
continue;
}
if (predecessorFormat == myFormat)
continue;
dataLog(
"Bad Flush merge at edge ", *predecessor, " -> ", *block,
", local variable r", m_live.operandForIndex(j), ": ", *predecessor,
" has ", predecessorFormat, " and ", *block, " has ", myFormat, ".\n");
dataLog("IR at time of error:\n");
m_graph.dump();
CRASH();
}
}
}