bool fixSSA(Procedure& proc)
{
PhaseScope phaseScope(proc, "fixSSA");
// Collect the stack "variables". If there aren't any, then we don't have anything to do.
// That's a fairly common case.
HashMap<StackSlotValue*, Type> stackVariable;
for (Value* value : proc.values()) {
if (StackSlotValue* stack = value->as<StackSlotValue>()) {
if (stack->kind() == StackSlotKind::Anonymous)
stackVariable.add(stack, Void);
}
}
if (stackVariable.isEmpty())
return false;
// Make sure that we know how to optimize all of these. We only know how to handle Load and
// Store on anonymous variables.
for (Value* value : proc.values()) {
auto reject = [&] (Value* value) {
if (StackSlotValue* stack = value->as<StackSlotValue>())
stackVariable.remove(stack);
};
auto handleAccess = [&] (Value* access, Type type) {
StackSlotValue* stack = access->lastChild()->as<StackSlotValue>();
if (!stack)
return;
if (value->as<MemoryValue>()->offset()) {
stackVariable.remove(stack);
return;
}
auto result = stackVariable.find(stack);
if (result == stackVariable.end())
return;
if (result->value == Void) {
result->value = type;
return;
}
if (result->value == type)
return;
stackVariable.remove(result);
};
switch (value->opcode()) {
case Load:
// We're OK with loads from stack variables at an offset of zero.
handleAccess(value, value->type());
break;
case Store:
// We're OK with stores to stack variables, but not storing stack variables.
reject(value->child(0));
handleAccess(value, value->child(0)->type());
break;
default:
for (Value* child : value->children())
reject(child);
break;
}
}
Vector<StackSlotValue*> deadValues;
for (auto& entry : stackVariable) {
if (entry.value == Void)
deadValues.append(entry.key);
}
for (StackSlotValue* deadValue : deadValues) {
deadValue->replaceWithNop();
stackVariable.remove(deadValue);
}
if (stackVariable.isEmpty())
return false;
// We know that we have variables to optimize, so do that now.
breakCriticalEdges(proc);
SSACalculator ssa(proc);
// Create a SSACalculator::Variable for every stack variable.
Vector<StackSlotValue*> variableToStack;
HashMap<StackSlotValue*, SSACalculator::Variable*> stackToVariable;
for (auto& entry : stackVariable) {
StackSlotValue* stack = entry.key;
SSACalculator::Variable* variable = ssa.newVariable();
RELEASE_ASSERT(variable->index() == variableToStack.size());
variableToStack.append(stack);
stackToVariable.add(stack, variable);
}
// Create Defs for all of the stores to the stack variable.
for (BasicBlock* block : proc) {
for (Value* value : *block) {
if (value->opcode() != Store)
continue;
StackSlotValue* stack = value->child(1)->as<StackSlotValue>();
if (!stack)
continue;
if (SSACalculator::Variable* variable = stackToVariable.get(stack))
ssa.newDef(variable, block, value->child(0));
}
}
// Decide where Phis are to be inserted. This creates them but does not insert them.
ssa.computePhis(
[&] (SSACalculator::Variable* variable, BasicBlock* block) -> Value* {
StackSlotValue* stack = variableToStack[variable->index()];
Value* phi = proc.add<Value>(Phi, stackVariable.get(stack), stack->origin());
if (verbose) {
dataLog(
"Adding Phi for ", pointerDump(stack), " at ", *block, ": ",
deepDump(proc, phi), "\n");
}
return phi;
});
// Now perform the conversion.
InsertionSet insertionSet(proc);
HashMap<StackSlotValue*, Value*> mapping;
for (BasicBlock* block : proc.blocksInPreOrder()) {
mapping.clear();
for (auto& entry : stackToVariable) {
StackSlotValue* stack = entry.key;
SSACalculator::Variable* variable = entry.value;
SSACalculator::Def* def = ssa.reachingDefAtHead(block, variable);
if (def)
mapping.set(stack, def->value());
}
for (SSACalculator::Def* phiDef : ssa.phisForBlock(block)) {
StackSlotValue* stack = variableToStack[phiDef->variable()->index()];
insertionSet.insertValue(0, phiDef->value());
mapping.set(stack, phiDef->value());
}
for (unsigned valueIndex = 0; valueIndex < block->size(); ++valueIndex) {
Value* value = block->at(valueIndex);
value->performSubstitution();
switch (value->opcode()) {
case Load: {
if (StackSlotValue* stack = value->child(0)->as<StackSlotValue>()) {
if (Value* replacement = mapping.get(stack))
value->replaceWithIdentity(replacement);
}
break;
}
case Store: {
if (StackSlotValue* stack = value->child(1)->as<StackSlotValue>()) {
if (stackToVariable.contains(stack)) {
mapping.set(stack, value->child(0));
value->replaceWithNop();
}
}
break;
}
default:
break;
}
}
unsigned upsilonInsertionPoint = block->size() - 1;
Origin upsilonOrigin = block->last()->origin();
for (BasicBlock* successorBlock : block->successorBlocks()) {
for (SSACalculator::Def* phiDef : ssa.phisForBlock(successorBlock)) {
Value* phi = phiDef->value();
SSACalculator::Variable* variable = phiDef->variable();
StackSlotValue* stack = variableToStack[variable->index()];
Value* mappedValue = mapping.get(stack);
if (verbose) {
dataLog(
"Mapped value for ", *stack, " with successor Phi ", *phi, " at end of ",
*block, ": ", pointerDump(mappedValue), "\n");
}
if (!mappedValue)
mappedValue = insertionSet.insertBottom(upsilonInsertionPoint, phi);
insertionSet.insert<UpsilonValue>(
upsilonInsertionPoint, upsilonOrigin, mappedValue, phi);
}
}
insertionSet.execute(block);
}
// Finally, kill the stack slots.
for (StackSlotValue* stack : variableToStack)
stack->replaceWithNop();
if (verbose) {
dataLog("B3 after SSA conversion:\n");
dataLog(proc);
}
return true;
}
bool run()
{
RELEASE_ASSERT(m_graph.m_form == ThreadedCPS);
m_graph.clearReplacements();
m_graph.m_dominators.computeIfNecessary(m_graph);
if (verbose) {
dataLog("Graph before SSA transformation:\n");
m_graph.dump();
}
// Create a SSACalculator::Variable for every root VariableAccessData.
for (VariableAccessData& variable : m_graph.m_variableAccessData) {
if (!variable.isRoot() || variable.isCaptured())
continue;
SSACalculator::Variable* ssaVariable = m_calculator.newVariable();
ASSERT(ssaVariable->index() == m_variableForSSAIndex.size());
m_variableForSSAIndex.append(&variable);
m_ssaVariableForVariable.add(&variable, ssaVariable);
}
// Find all SetLocals and create Defs for them. We handle SetArgument by creating a
// GetLocal, and recording the flush format.
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
// Must process the block in forward direction because we want to see the last
// assignment for every local.
for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
Node* node = block->at(nodeIndex);
if (node->op() != SetLocal && node->op() != SetArgument)
continue;
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured())
continue;
Node* childNode;
if (node->op() == SetLocal)
childNode = node->child1().node();
else {
ASSERT(node->op() == SetArgument);
childNode = m_insertionSet.insertNode(
nodeIndex, node->variableAccessData()->prediction(),
GetStack, node->origin,
OpInfo(m_graph.m_stackAccessData.add(variable->local(), variable->flushFormat())));
if (!ASSERT_DISABLED)
m_argumentGetters.add(childNode);
m_argumentMapping.add(node, childNode);
}
m_calculator.newDef(
m_ssaVariableForVariable.get(variable), block, childNode);
}
m_insertionSet.execute(block);
}
// Decide where Phis are to be inserted. This creates the Phi's but doesn't insert them
// yet. We will later know where to insert them because SSACalculator is such a bro.
m_calculator.computePhis(
[&] (SSACalculator::Variable* ssaVariable, BasicBlock* block) -> Node* {
VariableAccessData* variable = m_variableForSSAIndex[ssaVariable->index()];
// Prune by liveness. This doesn't buy us much other than compile times.
Node* headNode = block->variablesAtHead.operand(variable->local());
if (!headNode)
return nullptr;
// There is the possibiltiy of "rebirths". The SSA calculator will already prune
// rebirths for the same VariableAccessData. But it will not be able to prune
// rebirths that arose from the same local variable number but a different
// VariableAccessData. We do that pruning here.
//
// Here's an example of a rebirth that this would catch:
//
// var x;
// if (foo) {
// if (bar) {
// x = 42;
// } else {
// x = 43;
// }
// print(x);
// x = 44;
// } else {
// x = 45;
// }
// print(x); // Without this check, we'd have a Phi for x = 42|43 here.
//
// FIXME: Consider feeding local variable numbers, not VariableAccessData*'s, as
// the "variables" for SSACalculator. That would allow us to eliminate this
// special case.
// https://bugs.webkit.org/show_bug.cgi?id=136641
if (headNode->variableAccessData() != variable)
return nullptr;
Node* phiNode = m_graph.addNode(
variable->prediction(), Phi, NodeOrigin());
FlushFormat format = variable->flushFormat();
NodeFlags result = resultFor(format);
phiNode->mergeFlags(result);
return phiNode;
});
if (verbose) {
dataLog("Computed Phis, about to transform the graph.\n");
dataLog("\n");
dataLog("Graph:\n");
m_graph.dump();
dataLog("\n");
dataLog("Mappings:\n");
for (unsigned i = 0; i < m_variableForSSAIndex.size(); ++i)
dataLog(" ", i, ": ", VariableAccessDataDump(m_graph, m_variableForSSAIndex[i]), "\n");
dataLog("\n");
dataLog("SSA calculator: ", m_calculator, "\n");
}
// Do the bulk of the SSA conversion. For each block, this tracks the operand->Node
// mapping based on a combination of what the SSACalculator tells us, and us walking over
// the block in forward order. We use our own data structure, valueForOperand, for
// determining the local mapping, but we rely on SSACalculator for the non-local mapping.
//
// This does three things at once:
//
// - Inserts the Phis in all of the places where they need to go. We've already created
// them and they are accounted for in the SSACalculator's data structures, but we
// haven't inserted them yet, mostly because we want to insert all of a block's Phis in
// one go to amortize the cost of node insertion.
//
// - Create and insert Upsilons.
//
// - Convert all of the preexisting SSA nodes (other than the old CPS Phi nodes) into SSA
// form by replacing as follows:
//
// - MovHint has KillLocal prepended to it.
//
// - GetLocal over captured variables lose their phis and become GetStack.
//
// - GetLocal over uncaptured variables die and get replaced with references to the node
// specified by valueForOperand.
//
// - SetLocal turns into PutStack if it's flushed, or turns into a Check otherwise.
//
// - Flush loses its children and turns into a Phantom.
//
// - PhantomLocal becomes Phantom, and its child is whatever is specified by
// valueForOperand.
//
// - SetArgument is removed. Note that GetStack nodes have already been inserted.
Operands<Node*> valueForOperand(OperandsLike, m_graph.block(0)->variablesAtHead);
for (BasicBlock* block : m_graph.blocksInPreOrder()) {
valueForOperand.clear();
// CPS will claim that the root block has all arguments live. But we have already done
// the first step of SSA conversion: argument locals are no longer live at head;
// instead we have GetStack nodes for extracting the values of arguments. So, we
// skip the at-head available value calculation for the root block.
if (block != m_graph.block(0)) {
for (size_t i = valueForOperand.size(); i--;) {
Node* nodeAtHead = block->variablesAtHead[i];
if (!nodeAtHead)
continue;
VariableAccessData* variable = nodeAtHead->variableAccessData();
if (variable->isCaptured())
continue;
if (verbose)
dataLog("Considering live variable ", VariableAccessDataDump(m_graph, variable), " at head of block ", *block, "\n");
SSACalculator::Variable* ssaVariable = m_ssaVariableForVariable.get(variable);
SSACalculator::Def* def = m_calculator.reachingDefAtHead(block, ssaVariable);
if (!def) {
// If we are required to insert a Phi, then we won't have a reaching def
// at head.
continue;
}
Node* node = def->value();
if (node->replacement) {
// This will occur when a SetLocal had a GetLocal as its source. The
// GetLocal would get replaced with an actual SSA value by the time we get
// here. Note that the SSA value with which the GetLocal got replaced
// would not in turn have a replacement.
node = node->replacement;
ASSERT(!node->replacement);
}
if (verbose)
dataLog("Mapping: ", VirtualRegister(valueForOperand.operandForIndex(i)), " -> ", node, "\n");
valueForOperand[i] = node;
}
}
// Insert Phis by asking the calculator what phis there are in this block. Also update
// valueForOperand with those Phis. For Phis associated with variables that are not
// flushed, we also insert a MovHint.
size_t phiInsertionPoint = 0;
for (SSACalculator::Def* phiDef : m_calculator.phisForBlock(block)) {
VariableAccessData* variable = m_variableForSSAIndex[phiDef->variable()->index()];
m_insertionSet.insert(phiInsertionPoint, phiDef->value());
valueForOperand.operand(variable->local()) = phiDef->value();
m_insertionSet.insertNode(
phiInsertionPoint, SpecNone, MovHint, NodeOrigin(),
OpInfo(variable->local().offset()), phiDef->value()->defaultEdge());
}
for (unsigned nodeIndex = 0; nodeIndex < block->size(); ++nodeIndex) {
Node* node = block->at(nodeIndex);
if (verbose) {
dataLog("Processing node ", node, ":\n");
m_graph.dump(WTF::dataFile(), " ", node);
}
m_graph.performSubstitution(node);
switch (node->op()) {
case MovHint: {
m_insertionSet.insertNode(
nodeIndex, SpecNone, KillStack, node->origin,
OpInfo(node->unlinkedLocal().offset()));
break;
}
case SetLocal: {
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured() || !!(node->flags() & NodeIsFlushed)) {
node->convertToPutStack(
m_graph.m_stackAccessData.add(
variable->local(), variable->flushFormat()));
} else
node->setOpAndDefaultFlags(Check);
if (!variable->isCaptured()) {
if (verbose)
dataLog("Mapping: ", variable->local(), " -> ", node->child1().node(), "\n");
valueForOperand.operand(variable->local()) = node->child1().node();
}
break;
}
case GetStack: {
ASSERT(m_argumentGetters.contains(node));
valueForOperand.operand(node->stackAccessData()->local) = node;
break;
}
case GetLocal: {
VariableAccessData* variable = node->variableAccessData();
node->children.reset();
if (variable->isCaptured()) {
node->convertToGetStack(m_graph.m_stackAccessData.add(variable->local(), variable->flushFormat()));
break;
}
node->convertToPhantom();
if (verbose)
dataLog("Replacing node ", node, " with ", valueForOperand.operand(variable->local()), "\n");
node->replacement = valueForOperand.operand(variable->local());
break;
}
case Flush: {
node->children.reset();
node->convertToPhantom();
break;
}
case PhantomLocal: {
ASSERT(node->child1().useKind() == UntypedUse);
VariableAccessData* variable = node->variableAccessData();
if (variable->isCaptured()) {
// This is a fun case. We could have a captured variable that had some
// or all of its uses strength reduced to phantoms rather than flushes.
// SSA conversion will currently still treat it as flushed, in the sense
// that it will just keep the SetLocal. Therefore, there is nothing that
// needs to be done here: we don't need to also keep the source value
// alive. And even if we did want to keep the source value alive, we
// wouldn't be able to, because the variablesAtHead value for a captured
// local wouldn't have been computed by the Phi reduction algorithm
// above.
node->children.reset();
} else
node->child1() = valueForOperand.operand(variable->local())->defaultEdge();
node->convertToPhantom();
break;
}
case SetArgument: {
node->convertToPhantom();
break;
}
default:
break;
}
}
// We want to insert Upsilons just before the end of the block. On the surface this
// seems dangerous because the Upsilon will have a checking UseKind. But, we will not
// actually be performing the check at the point of the Upsilon; the check will
// already have been performed at the point where the original SetLocal was.
size_t upsilonInsertionPoint = block->size() - 1;
NodeOrigin upsilonOrigin = block->last()->origin;
for (unsigned successorIndex = block->numSuccessors(); successorIndex--;) {
BasicBlock* successorBlock = block->successor(successorIndex);
for (SSACalculator::Def* phiDef : m_calculator.phisForBlock(successorBlock)) {
Node* phiNode = phiDef->value();
SSACalculator::Variable* ssaVariable = phiDef->variable();
VariableAccessData* variable = m_variableForSSAIndex[ssaVariable->index()];
FlushFormat format = variable->flushFormat();
UseKind useKind = useKindFor(format);
m_insertionSet.insertNode(
upsilonInsertionPoint, SpecNone, Upsilon, upsilonOrigin,
OpInfo(phiNode), Edge(
valueForOperand.operand(variable->local()),
useKind));
}
}
m_insertionSet.execute(block);
}
// Free all CPS phis and reset variables vectors.
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned phiIndex = block->phis.size(); phiIndex--;)
m_graph.m_allocator.free(block->phis[phiIndex]);
block->phis.clear();
block->variablesAtHead.clear();
block->variablesAtTail.clear();
block->valuesAtHead.clear();
block->valuesAtHead.clear();
block->ssa = std::make_unique<BasicBlock::SSAData>(block);
}
m_graph.m_argumentFormats.resize(m_graph.m_arguments.size());
for (unsigned i = m_graph.m_arguments.size(); i--;) {
FlushFormat format = FlushedJSValue;
Node* node = m_argumentMapping.get(m_graph.m_arguments[i]);
// m_argumentMapping.get could return null for a captured local. That's fine. We only
// track the argument loads of those arguments for which we speculate type. We don't
// speculate type for captured arguments.
if (node)
format = node->stackAccessData()->format;
m_graph.m_argumentFormats[i] = format;
m_graph.m_arguments[i] = node; // Record the load that loads the arguments for the benefit of exit profiling.
}
m_graph.m_form = SSA;
if (verbose) {
dataLog("Graph after SSA transformation:\n");
m_graph.dump();
}
return true;
}