void fixPartialRegisterStalls(Code& code)
{
if (!isX86())
return;
PhaseScope phaseScope(code, "fixPartialRegisterStalls");
Vector<BasicBlock*> candidates;
for (BasicBlock* block : code) {
for (const Inst& inst : *block) {
if (hasPartialXmmRegUpdate(inst)) {
candidates.append(block);
break;
}
}
}
// Fortunately, Partial Stalls are rarely used. Return early if no block
// cares about them.
if (candidates.isEmpty())
return;
// For each block, this provides the distance to the last instruction setting each register
// on block *entry*.
IndexMap<BasicBlock, FPDefDistance> lastDefDistance(code.size());
// Blocks with dirty distance at head.
IndexSet<BasicBlock> dirty;
// First, we compute the local distance for each block and push it to the successors.
for (BasicBlock* block : code) {
FPDefDistance localDistance;
unsigned distanceToBlockEnd = block->size();
for (Inst& inst : *block)
updateDistances(inst, localDistance, distanceToBlockEnd);
for (BasicBlock* successor : block->successorBlocks()) {
if (lastDefDistance[successor].updateFromPrecessor(localDistance))
dirty.add(successor);
}
}
// Now we propagate the minimums accross blocks.
bool changed;
do {
changed = false;
for (BasicBlock* block : code) {
if (!dirty.remove(block))
continue;
// Little shortcut: if the block is big enough, propagating it won't add any information.
if (block->size() >= minimumSafeDistance)
continue;
unsigned blockSize = block->size();
FPDefDistance& blockDistance = lastDefDistance[block];
for (BasicBlock* successor : block->successorBlocks()) {
if (lastDefDistance[successor].updateFromPrecessor(blockDistance, blockSize)) {
dirty.add(successor);
changed = true;
}
}
}
} while (changed);
// Finally, update each block as needed.
InsertionSet insertionSet(code);
for (BasicBlock* block : candidates) {
unsigned distanceToBlockEnd = block->size();
FPDefDistance& localDistance = lastDefDistance[block];
for (unsigned i = 0; i < block->size(); ++i) {
Inst& inst = block->at(i);
if (hasPartialXmmRegUpdate(inst)) {
RegisterSet defs;
RegisterSet uses;
inst.forEachTmp([&] (Tmp& tmp, Arg::Role role, Arg::Type) {
if (tmp.isFPR()) {
if (Arg::isDef(role))
defs.set(tmp.fpr());
if (Arg::isAnyUse(role))
uses.set(tmp.fpr());
}
});
// We only care about values we define but not use. Otherwise we have to wait
// for the value to be resolved anyway.
defs.exclude(uses);
defs.forEach([&] (Reg reg) {
if (localDistance.distance[MacroAssembler::fpRegisterIndex(reg.fpr())] < minimumSafeDistance)
insertionSet.insert(i, MoveZeroToDouble, inst.origin, Tmp(reg));
});
}
updateDistances(inst, localDistance, distanceToBlockEnd);
}
insertionSet.execute(block);
}
}
void fixupNode(Node& node)
{
if (!node.shouldGenerate())
return;
NodeType op = node.op();
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog(" %s @%u: ", Graph::opName(op), m_compileIndex);
#endif
switch (op) {
case GetById: {
if (!isInt32Speculation(m_graph[m_compileIndex].prediction()))
break;
if (codeBlock()->identifier(node.identifierNumber()) != globalData().propertyNames->length)
break;
bool isArray = isArraySpeculation(m_graph[node.child1()].prediction());
bool isArguments = isArgumentsSpeculation(m_graph[node.child1()].prediction());
bool isString = isStringSpeculation(m_graph[node.child1()].prediction());
bool isInt8Array = m_graph[node.child1()].shouldSpeculateInt8Array();
bool isInt16Array = m_graph[node.child1()].shouldSpeculateInt16Array();
bool isInt32Array = m_graph[node.child1()].shouldSpeculateInt32Array();
bool isUint8Array = m_graph[node.child1()].shouldSpeculateUint8Array();
bool isUint8ClampedArray = m_graph[node.child1()].shouldSpeculateUint8ClampedArray();
bool isUint16Array = m_graph[node.child1()].shouldSpeculateUint16Array();
bool isUint32Array = m_graph[node.child1()].shouldSpeculateUint32Array();
bool isFloat32Array = m_graph[node.child1()].shouldSpeculateFloat32Array();
bool isFloat64Array = m_graph[node.child1()].shouldSpeculateFloat64Array();
if (!isArray && !isArguments && !isString && !isInt8Array && !isInt16Array && !isInt32Array && !isUint8Array && !isUint8ClampedArray && !isUint16Array && !isUint32Array && !isFloat32Array && !isFloat64Array)
break;
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog(" @%u -> %s", m_compileIndex, isArray ? "GetArrayLength" : "GetStringLength");
#endif
if (isArray) {
node.setOp(GetArrayLength);
ASSERT(node.flags() & NodeMustGenerate);
node.clearFlags(NodeMustGenerate);
m_graph.deref(m_compileIndex);
ArrayProfile* arrayProfile =
m_graph.baselineCodeBlockFor(node.codeOrigin)->getArrayProfile(
node.codeOrigin.bytecodeIndex);
if (!arrayProfile)
break;
arrayProfile->computeUpdatedPrediction();
if (!arrayProfile->hasDefiniteStructure())
break;
m_graph.ref(node.child1());
Node checkStructure(CheckStructure, node.codeOrigin, OpInfo(m_graph.addStructureSet(arrayProfile->expectedStructure())), node.child1().index());
checkStructure.ref();
NodeIndex checkStructureIndex = m_graph.size();
m_graph.append(checkStructure);
m_insertionSet.append(m_indexInBlock, checkStructureIndex);
break;
}
if (isArguments)
node.setOp(GetArgumentsLength);
else if (isString)
node.setOp(GetStringLength);
else if (isInt8Array)
node.setOp(GetInt8ArrayLength);
else if (isInt16Array)
node.setOp(GetInt16ArrayLength);
else if (isInt32Array)
node.setOp(GetInt32ArrayLength);
else if (isUint8Array)
node.setOp(GetUint8ArrayLength);
else if (isUint8ClampedArray)
node.setOp(GetUint8ClampedArrayLength);
else if (isUint16Array)
node.setOp(GetUint16ArrayLength);
else if (isUint32Array)
node.setOp(GetUint32ArrayLength);
else if (isFloat32Array)
node.setOp(GetFloat32ArrayLength);
else if (isFloat64Array)
node.setOp(GetFloat64ArrayLength);
else
ASSERT_NOT_REACHED();
// No longer MustGenerate
ASSERT(node.flags() & NodeMustGenerate);
node.clearFlags(NodeMustGenerate);
m_graph.deref(m_compileIndex);
break;
}
case GetIndexedPropertyStorage: {
if (!m_graph[node.child1()].prediction()
|| !m_graph[node.child2()].shouldSpeculateInteger()
|| m_graph[node.child1()].shouldSpeculateArguments()) {
node.setOpAndDefaultFlags(Nop);
m_graph.clearAndDerefChild1(node);
m_graph.clearAndDerefChild2(node);
m_graph.clearAndDerefChild3(node);
node.setRefCount(0);
}
break;
}
case GetByVal:
case StringCharAt:
case StringCharCodeAt: {
if (!!node.child3() && m_graph[node.child3()].op() == Nop)
node.children.child3() = Edge();
break;
}
case ValueToInt32: {
if (m_graph[node.child1()].shouldSpeculateNumber()
&& node.mustGenerate()) {
node.clearFlags(NodeMustGenerate);
m_graph.deref(m_compileIndex);
}
break;
}
case BitAnd:
case BitOr:
case BitXor:
case BitRShift:
case BitLShift:
case BitURShift: {
fixIntEdge(node.children.child1());
fixIntEdge(node.children.child2());
break;
}
case CompareEq:
case CompareLess:
case CompareLessEq:
case CompareGreater:
case CompareGreaterEq:
case CompareStrictEq: {
if (Node::shouldSpeculateInteger(m_graph[node.child1()], m_graph[node.child2()]))
break;
if (!Node::shouldSpeculateNumber(m_graph[node.child1()], m_graph[node.child2()]))
break;
fixDoubleEdge(0);
fixDoubleEdge(1);
break;
}
case LogicalNot: {
if (m_graph[node.child1()].shouldSpeculateInteger())
break;
if (!m_graph[node.child1()].shouldSpeculateNumber())
break;
fixDoubleEdge(0);
break;
}
case Branch: {
if (!m_graph[node.child1()].shouldSpeculateInteger()
&& m_graph[node.child1()].shouldSpeculateNumber())
fixDoubleEdge(0);
Node& myNode = m_graph[m_compileIndex]; // reload because the graph may have changed
Edge logicalNotEdge = myNode.child1();
Node& logicalNot = m_graph[logicalNotEdge];
if (logicalNot.op() == LogicalNot
&& logicalNot.adjustedRefCount() == 1) {
Edge newChildEdge = logicalNot.child1();
if (m_graph[newChildEdge].hasBooleanResult()) {
m_graph.ref(newChildEdge);
m_graph.deref(logicalNotEdge);
myNode.children.setChild1(newChildEdge);
BlockIndex toBeTaken = myNode.notTakenBlockIndex();
BlockIndex toBeNotTaken = myNode.takenBlockIndex();
myNode.setTakenBlockIndex(toBeTaken);
myNode.setNotTakenBlockIndex(toBeNotTaken);
}
}
break;
}
case SetLocal: {
if (node.variableAccessData()->isCaptured())
break;
if (!node.variableAccessData()->shouldUseDoubleFormat())
break;
fixDoubleEdge(0);
break;
}
case ArithAdd:
case ValueAdd: {
if (m_graph.addShouldSpeculateInteger(node))
break;
if (!Node::shouldSpeculateNumber(m_graph[node.child1()], m_graph[node.child2()]))
break;
fixDoubleEdge(0);
fixDoubleEdge(1);
break;
}
case ArithSub: {
if (m_graph.addShouldSpeculateInteger(node)
&& node.canSpeculateInteger())
break;
fixDoubleEdge(0);
fixDoubleEdge(1);
break;
}
case ArithNegate: {
if (m_graph.negateShouldSpeculateInteger(node))
break;
fixDoubleEdge(0);
break;
}
case ArithMin:
case ArithMax:
case ArithMod: {
if (Node::shouldSpeculateInteger(m_graph[node.child1()], m_graph[node.child2()])
&& node.canSpeculateInteger())
break;
fixDoubleEdge(0);
fixDoubleEdge(1);
break;
}
case ArithMul: {
if (m_graph.mulShouldSpeculateInteger(node))
break;
fixDoubleEdge(0);
fixDoubleEdge(1);
break;
}
case ArithDiv: {
if (Node::shouldSpeculateInteger(m_graph[node.child1()], m_graph[node.child2()])
&& node.canSpeculateInteger()) {
if (isX86())
break;
fixDoubleEdge(0);
fixDoubleEdge(1);
Node& oldDivision = m_graph[m_compileIndex];
Node newDivision = oldDivision;
newDivision.setRefCount(2);
newDivision.predict(SpecDouble);
NodeIndex newDivisionIndex = m_graph.size();
oldDivision.setOp(DoubleAsInt32);
oldDivision.children.initialize(Edge(newDivisionIndex, DoubleUse), Edge(), Edge());
m_graph.append(newDivision);
m_insertionSet.append(m_indexInBlock, newDivisionIndex);
break;
}
fixDoubleEdge(0);
fixDoubleEdge(1);
break;
}
case ArithAbs: {
if (m_graph[node.child1()].shouldSpeculateInteger()
&& node.canSpeculateInteger())
break;
fixDoubleEdge(0);
break;
}
case ArithSqrt: {
fixDoubleEdge(0);
break;
}
case PutByVal:
case PutByValSafe: {
Edge child1 = m_graph.varArgChild(node, 0);
Edge child2 = m_graph.varArgChild(node, 1);
Edge child3 = m_graph.varArgChild(node, 2);
if (!m_graph[child1].prediction() || !m_graph[child2].prediction())
break;
if (!m_graph[child2].shouldSpeculateInteger())
break;
if (isActionableIntMutableArraySpeculation(m_graph[child1].prediction())) {
if (m_graph[child3].isConstant())
break;
if (m_graph[child3].shouldSpeculateInteger())
break;
fixDoubleEdge(2);
break;
}
if (isActionableFloatMutableArraySpeculation(m_graph[child1].prediction())) {
fixDoubleEdge(2);
break;
}
break;
}
default:
break;
}
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
if (!(node.flags() & NodeHasVarArgs)) {
dataLog("new children: ");
node.dumpChildren(WTF::dataFile());
}
dataLog("\n");
#endif
}
bool GeneralWxUtils::isX64()
{
return !isX86();
}
