void JITCode::reconstruct(
ExecState* exec, CodeBlock* codeBlock, CodeOrigin codeOrigin, unsigned streamIndex,
Operands<JSValue>& result)
{
Operands<ValueRecovery> recoveries;
reconstruct(codeBlock, codeOrigin, streamIndex, recoveries);
result = Operands<JSValue>(OperandsLike, recoveries);
for (size_t i = result.size(); i--;) {
int operand = result.operandForIndex(i);
if (operandIsArgument(operand)
&& !VirtualRegister(operand).toArgument()
&& codeBlock->codeType() == FunctionCode
&& codeBlock->specializationKind() == CodeForConstruct) {
// Ugh. If we're in a constructor, the 'this' argument may hold garbage. It will
// also never be used. It doesn't matter what we put into the value for this,
// but it has to be an actual value that can be grokked by subsequent DFG passes,
// so we sanitize it here by turning it into Undefined.
result[i] = jsUndefined();
continue;
}
ValueRecovery recovery = recoveries[i];
JSValue value;
switch (recovery.technique()) {
case AlreadyInJSStack:
case AlreadyInJSStackAsUnboxedCell:
case AlreadyInJSStackAsUnboxedBoolean:
value = exec->r(operand).jsValue();
break;
case AlreadyInJSStackAsUnboxedInt32:
value = jsNumber(exec->r(operand).unboxedInt32());
break;
case AlreadyInJSStackAsUnboxedInt52:
value = jsNumber(exec->r(operand).unboxedInt52());
break;
case AlreadyInJSStackAsUnboxedDouble:
value = jsDoubleNumber(exec->r(operand).unboxedDouble());
break;
case Constant:
value = recovery.constant();
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
result[i] = value;
}
}
void doRoundOfDoubleVoting()
{
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLog("Voting on double uses of locals [%u]\n", m_count);
#endif
for (unsigned i = 0; i < m_graph.m_variableAccessData.size(); ++i)
m_graph.m_variableAccessData[i].find()->clearVotes();
for (m_compileIndex = 0; m_compileIndex < m_graph.size(); ++m_compileIndex) {
Node& node = m_graph[m_compileIndex];
switch (node.op()) {
case ValueAdd:
case ArithAdd:
case ArithSub: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
DoubleBallot ballot;
if (isNumberSpeculation(left) && isNumberSpeculation(right)
&& !m_graph.addShouldSpeculateInteger(node))
ballot = VoteDouble;
else
ballot = VoteValue;
m_graph.vote(node.child1(), ballot);
m_graph.vote(node.child2(), ballot);
break;
}
case ArithMul: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
DoubleBallot ballot;
if (isNumberSpeculation(left) && isNumberSpeculation(right)
&& !m_graph.mulShouldSpeculateInteger(node))
ballot = VoteDouble;
else
ballot = VoteValue;
m_graph.vote(node.child1(), ballot);
m_graph.vote(node.child2(), ballot);
break;
}
case ArithMin:
case ArithMax:
case ArithMod:
case ArithDiv: {
SpeculatedType left = m_graph[node.child1()].prediction();
SpeculatedType right = m_graph[node.child2()].prediction();
DoubleBallot ballot;
if (isNumberSpeculation(left) && isNumberSpeculation(right)
&& !(Node::shouldSpeculateInteger(m_graph[node.child1()], m_graph[node.child1()])
&& node.canSpeculateInteger()))
ballot = VoteDouble;
else
ballot = VoteValue;
m_graph.vote(node.child1(), ballot);
m_graph.vote(node.child2(), ballot);
break;
}
case ArithAbs:
DoubleBallot ballot;
if (!(m_graph[node.child1()].shouldSpeculateInteger()
&& node.canSpeculateInteger()))
ballot = VoteDouble;
else
ballot = VoteValue;
m_graph.vote(node.child1(), ballot);
break;
case ArithSqrt:
m_graph.vote(node.child1(), VoteDouble);
break;
case SetLocal: {
SpeculatedType prediction = m_graph[node.child1()].prediction();
if (isDoubleSpeculation(prediction))
node.variableAccessData()->vote(VoteDouble);
else if (!isNumberSpeculation(prediction) || isInt32Speculation(prediction))
node.variableAccessData()->vote(VoteValue);
break;
}
default:
m_graph.vote(node, VoteValue);
break;
}
}
for (unsigned i = 0; i < m_graph.m_variableAccessData.size(); ++i) {
VariableAccessData* variableAccessData = &m_graph.m_variableAccessData[i];
if (!variableAccessData->isRoot())
continue;
if (operandIsArgument(variableAccessData->local())
|| variableAccessData->isCaptured())
continue;
m_changed |= variableAccessData->tallyVotesForShouldUseDoubleFormat();
}
for (unsigned i = 0; i < m_graph.m_argumentPositions.size(); ++i)
m_changed |= m_graph.m_argumentPositions[i].mergeArgumentAwareness();
for (unsigned i = 0; i < m_graph.m_variableAccessData.size(); ++i) {
VariableAccessData* variableAccessData = &m_graph.m_variableAccessData[i];
if (!variableAccessData->isRoot())
continue;
if (operandIsArgument(variableAccessData->local())
|| variableAccessData->isCaptured())
continue;
m_changed |= variableAccessData->makePredictionForDoubleFormat();
}
}
void Graph::dump(const char* prefix, NodeIndex nodeIndex)
{
Node& node = at(nodeIndex);
NodeType op = node.op();
unsigned refCount = node.refCount();
bool skipped = !refCount;
bool mustGenerate = node.mustGenerate();
if (mustGenerate)
--refCount;
dataLog("%s", prefix);
printNodeWhiteSpace(node);
// Example/explanation of dataflow dump output
//
// 14: <!2:7> GetByVal(@3, @13)
// ^1 ^2 ^3 ^4 ^5
//
// (1) The nodeIndex of this operation.
// (2) The reference count. The number printed is the 'real' count,
// not including the 'mustGenerate' ref. If the node is
// 'mustGenerate' then the count it prefixed with '!'.
// (3) The virtual register slot assigned to this node.
// (4) The name of the operation.
// (5) The arguments to the operation. The may be of the form:
// @# - a NodeIndex referencing a prior node in the graph.
// arg# - an argument number.
// $# - the index in the CodeBlock of a constant { for numeric constants the value is displayed | for integers, in both decimal and hex }.
// id# - the index in the CodeBlock of an identifier { if codeBlock is passed to dump(), the string representation is displayed }.
// var# - the index of a var on the global object, used by GetGlobalVar/PutGlobalVar operations.
dataLog("% 4d:%s<%c%u:", (int)nodeIndex, skipped ? " skipped " : " ", mustGenerate ? '!' : ' ', refCount);
if (node.hasResult() && !skipped && node.hasVirtualRegister())
dataLog("%u", node.virtualRegister());
else
dataLog("-");
dataLog(">\t%s(", opName(op));
bool hasPrinted = false;
if (node.flags() & NodeHasVarArgs) {
for (unsigned childIdx = node.firstChild(); childIdx < node.firstChild() + node.numChildren(); childIdx++) {
if (hasPrinted)
dataLog(", ");
else
hasPrinted = true;
dataLog("%s@%u%s",
useKindToString(m_varArgChildren[childIdx].useKind()),
m_varArgChildren[childIdx].index(),
speculationToAbbreviatedString(
at(m_varArgChildren[childIdx]).prediction()));
}
} else {
if (!!node.child1()) {
dataLog("%s@%u%s",
useKindToString(node.child1().useKind()),
node.child1().index(),
speculationToAbbreviatedString(at(node.child1()).prediction()));
}
if (!!node.child2()) {
dataLog(", %s@%u%s",
useKindToString(node.child2().useKind()),
node.child2().index(),
speculationToAbbreviatedString(at(node.child2()).prediction()));
}
if (!!node.child3()) {
dataLog(", %s@%u%s",
useKindToString(node.child3().useKind()),
node.child3().index(),
speculationToAbbreviatedString(at(node.child3()).prediction()));
}
hasPrinted = !!node.child1();
}
if (strlen(nodeFlagsAsString(node.flags()))) {
dataLog("%s%s", hasPrinted ? ", " : "", nodeFlagsAsString(node.flags()));
hasPrinted = true;
}
if (node.hasArrayMode()) {
dataLog("%s%s", hasPrinted ? ", " : "", modeToString(node.arrayMode()));
hasPrinted = true;
}
if (node.hasVarNumber()) {
dataLog("%svar%u", hasPrinted ? ", " : "", node.varNumber());
hasPrinted = true;
}
if (node.hasRegisterPointer()) {
dataLog(
"%sglobal%u(%p)", hasPrinted ? ", " : "",
globalObjectFor(node.codeOrigin)->findRegisterIndex(node.registerPointer()),
node.registerPointer());
hasPrinted = true;
}
if (node.hasIdentifier()) {
dataLog("%sid%u{%s}", hasPrinted ? ", " : "", node.identifierNumber(), m_codeBlock->identifier(node.identifierNumber()).ustring().utf8().data());
hasPrinted = true;
}
if (node.hasStructureSet()) {
for (size_t i = 0; i < node.structureSet().size(); ++i) {
dataLog("%sstruct(%p)", hasPrinted ? ", " : "", node.structureSet()[i]);
hasPrinted = true;
}
}
if (node.hasStructure()) {
dataLog("%sstruct(%p)", hasPrinted ? ", " : "", node.structure());
hasPrinted = true;
}
if (node.hasStructureTransitionData()) {
dataLog("%sstruct(%p -> %p)", hasPrinted ? ", " : "", node.structureTransitionData().previousStructure, node.structureTransitionData().newStructure);
hasPrinted = true;
}
if (node.hasStorageAccessData()) {
StorageAccessData& storageAccessData = m_storageAccessData[node.storageAccessDataIndex()];
dataLog("%sid%u{%s}", hasPrinted ? ", " : "", storageAccessData.identifierNumber, m_codeBlock->identifier(storageAccessData.identifierNumber).ustring().utf8().data());
dataLog(", %lu", static_cast<unsigned long>(storageAccessData.offset));
hasPrinted = true;
}
ASSERT(node.hasVariableAccessData() == node.hasLocal());
if (node.hasVariableAccessData()) {
VariableAccessData* variableAccessData = node.variableAccessData();
int operand = variableAccessData->operand();
if (operandIsArgument(operand))
dataLog("%sarg%u(%s)", hasPrinted ? ", " : "", operandToArgument(operand), nameOfVariableAccessData(variableAccessData));
else
dataLog("%sr%u(%s)", hasPrinted ? ", " : "", operand, nameOfVariableAccessData(variableAccessData));
hasPrinted = true;
}
if (node.hasConstantBuffer()) {
if (hasPrinted)
dataLog(", ");
dataLog("%u:[", node.startConstant());
for (unsigned i = 0; i < node.numConstants(); ++i) {
if (i)
dataLog(", ");
dataLog("%s", m_codeBlock->constantBuffer(node.startConstant())[i].description());
}
dataLog("]");
hasPrinted = true;
}
if (op == JSConstant) {
dataLog("%s$%u", hasPrinted ? ", " : "", node.constantNumber());
JSValue value = valueOfJSConstant(nodeIndex);
dataLog(" = %s", value.description());
hasPrinted = true;
}
if (op == WeakJSConstant) {
dataLog("%s%p", hasPrinted ? ", " : "", node.weakConstant());
hasPrinted = true;
}
if (node.isBranch() || node.isJump()) {
dataLog("%sT:#%u", hasPrinted ? ", " : "", node.takenBlockIndex());
hasPrinted = true;
}
if (node.isBranch()) {
dataLog("%sF:#%u", hasPrinted ? ", " : "", node.notTakenBlockIndex());
hasPrinted = true;
}
dataLog("%sbc#%u", hasPrinted ? ", " : "", node.codeOrigin.bytecodeIndex);
hasPrinted = true;
(void)hasPrinted;
dataLog(")");
if (!skipped) {
if (node.hasVariableAccessData())
dataLog(" predicting %s%s", speculationToString(node.variableAccessData()->prediction()), node.variableAccessData()->shouldUseDoubleFormat() ? ", forcing double" : "");
else if (node.hasHeapPrediction())
dataLog(" predicting %s", speculationToString(node.getHeapPrediction()));
}
dataLog("\n");
}
void Graph::dump(NodeIndex nodeIndex, CodeBlock* codeBlock)
{
Node& node = at(nodeIndex);
NodeType op = node.op;
unsigned refCount = node.refCount();
if (!refCount)
return;
bool mustGenerate = node.mustGenerate();
if (mustGenerate)
--refCount;
// Example/explanation of dataflow dump output
//
// 14: <!2:7> GetByVal(@3, @13)
// ^1 ^2 ^3 ^4 ^5
//
// (1) The nodeIndex of this operation.
// (2) The reference count. The number printed is the 'real' count,
// not including the 'mustGenerate' ref. If the node is
// 'mustGenerate' then the count it prefixed with '!'.
// (3) The virtual register slot assigned to this node.
// (4) The name of the operation.
// (5) The arguments to the operation. The may be of the form:
// @# - a NodeIndex referencing a prior node in the graph.
// arg# - an argument number.
// $# - the index in the CodeBlock of a constant { for numeric constants the value is displayed | for integers, in both decimal and hex }.
// id# - the index in the CodeBlock of an identifier { if codeBlock is passed to dump(), the string representation is displayed }.
// var# - the index of a var on the global object, used by GetGlobalVar/PutGlobalVar operations.
printf("% 4d:\t<%c%u:", (int)nodeIndex, mustGenerate ? '!' : ' ', refCount);
if (node.hasResult())
printf("%u", node.virtualRegister());
else
printf("-");
printf(">\t%s(", dfgOpNames[op & NodeIdMask]);
if (node.child1 != NoNode)
printf("@%u", node.child1);
if (node.child2 != NoNode)
printf(", @%u", node.child2);
if (node.child3 != NoNode)
printf(", @%u", node.child3);
bool hasPrinted = node.child1 != NoNode;
if (node.hasVarNumber()) {
printf("%svar%u", hasPrinted ? ", " : "", node.varNumber());
hasPrinted = true;
}
if (node.hasIdentifier()) {
if (codeBlock)
printf("%sid%u{%s}", hasPrinted ? ", " : "", node.identifierNumber(), codeBlock->identifier(node.identifierNumber()).ustring().utf8().data());
else
printf("%sid%u", hasPrinted ? ", " : "", node.identifierNumber());
hasPrinted = true;
}
if (node.hasLocal()) {
int local = node.local();
if (operandIsArgument(local))
printf("%sarg%u", hasPrinted ? ", " : "", local - codeBlock->thisRegister());
else
printf("%sr%u", hasPrinted ? ", " : "", local);
hasPrinted = true;
}
if (op == Int32Constant) {
printf("%s$%u{%d|0x%08x}", hasPrinted ? ", " : "", node.constantNumber(), node.int32Constant(), node.int32Constant());
hasPrinted = true;
}
if (op == DoubleConstant) {
printf("%s$%u{%f})", hasPrinted ? ", " : "", node.constantNumber(), node.numericConstant());
hasPrinted = true;
}
if (op == JSConstant) {
printf("%s$%u", hasPrinted ? ", " : "", node.constantNumber());
hasPrinted = true;
}
if (node.isBranch() || node.isJump()) {
printf("%sT:#%u", hasPrinted ? ", " : "", blockIndexForBytecodeOffset(node.takenBytecodeOffset()));
hasPrinted = true;
}
if (node.isBranch()) {
printf("%sF:#%u", hasPrinted ? ", " : "", blockIndexForBytecodeOffset(node.notTakenBytecodeOffset()));
hasPrinted = true;
}
printf(")\n");
}
inline bool compile(CompileMode compileMode, ExecState* exec, CodeBlock* codeBlock, JITCode& jitCode, MacroAssemblerCodePtr* jitCodeWithArityCheck, unsigned osrEntryBytecodeIndex)
{
SamplingRegion samplingRegion("DFG Compilation (Driver)");
numCompilations++;
ASSERT(codeBlock);
ASSERT(codeBlock->alternative());
ASSERT(codeBlock->alternative()->getJITType() == JITCode::BaselineJIT);
ASSERT(osrEntryBytecodeIndex != UINT_MAX);
if (!Options::useDFGJIT())
return false;
if (!Options::bytecodeRangeToDFGCompile().isInRange(codeBlock->instructionCount()))
return false;
if (logCompilationChanges())
dataLog("DFG compiling ", *codeBlock, ", number of instructions = ", codeBlock->instructionCount(), "\n");
// Derive our set of must-handle values. The compilation must be at least conservative
// enough to allow for OSR entry with these values.
unsigned numVarsWithValues;
if (osrEntryBytecodeIndex)
numVarsWithValues = codeBlock->m_numVars;
else
numVarsWithValues = 0;
Operands<JSValue> mustHandleValues(codeBlock->numParameters(), numVarsWithValues);
for (size_t i = 0; i < mustHandleValues.size(); ++i) {
int operand = mustHandleValues.operandForIndex(i);
if (operandIsArgument(operand)
&& !operandToArgument(operand)
&& compileMode == CompileFunction
&& codeBlock->specializationKind() == CodeForConstruct) {
// Ugh. If we're in a constructor, the 'this' argument may hold garbage. It will
// also never be used. It doesn't matter what we put into the value for this,
// but it has to be an actual value that can be grokked by subsequent DFG passes,
// so we sanitize it here by turning it into Undefined.
mustHandleValues[i] = jsUndefined();
} else
mustHandleValues[i] = exec->uncheckedR(operand).jsValue();
}
Graph dfg(exec->vm(), codeBlock, osrEntryBytecodeIndex, mustHandleValues);
if (!parse(exec, dfg))
return false;
// By this point the DFG bytecode parser will have potentially mutated various tables
// in the CodeBlock. This is a good time to perform an early shrink, which is more
// powerful than a late one. It's safe to do so because we haven't generated any code
// that references any of the tables directly, yet.
codeBlock->shrinkToFit(CodeBlock::EarlyShrink);
if (validationEnabled())
validate(dfg);
performCPSRethreading(dfg);
performUnification(dfg);
performPredictionInjection(dfg);
if (validationEnabled())
validate(dfg);
performBackwardsPropagation(dfg);
performPredictionPropagation(dfg);
performFixup(dfg);
performTypeCheckHoisting(dfg);
dfg.m_fixpointState = FixpointNotConverged;
performCSE(dfg);
performArgumentsSimplification(dfg);
performCPSRethreading(dfg); // This should usually be a no-op since CSE rarely dethreads, and arguments simplification rarely does anything.
performCFA(dfg);
performConstantFolding(dfg);
performCFGSimplification(dfg);
dfg.m_fixpointState = FixpointConverged;
performStoreElimination(dfg);
performCPSRethreading(dfg);
performDCE(dfg);
performVirtualRegisterAllocation(dfg);
GraphDumpMode modeForFinalValidate = DumpGraph;
if (verboseCompilationEnabled()) {
dataLogF("Graph after optimization:\n");
dfg.dump();
modeForFinalValidate = DontDumpGraph;
}
if (validationEnabled())
validate(dfg, modeForFinalValidate);
JITCompiler dataFlowJIT(dfg);
bool result;
if (compileMode == CompileFunction) {
ASSERT(jitCodeWithArityCheck);
result = dataFlowJIT.compileFunction(jitCode, *jitCodeWithArityCheck);
} else {
ASSERT(compileMode == CompileOther);
ASSERT(!jitCodeWithArityCheck);
result = dataFlowJIT.compile(jitCode);
}
return result;
}
static CompilationResult compileImpl(
ExecState* exec, CodeBlock* codeBlock, CompilationMode mode,
unsigned osrEntryBytecodeIndex, PassRefPtr<DeferredCompilationCallback> callback,
Worklist* worklist)
{
SamplingRegion samplingRegion("DFG Compilation (Driver)");
numCompilations++;
ASSERT(codeBlock);
ASSERT(codeBlock->alternative());
ASSERT(codeBlock->alternative()->jitType() == JITCode::BaselineJIT);
ASSERT(osrEntryBytecodeIndex != UINT_MAX);
if (!Options::useDFGJIT() || !MacroAssembler::supportsFloatingPoint())
return CompilationFailed;
if (!Options::bytecodeRangeToDFGCompile().isInRange(codeBlock->instructionCount()))
return CompilationFailed;
if (logCompilationChanges())
dataLog("DFG(Driver) compiling ", *codeBlock, ", number of instructions = ", codeBlock->instructionCount(), "\n");
VM& vm = exec->vm();
// Make sure that any stubs that the DFG is going to use are initialized. We want to
// make sure that al JIT code generation does finalization on the main thread.
vm.getCTIStub(osrExitGenerationThunkGenerator);
vm.getCTIStub(throwExceptionFromCallSlowPathGenerator);
vm.getCTIStub(linkCallThunkGenerator);
vm.getCTIStub(linkConstructThunkGenerator);
vm.getCTIStub(linkClosureCallThunkGenerator);
vm.getCTIStub(virtualCallThunkGenerator);
vm.getCTIStub(virtualConstructThunkGenerator);
#if ENABLE(FTL_JIT)
vm.getCTIStub(FTL::osrExitGenerationThunkGenerator);
#endif
// Derive our set of must-handle values. The compilation must be at least conservative
// enough to allow for OSR entry with these values.
unsigned numVarsWithValues;
if (osrEntryBytecodeIndex)
numVarsWithValues = codeBlock->m_numVars;
else
numVarsWithValues = 0;
RefPtr<Plan> plan = adoptRef(
new Plan(codeBlock, mode, osrEntryBytecodeIndex, numVarsWithValues));
for (size_t i = 0; i < plan->mustHandleValues.size(); ++i) {
int operand = plan->mustHandleValues.operandForIndex(i);
if (operandIsArgument(operand)
&& !operandToArgument(operand)
&& codeBlock->codeType() == FunctionCode
&& codeBlock->specializationKind() == CodeForConstruct) {
// Ugh. If we're in a constructor, the 'this' argument may hold garbage. It will
// also never be used. It doesn't matter what we put into the value for this,
// but it has to be an actual value that can be grokked by subsequent DFG passes,
// so we sanitize it here by turning it into Undefined.
plan->mustHandleValues[i] = jsUndefined();
} else
plan->mustHandleValues[i] = exec->uncheckedR(operand).jsValue();
}
if (worklist) {
plan->callback = callback;
if (logCompilationChanges())
dataLog("Deferring DFG compilation of ", *codeBlock, " with queue length ", worklist->queueLength(), ".\n");
worklist->enqueue(plan);
return CompilationDeferred;
}
plan->compileInThread(*vm.dfgState);
return plan->finalizeWithoutNotifyingCallback();
}