JSCell* JIT_OPERATION operationCreateDirectArgumentsDuringExit(ExecState* exec, InlineCallFrame* inlineCallFrame, JSFunction* callee, int32_t argumentCount)
{
VM& vm = exec->vm();
NativeCallFrameTracer target(&vm, exec);
DeferGCForAWhile deferGC(vm.heap);
CodeBlock* codeBlock;
if (inlineCallFrame)
codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame);
else
codeBlock = exec->codeBlock();
unsigned length = argumentCount - 1;
unsigned capacity = std::max(length, static_cast<unsigned>(codeBlock->numParameters() - 1));
DirectArguments* result = DirectArguments::create(
vm, codeBlock->globalObject()->directArgumentsStructure(), length, capacity);
result->callee().set(vm, result, callee);
Register* arguments =
exec->registers() + (inlineCallFrame ? inlineCallFrame->stackOffset : 0) +
CallFrame::argumentOffset(0);
for (unsigned i = length; i--;)
result->setIndexQuickly(vm, i, arguments[i].jsValue());
return result;
}
JSCell* JIT_OPERATION operationCreateClonedArgumentsDuringExit(ExecState* exec, InlineCallFrame* inlineCallFrame, JSFunction* callee, int32_t argumentCount)
{
VM& vm = exec->vm();
NativeCallFrameTracer target(&vm, exec);
DeferGCForAWhile deferGC(vm.heap);
CodeBlock* codeBlock;
if (inlineCallFrame)
codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame);
else
codeBlock = exec->codeBlock();
unsigned length = argumentCount - 1;
ClonedArguments* result = ClonedArguments::createEmpty(
vm, codeBlock->globalObject()->outOfBandArgumentsStructure(), callee);
Register* arguments =
exec->registers() + (inlineCallFrame ? inlineCallFrame->stackOffset : 0) +
CallFrame::argumentOffset(0);
for (unsigned i = length; i--;)
result->putDirectIndex(exec, i, arguments[i].jsValue());
result->putDirect(vm, vm.propertyNames->length, jsNumber(length));
return result;
}
static void appendSourceToError(CallFrame* callFrame, ErrorInstance* exception, unsigned bytecodeOffset)
{
ErrorInstance::SourceAppender appender = exception->sourceAppender();
exception->clearSourceAppender();
RuntimeType type = exception->runtimeTypeForCause();
exception->clearRuntimeTypeForCause();
if (!callFrame->codeBlock()->hasExpressionInfo())
return;
int startOffset = 0;
int endOffset = 0;
int divotPoint = 0;
unsigned line = 0;
unsigned column = 0;
CodeBlock* codeBlock;
CodeOrigin codeOrigin = callFrame->codeOrigin();
if (codeOrigin && codeOrigin.inlineCallFrame)
codeBlock = baselineCodeBlockForInlineCallFrame(codeOrigin.inlineCallFrame);
else
codeBlock = callFrame->codeBlock();
codeBlock->expressionRangeForBytecodeOffset(bytecodeOffset, divotPoint, startOffset, endOffset, line, column);
int expressionStart = divotPoint - startOffset;
int expressionStop = divotPoint + endOffset;
StringView sourceString = codeBlock->source()->source();
if (!expressionStop || expressionStart > static_cast<int>(sourceString.length()))
return;
VM* vm = &callFrame->vm();
JSValue jsMessage = exception->getDirect(*vm, vm->propertyNames->message);
if (!jsMessage || !jsMessage.isString())
return;
String message = asString(jsMessage)->value(callFrame);
if (expressionStart < expressionStop)
message = appender(message, codeBlock->source()->getRange(expressionStart, expressionStop).toString(), type, ErrorInstance::FoundExactSource);
else {
// No range information, so give a few characters of context.
int dataLength = sourceString.length();
int start = expressionStart;
int stop = expressionStart;
// Get up to 20 characters of context to the left and right of the divot, clamping to the line.
// Then strip whitespace.
while (start > 0 && (expressionStart - start < 20) && sourceString[start - 1] != '\n')
start--;
while (start < (expressionStart - 1) && isStrWhiteSpace(sourceString[start]))
start++;
while (stop < dataLength && (stop - expressionStart < 20) && sourceString[stop] != '\n')
stop++;
while (stop > expressionStart && isStrWhiteSpace(sourceString[stop - 1]))
stop--;
message = appender(message, codeBlock->source()->getRange(start, stop).toString(), type, ErrorInstance::FoundApproximateSource);
}
exception->putDirect(*vm, vm->propertyNames->message, jsString(vm, message));
}
static unsigned computeNumVariablesForCodeOrigin(
CodeBlock* codeBlock, const CodeOrigin& codeOrigin)
{
if (!codeOrigin.inlineCallFrame)
return codeBlock->m_numCalleeRegisters;
return
codeOrigin.inlineCallFrame->stackOffset +
baselineCodeBlockForInlineCallFrame(codeOrigin.inlineCallFrame)->m_numCalleeRegisters;
}
bool run()
{
ScoreBoard scoreBoard(m_graph.m_nextMachineLocal);
scoreBoard.assertClear();
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
bool needsNewLine = false;
#endif
for (size_t blockIndex = 0; blockIndex < m_graph.numBlocks(); ++blockIndex) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
if (!block->isReachable)
continue;
for (size_t indexInBlock = 0; indexInBlock < block->size(); ++indexInBlock) {
Node* node = block->at(indexInBlock);
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
if (needsNewLine)
dataLogF("\n");
dataLogF(" @%u:", node->index());
needsNewLine = true;
#endif
if (!node->shouldGenerate())
continue;
switch (node->op()) {
case Phi:
case Flush:
case PhantomLocal:
continue;
case GetLocal:
ASSERT(!node->child1()->hasResult());
break;
default:
break;
}
// First, call use on all of the current node's children, then
// allocate a VirtualRegister for this node. We do so in this
// order so that if a child is on its last use, and a
// VirtualRegister is freed, then it may be reused for node.
if (node->flags() & NodeHasVarArgs) {
for (unsigned childIdx = node->firstChild(); childIdx < node->firstChild() + node->numChildren(); childIdx++)
scoreBoard.useIfHasResult(m_graph.m_varArgChildren[childIdx]);
} else {
scoreBoard.useIfHasResult(node->child1());
scoreBoard.useIfHasResult(node->child2());
scoreBoard.useIfHasResult(node->child3());
}
if (!node->hasResult())
continue;
VirtualRegister virtualRegister = scoreBoard.allocate();
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
dataLogF(
" Assigning virtual register %u to node %u.",
virtualRegister, node->index());
#endif
node->setVirtualRegister(virtualRegister);
// 'mustGenerate' nodes have their useCount artificially elevated,
// call use now to account for this.
if (node->mustGenerate())
scoreBoard.use(node);
}
scoreBoard.assertClear();
}
#if DFG_ENABLE(DEBUG_PROPAGATION_VERBOSE)
if (needsNewLine)
dataLogF("\n");
#endif
// Record the number of virtual registers we're using. This is used by calls
// to figure out where to put the parameters.
m_graph.m_nextMachineLocal = scoreBoard.highWatermark();
// 'm_numCalleeRegisters' is the number of locals and temporaries allocated
// for the function (and checked for on entry). Since we perform a new and
// different allocation of temporaries, more registers may now be required.
// This also accounts for the number of temporaries that may be needed if we
// OSR exit, due to inlining. Hence this computes the number of temporaries
// that could be used by this code block even if it exits; it may be more
// than what this code block needs if it never exits.
unsigned calleeRegisters = scoreBoard.highWatermark() + m_graph.m_parameterSlots;
for (InlineCallFrameSet::iterator iter = m_graph.m_inlineCallFrames->begin(); !!iter; ++iter) {
InlineCallFrame* inlineCallFrame = *iter;
CodeBlock* codeBlock = baselineCodeBlockForInlineCallFrame(inlineCallFrame);
unsigned requiredCalleeRegisters = VirtualRegister(inlineCallFrame->stackOffset).toLocal() + 1 + codeBlock->m_numCalleeRegisters;
if (requiredCalleeRegisters > calleeRegisters)
calleeRegisters = requiredCalleeRegisters;
}
if ((unsigned)codeBlock()->m_numCalleeRegisters < calleeRegisters)
codeBlock()->m_numCalleeRegisters = calleeRegisters;
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLogF("Num callee registers: %u\n", calleeRegisters);
#endif
return true;
}
void VariableEventStream::reconstruct(
CodeBlock* codeBlock, CodeOrigin codeOrigin, MinifiedGraph& graph,
unsigned index, Operands<ValueRecovery>& valueRecoveries) const
{
ASSERT(codeBlock->jitType() == JITCode::DFGJIT);
CodeBlock* baselineCodeBlock = codeBlock->baselineVersion();
unsigned numVariables;
if (codeOrigin.inlineCallFrame)
numVariables = baselineCodeBlockForInlineCallFrame(codeOrigin.inlineCallFrame)->m_numCalleeRegisters + VirtualRegister(codeOrigin.inlineCallFrame->stackOffset).toLocal() + 1;
else
numVariables = baselineCodeBlock->m_numCalleeRegisters;
// Crazy special case: if we're at index == 0 then this must be an argument check
// failure, in which case all variables are already set up. The recoveries should
// reflect this.
if (!index) {
valueRecoveries = Operands<ValueRecovery>(codeBlock->numParameters(), numVariables);
for (size_t i = 0; i < valueRecoveries.size(); ++i) {
valueRecoveries[i] = ValueRecovery::displacedInJSStack(
VirtualRegister(valueRecoveries.operandForIndex(i)), DataFormatJS);
}
return;
}
// Step 1: Find the last checkpoint, and figure out the number of virtual registers as we go.
unsigned startIndex = index - 1;
while (at(startIndex).kind() != Reset)
startIndex--;
// Step 2: Create a mock-up of the DFG's state and execute the events.
Operands<ValueSource> operandSources(codeBlock->numParameters(), numVariables);
for (unsigned i = operandSources.size(); i--;)
operandSources[i] = ValueSource(SourceIsDead);
HashMap<MinifiedID, MinifiedGenerationInfo> generationInfos;
for (unsigned i = startIndex; i < index; ++i) {
const VariableEvent& event = at(i);
switch (event.kind()) {
case Reset:
// nothing to do.
break;
case BirthToFill:
case BirthToSpill:
case Birth: {
MinifiedGenerationInfo info;
info.update(event);
generationInfos.add(event.id(), info);
break;
}
case Fill:
case Spill:
case Death: {
HashMap<MinifiedID, MinifiedGenerationInfo>::iterator iter = generationInfos.find(event.id());
ASSERT(iter != generationInfos.end());
iter->value.update(event);
break;
}
case MovHintEvent:
if (operandSources.hasOperand(event.bytecodeRegister()))
operandSources.setOperand(event.bytecodeRegister(), ValueSource(event.id()));
break;
case SetLocalEvent:
if (operandSources.hasOperand(event.bytecodeRegister()))
operandSources.setOperand(event.bytecodeRegister(), ValueSource::forDataFormat(event.machineRegister(), event.dataFormat()));
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
}
// Step 3: Compute value recoveries!
valueRecoveries = Operands<ValueRecovery>(codeBlock->numParameters(), numVariables);
for (unsigned i = 0; i < operandSources.size(); ++i) {
ValueSource& source = operandSources[i];
if (source.isTriviallyRecoverable()) {
valueRecoveries[i] = source.valueRecovery();
continue;
}
ASSERT(source.kind() == HaveNode);
MinifiedNode* node = graph.at(source.id());
MinifiedGenerationInfo info = generationInfos.get(source.id());
if (!info.alive) {
valueRecoveries[i] = ValueRecovery::constant(jsUndefined());
continue;
}
if (tryToSetConstantRecovery(valueRecoveries[i], node))
continue;
ASSERT(info.format != DataFormatNone);
if (info.filled) {
if (info.format == DataFormatDouble) {
valueRecoveries[i] = ValueRecovery::inFPR(info.u.fpr, DataFormatDouble);
continue;
}
#if USE(JSVALUE32_64)
if (info.format & DataFormatJS) {
valueRecoveries[i] = ValueRecovery::inPair(info.u.pair.tagGPR, info.u.pair.payloadGPR);
continue;
}
#endif
valueRecoveries[i] = ValueRecovery::inGPR(info.u.gpr, info.format);
continue;
}
valueRecoveries[i] =
ValueRecovery::displacedInJSStack(static_cast<VirtualRegister>(info.u.virtualReg), info.format);
}
}
void VariableEventStream::reconstruct(
CodeBlock* codeBlock, CodeOrigin codeOrigin, MinifiedGraph& graph,
unsigned index, Operands<ValueRecovery>& valueRecoveries) const
{
ASSERT(codeBlock->getJITType() == JITCode::DFGJIT);
CodeBlock* baselineCodeBlock = codeBlock->baselineVersion();
unsigned numVariables;
if (codeOrigin.inlineCallFrame)
numVariables = baselineCodeBlockForInlineCallFrame(codeOrigin.inlineCallFrame)->m_numCalleeRegisters + codeOrigin.inlineCallFrame->stackOffset;
else
numVariables = baselineCodeBlock->m_numCalleeRegisters;
// Crazy special case: if we're at index == 0 then this must be an argument check
// failure, in which case all variables are already set up. The recoveries should
// reflect this.
if (!index) {
valueRecoveries = Operands<ValueRecovery>(codeBlock->numParameters(), numVariables);
for (size_t i = 0; i < valueRecoveries.size(); ++i)
valueRecoveries[i] = ValueRecovery::alreadyInJSStack();
return;
}
// Step 1: Find the last checkpoint, and figure out the number of virtual registers as we go.
unsigned startIndex = index - 1;
while (at(startIndex).kind() != Reset)
startIndex--;
#if DFG_ENABLE(DEBUG_VERBOSE)
dataLogF("Computing OSR exit recoveries starting at seq#%u.\n", startIndex);
#endif
// Step 2: Create a mock-up of the DFG's state and execute the events.
Operands<ValueSource> operandSources(codeBlock->numParameters(), numVariables);
Vector<MinifiedGenerationInfo, 32> generationInfos(graph.originalGraphSize());
for (unsigned i = startIndex; i < index; ++i) {
const VariableEvent& event = at(i);
switch (event.kind()) {
case Reset:
// nothing to do.
break;
case BirthToFill:
case BirthToSpill:
case Fill:
case Spill:
case Death:
generationInfos[event.nodeIndex()].update(event);
break;
case MovHint:
if (operandSources.hasOperand(event.operand()))
operandSources.setOperand(event.operand(), ValueSource(event.nodeIndex()));
break;
case SetLocalEvent:
if (operandSources.hasOperand(event.operand()))
operandSources.setOperand(event.operand(), ValueSource::forDataFormat(event.dataFormat()));
break;
default:
RELEASE_ASSERT_NOT_REACHED();
break;
}
}
// Step 3: Record the things that are live, so we can get to them more quickly.
Vector<unsigned, 16> indicesOfLiveThings;
for (unsigned i = 0; i < generationInfos.size(); ++i) {
if (generationInfos[i].format != DataFormatNone)
indicesOfLiveThings.append(i);
}
// Step 4: Compute value recoveries!
valueRecoveries = Operands<ValueRecovery>(codeBlock->numParameters(), numVariables);
for (unsigned i = 0; i < operandSources.size(); ++i) {
ValueSource& source = operandSources[i];
if (source.isTriviallyRecoverable()) {
valueRecoveries[i] = source.valueRecovery();
continue;
}
ASSERT(source.kind() == HaveNode);
MinifiedNode* node = graph.at(source.nodeIndex());
if (node) {
if (node->hasConstantNumber()) {
valueRecoveries[i] = ValueRecovery::constant(
codeBlock->constantRegister(
FirstConstantRegisterIndex + node->constantNumber()).get());
continue;
}
if (node->hasWeakConstant()) {
valueRecoveries[i] = ValueRecovery::constant(node->weakConstant());
continue;
}
if (node->op() == PhantomArguments) {
valueRecoveries[i] = ValueRecovery::argumentsThatWereNotCreated();
continue;
}
}
MinifiedGenerationInfo* info = &generationInfos[source.nodeIndex()];
if (info->format == DataFormatNone) {
// Try to see if there is an alternate node that would contain the value we want.
// There are four possibilities:
//
// Int32ToDouble: We can use this in place of the original node, but
// we'd rather not; so we use it only if it is the only remaining
// live version.
//
// ValueToInt32: If the only remaining live version of the value is
// ValueToInt32, then we can use it.
//
// UInt32ToNumber: If the only live version of the value is a UInt32ToNumber
// then the only remaining uses are ones that want a properly formed number
// rather than a UInt32 intermediate.
//
// DoubleAsInt32: Same as UInt32ToNumber.
//
// The reverse of the above: This node could be a UInt32ToNumber, but its
// alternative is still alive. This means that the only remaining uses of
// the number would be fine with a UInt32 intermediate.
bool found = false;
if (node && node->op() == UInt32ToNumber) {
NodeIndex nodeIndex = node->child1();
node = graph.at(nodeIndex);
info = &generationInfos[nodeIndex];
if (info->format != DataFormatNone)
found = true;
}
if (!found) {
NodeIndex int32ToDoubleIndex = NoNode;
NodeIndex valueToInt32Index = NoNode;
NodeIndex uint32ToNumberIndex = NoNode;
NodeIndex doubleAsInt32Index = NoNode;
for (unsigned i = 0; i < indicesOfLiveThings.size(); ++i) {
NodeIndex nodeIndex = indicesOfLiveThings[i];
node = graph.at(nodeIndex);
if (!node)
continue;
if (!node->hasChild1())
continue;
if (node->child1() != source.nodeIndex())
continue;
ASSERT(generationInfos[nodeIndex].format != DataFormatNone);
switch (node->op()) {
case Int32ToDouble:
int32ToDoubleIndex = nodeIndex;
break;
case ValueToInt32:
valueToInt32Index = nodeIndex;
break;
case UInt32ToNumber:
uint32ToNumberIndex = nodeIndex;
break;
case DoubleAsInt32:
doubleAsInt32Index = nodeIndex;
break;
default:
break;
}
}
NodeIndex nodeIndexToUse;
if (doubleAsInt32Index != NoNode)
nodeIndexToUse = doubleAsInt32Index;
else if (int32ToDoubleIndex != NoNode)
nodeIndexToUse = int32ToDoubleIndex;
else if (valueToInt32Index != NoNode)
nodeIndexToUse = valueToInt32Index;
else if (uint32ToNumberIndex != NoNode)
nodeIndexToUse = uint32ToNumberIndex;
else
nodeIndexToUse = NoNode;
if (nodeIndexToUse != NoNode) {
info = &generationInfos[nodeIndexToUse];
ASSERT(info->format != DataFormatNone);
found = true;
}
}
if (!found) {
valueRecoveries[i] = ValueRecovery::constant(jsUndefined());
continue;
}
}
ASSERT(info->format != DataFormatNone);
if (info->filled) {
if (info->format == DataFormatDouble) {
valueRecoveries[i] = ValueRecovery::inFPR(info->u.fpr);
continue;
}
#if USE(JSVALUE32_64)
if (info->format & DataFormatJS) {
valueRecoveries[i] = ValueRecovery::inPair(info->u.pair.tagGPR, info->u.pair.payloadGPR);
continue;
}
#endif
valueRecoveries[i] = ValueRecovery::inGPR(info->u.gpr, info->format);
continue;
}
valueRecoveries[i] =
ValueRecovery::displacedInJSStack(static_cast<VirtualRegister>(info->u.virtualReg), info->format);
}
// Step 5: Make sure that for locals that coincide with true call frame headers, the exit compiler knows
// that those values don't have to be recovered. Signal this by using ValueRecovery::alreadyInJSStack()
for (InlineCallFrame* inlineCallFrame = codeOrigin.inlineCallFrame; inlineCallFrame; inlineCallFrame = inlineCallFrame->caller.inlineCallFrame) {
for (unsigned i = JSStack::CallFrameHeaderSize; i--;)
valueRecoveries.setLocal(inlineCallFrame->stackOffset - i - 1, ValueRecovery::alreadyInJSStack());
}
}
