void JITCompiler::noticeOSREntry(BasicBlock& basicBlock, JITCompiler::Label blockHead, LinkBuffer& linkBuffer)
{
// OSR entry is not allowed into blocks deemed unreachable by control flow analysis.
if (!basicBlock.intersectionOfCFAHasVisited)
return;
OSREntryData* entry = m_jitCode->appendOSREntryData(basicBlock.bytecodeBegin, linkBuffer.offsetOf(blockHead));
entry->m_expectedValues = basicBlock.intersectionOfPastValuesAtHead;
// Fix the expected values: in our protocol, a dead variable will have an expected
// value of (None, []). But the old JIT may stash some values there. So we really
// need (Top, TOP).
for (size_t argument = 0; argument < basicBlock.variablesAtHead.numberOfArguments(); ++argument) {
Node* node = basicBlock.variablesAtHead.argument(argument);
if (!node || !node->shouldGenerate())
entry->m_expectedValues.argument(argument).makeHeapTop();
}
for (size_t local = 0; local < basicBlock.variablesAtHead.numberOfLocals(); ++local) {
Node* node = basicBlock.variablesAtHead.local(local);
if (!node || !node->shouldGenerate())
entry->m_expectedValues.local(local).makeHeapTop();
else {
VariableAccessData* variable = node->variableAccessData();
entry->m_machineStackUsed.set(variable->machineLocal().toLocal());
switch (variable->flushFormat()) {
case FlushedDouble:
entry->m_localsForcedDouble.set(local);
break;
case FlushedInt52:
entry->m_localsForcedMachineInt.set(local);
break;
default:
break;
}
if (variable->local() != variable->machineLocal()) {
entry->m_reshufflings.append(
OSREntryReshuffling(
variable->local().offset(), variable->machineLocal().offset()));
}
}
}
entry->m_reshufflings.shrinkToFit();
}
bool run()
{
// This enumerates the locals that we actually care about and packs them. So for example
// if we use local 1, 3, 4, 5, 7, then we remap them: 1->0, 3->1, 4->2, 5->3, 7->4. We
// treat a variable as being "used" if there exists an access to it (SetLocal, GetLocal,
// Flush, PhantomLocal).
BitVector usedLocals;
// Collect those variables that are used from IR.
bool hasNodesThatNeedFixup = false;
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned nodeIndex = block->size(); nodeIndex--;) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case GetLocal:
case SetLocal:
case Flush:
case PhantomLocal: {
VariableAccessData* variable = node->variableAccessData();
if (variable->local().isArgument())
break;
usedLocals.set(variable->local().toLocal());
break;
}
case GetLocalUnlinked: {
VirtualRegister operand = node->unlinkedLocal();
if (operand.isArgument())
break;
usedLocals.set(operand.toLocal());
hasNodesThatNeedFixup = true;
break;
}
case LoadVarargs:
case ForwardVarargs: {
LoadVarargsData* data = node->loadVarargsData();
if (data->count.isLocal())
usedLocals.set(data->count.toLocal());
if (data->start.isLocal()) {
// This part really relies on the contiguity of stack layout
// assignments.
ASSERT(VirtualRegister(data->start.offset() + data->limit - 1).isLocal());
for (unsigned i = data->limit; i--;)
usedLocals.set(VirtualRegister(data->start.offset() + i).toLocal());
} // the else case shouldn't happen.
hasNodesThatNeedFixup = true;
break;
}
case PutStack:
case GetStack: {
StackAccessData* stack = node->stackAccessData();
if (stack->local.isArgument())
break;
usedLocals.set(stack->local.toLocal());
break;
}
default:
break;
}
}
}
for (InlineCallFrameSet::iterator iter = m_graph.m_plan.inlineCallFrames->begin(); !!iter; ++iter) {
InlineCallFrame* inlineCallFrame = *iter;
if (inlineCallFrame->isVarargs()) {
usedLocals.set(VirtualRegister(
JSStack::ArgumentCount + inlineCallFrame->stackOffset).toLocal());
}
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
usedLocals.set(VirtualRegister(
virtualRegisterForArgument(argument).offset() +
inlineCallFrame->stackOffset).toLocal());
}
}
Vector<unsigned> allocation(usedLocals.size());
m_graph.m_nextMachineLocal = 0;
for (unsigned i = 0; i < usedLocals.size(); ++i) {
if (!usedLocals.get(i)) {
allocation[i] = UINT_MAX;
continue;
}
allocation[i] = m_graph.m_nextMachineLocal++;
}
for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
VariableAccessData* variable = &m_graph.m_variableAccessData[i];
if (!variable->isRoot())
continue;
if (variable->local().isArgument()) {
variable->machineLocal() = variable->local();
continue;
}
size_t local = variable->local().toLocal();
if (local >= allocation.size())
continue;
if (allocation[local] == UINT_MAX)
continue;
variable->machineLocal() = assign(allocation, variable->local());
}
for (StackAccessData* data : m_graph.m_stackAccessData) {
if (!data->local.isLocal()) {
data->machineLocal = data->local;
continue;
}
if (static_cast<size_t>(data->local.toLocal()) >= allocation.size())
continue;
if (allocation[data->local.toLocal()] == UINT_MAX)
continue;
data->machineLocal = assign(allocation, data->local);
}
// This register is never valid for DFG code blocks.
codeBlock()->setActivationRegister(VirtualRegister());
if (LIKELY(!m_graph.hasDebuggerEnabled()))
codeBlock()->setScopeRegister(VirtualRegister());
else
codeBlock()->setScopeRegister(assign(allocation, codeBlock()->scopeRegister()));
for (unsigned i = m_graph.m_inlineVariableData.size(); i--;) {
InlineVariableData data = m_graph.m_inlineVariableData[i];
InlineCallFrame* inlineCallFrame = data.inlineCallFrame;
if (inlineCallFrame->isVarargs()) {
inlineCallFrame->argumentCountRegister = assign(
allocation, VirtualRegister(inlineCallFrame->stackOffset + JSStack::ArgumentCount));
}
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
ArgumentPosition& position = m_graph.m_argumentPositions[
data.argumentPositionStart + argument];
VariableAccessData* variable = position.someVariable();
ValueSource source;
if (!variable)
source = ValueSource(SourceIsDead);
else {
source = ValueSource::forFlushFormat(
variable->machineLocal(), variable->flushFormat());
}
inlineCallFrame->arguments[argument] = source.valueRecovery();
}
RELEASE_ASSERT(inlineCallFrame->isClosureCall == !!data.calleeVariable);
if (inlineCallFrame->isClosureCall) {
VariableAccessData* variable = data.calleeVariable->find();
ValueSource source = ValueSource::forFlushFormat(
variable->machineLocal(),
variable->flushFormat());
inlineCallFrame->calleeRecovery = source.valueRecovery();
} else
RELEASE_ASSERT(inlineCallFrame->calleeRecovery.isConstant());
}
// Fix GetLocalUnlinked's variable references.
if (hasNodesThatNeedFixup) {
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned nodeIndex = block->size(); nodeIndex--;) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case GetLocalUnlinked: {
node->setUnlinkedMachineLocal(assign(allocation, node->unlinkedLocal()));
break;
}
case LoadVarargs:
case ForwardVarargs: {
LoadVarargsData* data = node->loadVarargsData();
data->machineCount = assign(allocation, data->count);
data->machineStart = assign(allocation, data->start);
break;
}
default:
break;
}
}
}
}
return true;
}
bool run()
{
SharedSymbolTable* symbolTable = codeBlock()->symbolTable();
// This enumerates the locals that we actually care about and packs them. So for example
// if we use local 1, 3, 4, 5, 7, then we remap them: 1->0, 3->1, 4->2, 5->3, 7->4. We
// treat a variable as being "used" if there exists an access to it (SetLocal, GetLocal,
// Flush, PhantomLocal).
BitVector usedLocals;
// Collect those variables that are used from IR.
bool hasGetLocalUnlinked = false;
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned nodeIndex = block->size(); nodeIndex--;) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case GetLocal:
case SetLocal:
case Flush:
case PhantomLocal: {
VariableAccessData* variable = node->variableAccessData();
if (variable->local().isArgument())
break;
usedLocals.set(variable->local().toLocal());
break;
}
case GetLocalUnlinked: {
VirtualRegister operand = node->unlinkedLocal();
if (operand.isArgument())
break;
usedLocals.set(operand.toLocal());
hasGetLocalUnlinked = true;
break;
}
default:
break;
}
}
}
// Ensure that captured variables and captured inline arguments are pinned down.
// They should have been because of flushes, except that the flushes can be optimized
// away.
if (symbolTable) {
for (int i = symbolTable->captureStart(); i > symbolTable->captureEnd(); i--)
usedLocals.set(VirtualRegister(i).toLocal());
}
if (codeBlock()->usesArguments()) {
usedLocals.set(codeBlock()->argumentsRegister().toLocal());
usedLocals.set(unmodifiedArgumentsRegister(codeBlock()->argumentsRegister()).toLocal());
}
if (codeBlock()->uncheckedActivationRegister().isValid())
usedLocals.set(codeBlock()->activationRegister().toLocal());
for (InlineCallFrameSet::iterator iter = m_graph.m_inlineCallFrames->begin(); !!iter; ++iter) {
InlineCallFrame* inlineCallFrame = *iter;
if (!inlineCallFrame->executable->usesArguments())
continue;
VirtualRegister argumentsRegister = m_graph.argumentsRegisterFor(inlineCallFrame);
usedLocals.set(argumentsRegister.toLocal());
usedLocals.set(unmodifiedArgumentsRegister(argumentsRegister).toLocal());
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
usedLocals.set(VirtualRegister(
virtualRegisterForArgument(argument).offset() +
inlineCallFrame->stackOffset).toLocal());
}
}
Vector<unsigned> allocation(usedLocals.size());
m_graph.m_nextMachineLocal = 0;
for (unsigned i = 0; i < usedLocals.size(); ++i) {
if (!usedLocals.get(i)) {
allocation[i] = UINT_MAX;
continue;
}
allocation[i] = m_graph.m_nextMachineLocal++;
}
for (unsigned i = m_graph.m_variableAccessData.size(); i--;) {
VariableAccessData* variable = &m_graph.m_variableAccessData[i];
if (!variable->isRoot())
continue;
if (variable->local().isArgument()) {
variable->machineLocal() = variable->local();
continue;
}
size_t local = variable->local().toLocal();
if (local >= allocation.size())
continue;
if (allocation[local] == UINT_MAX)
continue;
variable->machineLocal() = virtualRegisterForLocal(
allocation[variable->local().toLocal()]);
}
if (codeBlock()->usesArguments()) {
VirtualRegister argumentsRegister = virtualRegisterForLocal(
allocation[codeBlock()->argumentsRegister().toLocal()]);
RELEASE_ASSERT(
virtualRegisterForLocal(allocation[
unmodifiedArgumentsRegister(
codeBlock()->argumentsRegister()).toLocal()])
== unmodifiedArgumentsRegister(argumentsRegister));
codeBlock()->setArgumentsRegister(argumentsRegister);
}
if (codeBlock()->uncheckedActivationRegister().isValid()) {
codeBlock()->setActivationRegister(
virtualRegisterForLocal(allocation[codeBlock()->activationRegister().toLocal()]));
}
for (unsigned i = m_graph.m_inlineVariableData.size(); i--;) {
InlineVariableData data = m_graph.m_inlineVariableData[i];
InlineCallFrame* inlineCallFrame = data.inlineCallFrame;
if (inlineCallFrame->executable->usesArguments()) {
inlineCallFrame->argumentsRegister = virtualRegisterForLocal(
allocation[m_graph.argumentsRegisterFor(inlineCallFrame).toLocal()]);
RELEASE_ASSERT(
virtualRegisterForLocal(allocation[unmodifiedArgumentsRegister(
m_graph.argumentsRegisterFor(inlineCallFrame)).toLocal()])
== unmodifiedArgumentsRegister(inlineCallFrame->argumentsRegister));
}
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
ArgumentPosition& position = m_graph.m_argumentPositions[
data.argumentPositionStart + argument];
VariableAccessData* variable = position.someVariable();
ValueSource source;
if (!variable)
source = ValueSource(SourceIsDead);
else {
source = ValueSource::forFlushFormat(
variable->machineLocal(), variable->flushFormat());
}
inlineCallFrame->arguments[argument] = source.valueRecovery();
}
RELEASE_ASSERT(inlineCallFrame->isClosureCall == !!data.calleeVariable);
if (inlineCallFrame->isClosureCall) {
ValueSource source = ValueSource::forFlushFormat(
data.calleeVariable->machineLocal(),
data.calleeVariable->flushFormat());
inlineCallFrame->calleeRecovery = source.valueRecovery();
} else
RELEASE_ASSERT(inlineCallFrame->calleeRecovery.isConstant());
}
if (symbolTable) {
if (symbolTable->captureCount()) {
unsigned captureStartLocal = allocation[
VirtualRegister(codeBlock()->symbolTable()->captureStart()).toLocal()];
ASSERT(captureStartLocal != UINT_MAX);
m_graph.m_machineCaptureStart = virtualRegisterForLocal(captureStartLocal).offset();
} else
m_graph.m_machineCaptureStart = virtualRegisterForLocal(0).offset();
// This is an abomination. If we had captured an argument then the argument ends
// up being "slow", meaning that loads of the argument go through an extra lookup
// table.
if (const SlowArgument* slowArguments = symbolTable->slowArguments()) {
auto newSlowArguments = std::make_unique<SlowArgument[]>(
symbolTable->parameterCount());
for (size_t i = symbolTable->parameterCount(); i--;) {
newSlowArguments[i] = slowArguments[i];
VirtualRegister reg = VirtualRegister(slowArguments[i].index);
if (reg.isLocal())
newSlowArguments[i].index = virtualRegisterForLocal(allocation[reg.toLocal()]).offset();
}
m_graph.m_slowArguments = std::move(newSlowArguments);
}
}
// Fix GetLocalUnlinked's variable references.
if (hasGetLocalUnlinked) {
for (BlockIndex blockIndex = m_graph.numBlocks(); blockIndex--;) {
BasicBlock* block = m_graph.block(blockIndex);
if (!block)
continue;
for (unsigned nodeIndex = block->size(); nodeIndex--;) {
Node* node = block->at(nodeIndex);
switch (node->op()) {
case GetLocalUnlinked: {
VirtualRegister operand = node->unlinkedLocal();
if (operand.isLocal())
operand = virtualRegisterForLocal(allocation[operand.toLocal()]);
node->setUnlinkedMachineLocal(operand);
break;
}
default:
break;
}
}
}
}
return true;
}