void JIT::compileCallEvalSlowCase(Instruction* instruction, Vector<SlowCaseEntry>::iterator& iter)
{
CallLinkInfo* info = m_codeBlock->addCallLinkInfo();
info->setUpCall(CallLinkInfo::Call, CodeOrigin(m_bytecodeOffset), regT0);
linkSlowCase(iter);
int registerOffset = -instruction[4].u.operand;
addPtr(TrustedImm32(registerOffset * sizeof(Register) + sizeof(CallerFrameAndPC)), callFrameRegister, stackPointerRegister);
loadPtr(Address(stackPointerRegister, sizeof(Register) * JSStack::Callee - sizeof(CallerFrameAndPC)), regT0);
loadPtr(Address(stackPointerRegister, sizeof(Register) * JSStack::Callee - sizeof(CallerFrameAndPC)), regT1);
move(TrustedImmPtr(info), regT2);
emitLoad(JSStack::Callee, regT1, regT0);
MacroAssemblerCodeRef virtualThunk = virtualThunkFor(m_vm, *info);
info->setSlowStub(createJITStubRoutine(virtualThunk, *m_vm, nullptr, true));
emitNakedCall(virtualThunk.code());
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
checkStackPointerAlignment();
sampleCodeBlock(m_codeBlock);
emitPutCallResult(instruction);
}
void JIT::compileLoadVarargs(Instruction* instruction)
{
int thisValue = instruction[2].u.operand;
int arguments = instruction[3].u.operand;
int firstFreeRegister = instruction[4].u.operand;
killLastResultRegister();
JumpList slowCase;
JumpList end;
if (m_codeBlock->usesArguments() && arguments == m_codeBlock->argumentsRegister()) {
emitGetVirtualRegister(arguments, regT0);
slowCase.append(branchPtr(NotEqual, regT0, TrustedImmPtr(JSValue::encode(JSValue()))));
emitGetFromCallFrameHeader32(RegisterFile::ArgumentCount, regT0);
slowCase.append(branch32(Above, regT0, TrustedImm32(Arguments::MaxArguments + 1)));
// regT0: argumentCountIncludingThis
move(regT0, regT1);
add32(TrustedImm32(firstFreeRegister + RegisterFile::CallFrameHeaderSize), regT1);
lshift32(TrustedImm32(3), regT1);
addPtr(callFrameRegister, regT1);
// regT1: newCallFrame
slowCase.append(branchPtr(Below, AbsoluteAddress(m_globalData->interpreter->registerFile().addressOfEnd()), regT1));
// Initialize ArgumentCount.
emitFastArithReTagImmediate(regT0, regT2);
storePtr(regT2, Address(regT1, RegisterFile::ArgumentCount * static_cast<int>(sizeof(Register))));
// Initialize 'this'.
emitGetVirtualRegister(thisValue, regT2);
storePtr(regT2, Address(regT1, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
// Copy arguments.
neg32(regT0);
signExtend32ToPtr(regT0, regT0);
end.append(branchAddPtr(Zero, Imm32(1), regT0));
// regT0: -argumentCount
Label copyLoop = label();
loadPtr(BaseIndex(callFrameRegister, regT0, TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))), regT2);
storePtr(regT2, BaseIndex(regT1, regT0, TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
branchAddPtr(NonZero, Imm32(1), regT0).linkTo(copyLoop, this);
end.append(jump());
}
if (m_codeBlock->usesArguments() && arguments == m_codeBlock->argumentsRegister())
slowCase.link(this);
JITStubCall stubCall(this, cti_op_load_varargs);
stubCall.addArgument(thisValue, regT0);
stubCall.addArgument(arguments, regT0);
stubCall.addArgument(Imm32(firstFreeRegister));
stubCall.call(regT1);
if (m_codeBlock->usesArguments() && arguments == m_codeBlock->argumentsRegister())
end.link(this);
}
void JIT::emit_op_ret_object_or_this(Instruction* currentInstruction)
{
unsigned result = currentInstruction[1].u.operand;
unsigned thisReg = currentInstruction[2].u.operand;
// We could JIT generate the deref, only calling out to C when the refcount hits zero.
if (m_codeBlock->needsFullScopeChain())
JITStubCall(this, cti_op_ret_scopeChain).call();
emitLoad(result, regT1, regT0);
Jump notJSCell = branch32(NotEqual, regT1, Imm32(JSValue::CellTag));
loadPtr(Address(regT0, OBJECT_OFFSETOF(JSCell, m_structure)), regT2);
Jump notObject = branch8(NotEqual, Address(regT2, OBJECT_OFFSETOF(Structure, m_typeInfo) + OBJECT_OFFSETOF(TypeInfo, m_type)), Imm32(ObjectType));
emitGetFromCallFrameHeaderPtr(RegisterFile::ReturnPC, regT2);
emitGetFromCallFrameHeaderPtr(RegisterFile::CallerFrame, callFrameRegister);
restoreReturnAddressBeforeReturn(regT2);
ret();
notJSCell.link(this);
notObject.link(this);
emitLoad(thisReg, regT1, regT0);
emitGetFromCallFrameHeaderPtr(RegisterFile::ReturnPC, regT2);
emitGetFromCallFrameHeaderPtr(RegisterFile::CallerFrame, callFrameRegister);
restoreReturnAddressBeforeReturn(regT2);
ret();
}
void JIT::compileOpCall(OpcodeID opcodeID, Instruction* instruction, unsigned callLinkInfoIndex)
{
int callee = instruction[1].u.operand;
/* Caller always:
- Updates callFrameRegister to callee callFrame.
- Initializes ArgumentCount; CallerFrame; Callee.
For a JS call:
- Caller initializes ScopeChain.
- Callee initializes ReturnPC; CodeBlock.
- Callee restores callFrameRegister before return.
For a non-JS call:
- Caller initializes ScopeChain; ReturnPC; CodeBlock.
- Caller restores callFrameRegister after return.
*/
if (opcodeID == op_call_varargs)
compileLoadVarargs(instruction);
else {
int argCount = instruction[2].u.operand;
int registerOffset = instruction[3].u.operand;
addPtr(TrustedImm32(registerOffset * sizeof(Register)), callFrameRegister, regT3);
store32(TrustedImm32(argCount), payloadFor(RegisterFile::ArgumentCount, regT3));
} // regT3 holds newCallFrame with ArgumentCount initialized.
emitLoad(callee, regT1, regT0); // regT1, regT0 holds callee.
storePtr(callFrameRegister, Address(regT3, RegisterFile::CallerFrame * static_cast<int>(sizeof(Register))));
emitStore(RegisterFile::Callee, regT1, regT0, regT3);
move(regT3, callFrameRegister);
if (opcodeID == op_call_eval) {
compileCallEval();
return;
}
DataLabelPtr addressOfLinkedFunctionCheck;
BEGIN_UNINTERRUPTED_SEQUENCE(sequenceOpCall);
Jump slowCase = branchPtrWithPatch(NotEqual, regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
END_UNINTERRUPTED_SEQUENCE(sequenceOpCall);
addSlowCase(slowCase);
addSlowCase(branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag)));
ASSERT_JIT_OFFSET(differenceBetween(addressOfLinkedFunctionCheck, slowCase), patchOffsetOpCallCompareToJump);
ASSERT(m_callStructureStubCompilationInfo.size() == callLinkInfoIndex);
m_callStructureStubCompilationInfo.append(StructureStubCompilationInfo());
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathBegin = addressOfLinkedFunctionCheck;
m_callStructureStubCompilationInfo[callLinkInfoIndex].callType = CallLinkInfo::callTypeFor(opcodeID);
m_callStructureStubCompilationInfo[callLinkInfoIndex].bytecodeIndex = m_bytecodeOffset;
loadPtr(Address(regT0, OBJECT_OFFSETOF(JSFunction, m_scopeChain)), regT1);
emitPutCellToCallFrameHeader(regT1, RegisterFile::ScopeChain);
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall();
sampleCodeBlock(m_codeBlock);
}
void JIT::emit_op_ret_object_or_this(Instruction* currentInstruction)
{
unsigned result = currentInstruction[1].u.operand;
unsigned thisReg = currentInstruction[2].u.operand;
emitLoad(result, regT1, regT0);
Jump notJSCell = branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag));
loadPtr(Address(regT0, JSCell::structureOffset()), regT2);
Jump notObject = emitJumpIfNotObject(regT2);
emitGetFromCallFrameHeaderPtr(JSStack::ReturnPC, regT2);
emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, callFrameRegister);
restoreReturnAddressBeforeReturn(regT2);
ret();
notJSCell.link(this);
notObject.link(this);
emitLoad(thisReg, regT1, regT0);
emitGetFromCallFrameHeaderPtr(JSStack::ReturnPC, regT2);
emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, callFrameRegister);
restoreReturnAddressBeforeReturn(regT2);
ret();
}
void JIT::compileOpCallInitializeCallFrame()
{
// regT0 holds callee, regT1 holds argCount
loadPtr(Address(regT0, OBJECT_OFFSETOF(JSFunction, m_scopeChain)), regT3); // scopeChain
emitPutIntToCallFrameHeader(regT1, RegisterFile::ArgumentCount);
emitPutCellToCallFrameHeader(regT0, RegisterFile::Callee);
emitPutCellToCallFrameHeader(regT3, RegisterFile::ScopeChain);
}
void JIT::compileOpCallInitializeCallFrame()
{
// regT0 holds callee, regT1 holds argCount
store32(regT1, Address(callFrameRegister, RegisterFile::ArgumentCount * static_cast<int>(sizeof(Register))));
loadPtr(Address(regT0, OBJECT_OFFSETOF(JSFunction, m_scopeChain) + OBJECT_OFFSETOF(ScopeChain, m_node)), regT3); // scopeChain
storePtr(regT0, Address(callFrameRegister, RegisterFile::Callee * static_cast<int>(sizeof(Register)))); // callee
storePtr(regT3, Address(callFrameRegister, RegisterFile::ScopeChain * static_cast<int>(sizeof(Register)))); // scopeChain
}
void JIT::compileOpCall(OpcodeID opcodeID, Instruction* instruction, unsigned callLinkInfoIndex)
{
int callee = instruction[1].u.operand;
int argCount = instruction[2].u.operand;
int registerOffset = instruction[3].u.operand;
Jump wasEval;
if (opcodeID == op_call_eval) {
JITStubCall stubCall(this, cti_op_call_eval);
stubCall.addArgument(callee);
stubCall.addArgument(JIT::Imm32(registerOffset));
stubCall.addArgument(JIT::Imm32(argCount));
stubCall.call();
wasEval = branch32(NotEqual, regT1, TrustedImm32(JSValue::EmptyValueTag));
}
emitLoad(callee, regT1, regT0);
DataLabelPtr addressOfLinkedFunctionCheck;
BEGIN_UNINTERRUPTED_SEQUENCE(sequenceOpCall);
Jump jumpToSlow = branchPtrWithPatch(NotEqual, regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
END_UNINTERRUPTED_SEQUENCE(sequenceOpCall);
addSlowCase(jumpToSlow);
ASSERT_JIT_OFFSET(differenceBetween(addressOfLinkedFunctionCheck, jumpToSlow), patchOffsetOpCallCompareToJump);
ASSERT(m_callStructureStubCompilationInfo.size() == callLinkInfoIndex);
m_callStructureStubCompilationInfo.append(StructureStubCompilationInfo());
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathBegin = addressOfLinkedFunctionCheck;
m_callStructureStubCompilationInfo[callLinkInfoIndex].isCall = opcodeID != op_construct;
m_callStructureStubCompilationInfo[callLinkInfoIndex].bytecodeIndex = m_bytecodeOffset;
addSlowCase(branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag)));
// The following is the fast case, only used whan a callee can be linked.
// Fast version of stack frame initialization, directly relative to edi.
// Note that this omits to set up RegisterFile::CodeBlock, which is set in the callee
loadPtr(Address(regT0, OBJECT_OFFSETOF(JSFunction, m_scopeChain)), regT2);
store32(TrustedImm32(JSValue::Int32Tag), tagFor(registerOffset + RegisterFile::ArgumentCount));
store32(Imm32(argCount), payloadFor(registerOffset + RegisterFile::ArgumentCount));
storePtr(callFrameRegister, payloadFor(RegisterFile::CallerFrame + registerOffset, callFrameRegister));
emitStore(registerOffset + RegisterFile::Callee, regT1, regT0);
store32(TrustedImm32(JSValue::CellTag), tagFor(registerOffset + RegisterFile::ScopeChain));
store32(regT2, payloadFor(registerOffset + RegisterFile::ScopeChain));
addPtr(Imm32(registerOffset * sizeof(Register)), callFrameRegister);
// Call to the callee
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall();
if (opcodeID == op_call_eval)
wasEval.link(this);
sampleCodeBlock(m_codeBlock);
}
void JIT::compileOpCallInitializeCallFrame()
{
store32(X86::edx, Address(callFrameRegister, RegisterFile::ArgumentCount * static_cast<int>(sizeof(Register))));
loadPtr(Address(X86::ecx, FIELD_OFFSET(JSFunction, m_scopeChain) + FIELD_OFFSET(ScopeChain, m_node)), X86::edx); // newScopeChain
storePtr(ImmPtr(JSValuePtr::encode(noValue())), Address(callFrameRegister, RegisterFile::OptionalCalleeArguments * static_cast<int>(sizeof(Register))));
storePtr(X86::ecx, Address(callFrameRegister, RegisterFile::Callee * static_cast<int>(sizeof(Register))));
storePtr(X86::edx, Address(callFrameRegister, RegisterFile::ScopeChain * static_cast<int>(sizeof(Register))));
}
void JIT::compileOpCall(OpcodeID opcodeID, Instruction* instruction, unsigned callLinkInfoIndex)
{
int callee = instruction[1].u.operand;
int argCount = instruction[2].u.operand;
int registerOffset = instruction[3].u.operand;
// Handle eval
Jump wasEval;
if (opcodeID == op_call_eval) {
JITStubCall stubCall(this, cti_op_call_eval);
stubCall.addArgument(callee, regT0);
stubCall.addArgument(JIT::Imm32(registerOffset));
stubCall.addArgument(JIT::Imm32(argCount));
stubCall.call();
wasEval = branchPtr(NotEqual, regT0, TrustedImmPtr(JSValue::encode(JSValue())));
}
// This plants a check for a cached JSFunction value, so we can plant a fast link to the callee.
// This deliberately leaves the callee in ecx, used when setting up the stack frame below
emitGetVirtualRegister(callee, regT0);
DataLabelPtr addressOfLinkedFunctionCheck;
BEGIN_UNINTERRUPTED_SEQUENCE(sequenceOpCall);
Jump jumpToSlow = branchPtrWithPatch(NotEqual, regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(JSValue::encode(JSValue())));
END_UNINTERRUPTED_SEQUENCE(sequenceOpCall);
addSlowCase(jumpToSlow);
ASSERT_JIT_OFFSET(differenceBetween(addressOfLinkedFunctionCheck, jumpToSlow), patchOffsetOpCallCompareToJump);
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathBegin = addressOfLinkedFunctionCheck;
m_callStructureStubCompilationInfo[callLinkInfoIndex].isCall = opcodeID != op_construct;
// The following is the fast case, only used whan a callee can be linked.
// Fast version of stack frame initialization, directly relative to edi.
// Note that this omits to set up RegisterFile::CodeBlock, which is set in the callee
loadPtr(Address(regT0, OBJECT_OFFSETOF(JSFunction, m_scopeChain)), regT1); // newScopeChain
store32(TrustedImm32(Int32Tag), intTagFor(registerOffset + RegisterFile::ArgumentCount));
store32(Imm32(argCount), intPayloadFor(registerOffset + RegisterFile::ArgumentCount));
storePtr(callFrameRegister, Address(callFrameRegister, (registerOffset + RegisterFile::CallerFrame) * static_cast<int>(sizeof(Register))));
storePtr(regT0, Address(callFrameRegister, (registerOffset + RegisterFile::Callee) * static_cast<int>(sizeof(Register))));
storePtr(regT1, Address(callFrameRegister, (registerOffset + RegisterFile::ScopeChain) * static_cast<int>(sizeof(Register))));
addPtr(Imm32(registerOffset * sizeof(Register)), callFrameRegister);
// Call to the callee
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall();
if (opcodeID == op_call_eval)
wasEval.link(this);
sampleCodeBlock(m_codeBlock);
}
void JIT::compileCallEvalSlowCase(Instruction* instruction, Vector<SlowCaseEntry>::iterator& iter)
{
linkSlowCase(iter);
int registerOffset = -instruction[4].u.operand;
addPtr(TrustedImm32(registerOffset * sizeof(Register) + sizeof(CallerFrameAndPC)), callFrameRegister, stackPointerRegister);
loadPtr(Address(stackPointerRegister, sizeof(Register) * JSStack::Callee - sizeof(CallerFrameAndPC)), regT0);
loadPtr(Address(stackPointerRegister, sizeof(Register) * JSStack::Callee - sizeof(CallerFrameAndPC)), regT1);
move(TrustedImmPtr(&CallLinkInfo::dummy()), regT2);
emitLoad(JSStack::Callee, regT1, regT0);
emitNakedCall(m_vm->getCTIStub(virtualCallThunkGenerator).code());
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
checkStackPointerAlignment();
sampleCodeBlock(m_codeBlock);
emitPutCallResult(instruction);
}
void JIT::privateCompileClosureCall(CallLinkInfo* callLinkInfo, CodeBlock* calleeCodeBlock, Structure* expectedStructure, ExecutableBase* expectedExecutable, MacroAssemblerCodePtr codePtr)
{
JumpList slowCases;
slowCases.append(branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag)));
slowCases.append(branchPtr(NotEqual, Address(regT0, JSCell::structureOffset()), TrustedImmPtr(expectedStructure)));
slowCases.append(branchPtr(NotEqual, Address(regT0, JSFunction::offsetOfExecutable()), TrustedImmPtr(expectedExecutable)));
loadPtr(Address(regT0, JSFunction::offsetOfScopeChain()), regT1);
emitPutCellToCallFrameHeader(regT1, JSStack::ScopeChain);
Call call = nearCall();
Jump done = jump();
slowCases.link(this);
move(TrustedImmPtr(callLinkInfo->callReturnLocation.executableAddress()), regT2);
restoreReturnAddressBeforeReturn(regT2);
Jump slow = jump();
LinkBuffer patchBuffer(*m_vm, this, m_codeBlock);
patchBuffer.link(call, FunctionPtr(codePtr.executableAddress()));
patchBuffer.link(done, callLinkInfo->hotPathOther.labelAtOffset(0));
patchBuffer.link(slow, CodeLocationLabel(m_vm->getCTIStub(virtualCallThunkGenerator).code()));
RefPtr<ClosureCallStubRoutine> stubRoutine = adoptRef(new ClosureCallStubRoutine(
FINALIZE_CODE(
patchBuffer,
("Baseline closure call stub for %s, return point %p, target %p (%s)",
toCString(*m_codeBlock).data(),
callLinkInfo->hotPathOther.labelAtOffset(0).executableAddress(),
codePtr.executableAddress(),
toCString(pointerDump(calleeCodeBlock)).data())),
*m_vm, m_codeBlock->ownerExecutable(), expectedStructure, expectedExecutable,
callLinkInfo->codeOrigin));
RepatchBuffer repatchBuffer(m_codeBlock);
repatchBuffer.replaceWithJump(
RepatchBuffer::startOfBranchPtrWithPatchOnRegister(callLinkInfo->hotPathBegin),
CodeLocationLabel(stubRoutine->code().code()));
repatchBuffer.relink(callLinkInfo->callReturnLocation, m_vm->getCTIStub(virtualCallThunkGenerator).code());
callLinkInfo->stub = stubRoutine.release();
}
// This method used to fill a TiValue to a GPR when linking speculative -> non-speculative.
void JITCompiler::fillToJS(NodeIndex nodeIndex, GPRReg gpr)
{
Node& node = graph()[nodeIndex];
if (node.isConstant()) {
if (isInt32Constant(nodeIndex)) {
TiValue jsValue = jsNumber(valueOfInt32Constant(nodeIndex));
move(MacroAssembler::ImmPtr(TiValue::encode(jsValue)), gpr);
} else if (isDoubleConstant(nodeIndex)) {
TiValue jsValue(TiValue::EncodeAsDouble, valueOfDoubleConstant(nodeIndex));
move(MacroAssembler::ImmPtr(TiValue::encode(jsValue)), gpr);
} else {
ASSERT(isJSConstant(nodeIndex));
TiValue jsValue = valueOfJSConstant(nodeIndex);
move(MacroAssembler::ImmPtr(TiValue::encode(jsValue)), gpr);
}
return;
}
loadPtr(addressFor(node.virtualRegister()), gpr);
}
void JIT::compileOpCall(OpcodeID opcodeID, Instruction* instruction, unsigned callLinkInfoIndex)
{
int callee = instruction[2].u.operand;
/* Caller always:
- Updates callFrameRegister to callee callFrame.
- Initializes ArgumentCount; CallerFrame; Callee.
For a JS call:
- Caller initializes ScopeChain.
- Callee initializes ReturnPC; CodeBlock.
- Callee restores callFrameRegister before return.
For a non-JS call:
- Caller initializes ScopeChain; ReturnPC; CodeBlock.
- Caller restores callFrameRegister after return.
*/
if (opcodeID == op_call_varargs)
compileLoadVarargs(instruction);
else {
int argCount = instruction[3].u.operand;
int registerOffset = -instruction[4].u.operand;
if (opcodeID == op_call && shouldEmitProfiling()) {
emitLoad(registerOffset + CallFrame::argumentOffsetIncludingThis(0), regT0, regT1);
Jump done = branch32(NotEqual, regT0, TrustedImm32(JSValue::CellTag));
loadPtr(Address(regT1, JSCell::structureOffset()), regT1);
storePtr(regT1, instruction[6].u.arrayProfile->addressOfLastSeenStructure());
done.link(this);
}
addPtr(TrustedImm32(registerOffset * sizeof(Register)), callFrameRegister, regT3);
store32(TrustedImm32(argCount), payloadFor(JSStack::ArgumentCount, regT3));
} // regT3 holds newCallFrame with ArgumentCount initialized.
uint32_t locationBits = CallFrame::Location::encodeAsBytecodeInstruction(instruction);
store32(TrustedImm32(locationBits), tagFor(JSStack::ArgumentCount, callFrameRegister));
emitLoad(callee, regT1, regT0); // regT1, regT0 holds callee.
storePtr(callFrameRegister, Address(regT3, JSStack::CallerFrame * static_cast<int>(sizeof(Register))));
emitStore(JSStack::Callee, regT1, regT0, regT3);
move(regT3, callFrameRegister);
if (opcodeID == op_call_eval) {
compileCallEval(instruction);
return;
}
DataLabelPtr addressOfLinkedFunctionCheck;
Jump slowCase = branchPtrWithPatch(NotEqual, regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
addSlowCase(slowCase);
addSlowCase(branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag)));
ASSERT(m_callStructureStubCompilationInfo.size() == callLinkInfoIndex);
m_callStructureStubCompilationInfo.append(StructureStubCompilationInfo());
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathBegin = addressOfLinkedFunctionCheck;
m_callStructureStubCompilationInfo[callLinkInfoIndex].callType = CallLinkInfo::callTypeFor(opcodeID);
m_callStructureStubCompilationInfo[callLinkInfoIndex].bytecodeIndex = m_bytecodeOffset;
loadPtr(Address(regT0, OBJECT_OFFSETOF(JSFunction, m_scope)), regT1);
emitPutCellToCallFrameHeader(regT1, JSStack::ScopeChain);
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall();
sampleCodeBlock(m_codeBlock);
emitPutCallResult(instruction);
}
void JIT::compileOpCall(OpcodeID opcodeID, Instruction* instruction, unsigned callLinkInfoIndex)
{
int callee = instruction[1].u.operand;
/* Caller always:
- Updates callFrameRegister to callee callFrame.
- Initializes ArgumentCount; CallerFrame; Callee.
For a JS call:
- Caller initializes ScopeChain.
- Callee initializes ReturnPC; CodeBlock.
- Callee restores callFrameRegister before return.
For a non-JS call:
- Caller initializes ScopeChain; ReturnPC; CodeBlock.
- Caller restores callFrameRegister after return.
*/
if (opcodeID == op_call_varargs)
compileLoadVarargs(instruction);
else {
int argCount = instruction[2].u.operand;
int registerOffset = instruction[3].u.operand;
if (opcodeID == op_call && canBeOptimized()) {
emitGetVirtualRegister(registerOffset + CallFrame::argumentOffsetIncludingThis(0), regT0);
Jump done = emitJumpIfNotJSCell(regT0);
loadPtr(Address(regT0, JSCell::structureOffset()), regT0);
storePtr(regT0, instruction[5].u.arrayProfile->addressOfLastSeenStructure());
done.link(this);
}
addPtr(TrustedImm32(registerOffset * sizeof(Register)), callFrameRegister, regT1);
store32(TrustedImm32(argCount), Address(regT1, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
} // regT1 holds newCallFrame with ArgumentCount initialized.
store32(TrustedImm32(instruction - m_codeBlock->instructions().begin()), Address(callFrameRegister, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)));
emitGetVirtualRegister(callee, regT0); // regT0 holds callee.
store64(callFrameRegister, Address(regT1, JSStack::CallerFrame * static_cast<int>(sizeof(Register))));
store64(regT0, Address(regT1, JSStack::Callee * static_cast<int>(sizeof(Register))));
move(regT1, callFrameRegister);
if (opcodeID == op_call_eval) {
compileCallEval();
return;
}
DataLabelPtr addressOfLinkedFunctionCheck;
BEGIN_UNINTERRUPTED_SEQUENCE(sequenceOpCall);
Jump slowCase = branchPtrWithPatch(NotEqual, regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
END_UNINTERRUPTED_SEQUENCE(sequenceOpCall);
addSlowCase(slowCase);
ASSERT(m_callStructureStubCompilationInfo.size() == callLinkInfoIndex);
m_callStructureStubCompilationInfo.append(StructureStubCompilationInfo());
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathBegin = addressOfLinkedFunctionCheck;
m_callStructureStubCompilationInfo[callLinkInfoIndex].callType = CallLinkInfo::callTypeFor(opcodeID);
m_callStructureStubCompilationInfo[callLinkInfoIndex].bytecodeIndex = m_bytecodeOffset;
loadPtr(Address(regT0, OBJECT_OFFSETOF(JSFunction, m_scope)), regT1);
emitPutToCallFrameHeader(regT1, JSStack::ScopeChain);
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall();
sampleCodeBlock(m_codeBlock);
}
void JITCompiler::compileFunction()
{
SamplingRegion samplingRegion("DFG Backend");
setStartOfCode();
compileEntry();
// === Function header code generation ===
// This is the main entry point, without performing an arity check.
// If we needed to perform an arity check we will already have moved the return address,
// so enter after this.
Label fromArityCheck(this);
// Plant a check that sufficient space is available in the JSStack.
addPtr(TrustedImm32(virtualRegisterForLocal(m_graph.requiredRegisterCountForExecutionAndExit() - 1).offset() * sizeof(Register)), GPRInfo::callFrameRegister, GPRInfo::regT1);
Jump stackOverflow = branchPtr(Above, AbsoluteAddress(m_vm->addressOfStackLimit()), GPRInfo::regT1);
// Move the stack pointer down to accommodate locals
addPtr(TrustedImm32(m_graph.stackPointerOffset() * sizeof(Register)), GPRInfo::callFrameRegister, stackPointerRegister);
checkStackPointerAlignment();
// === Function body code generation ===
m_speculative = adoptPtr(new SpeculativeJIT(*this));
compileBody();
setEndOfMainPath();
// === Function footer code generation ===
//
// Generate code to perform the stack overflow handling (if the stack check in
// the function header fails), and generate the entry point with arity check.
//
// Generate the stack overflow handling; if the stack check in the function head fails,
// we need to call out to a helper function to throw the StackOverflowError.
stackOverflow.link(this);
emitStoreCodeOrigin(CodeOrigin(0));
if (maxFrameExtentForSlowPathCall)
addPtr(TrustedImm32(-maxFrameExtentForSlowPathCall), stackPointerRegister);
m_speculative->callOperationWithCallFrameRollbackOnException(operationThrowStackOverflowError, m_codeBlock);
// The fast entry point into a function does not check the correct number of arguments
// have been passed to the call (we only use the fast entry point where we can statically
// determine the correct number of arguments have been passed, or have already checked).
// In cases where an arity check is necessary, we enter here.
// FIXME: change this from a cti call to a DFG style operation (normal C calling conventions).
m_arityCheck = label();
compileEntry();
load32(AssemblyHelpers::payloadFor((VirtualRegister)JSStack::ArgumentCount), GPRInfo::regT1);
branch32(AboveOrEqual, GPRInfo::regT1, TrustedImm32(m_codeBlock->numParameters())).linkTo(fromArityCheck, this);
emitStoreCodeOrigin(CodeOrigin(0));
if (maxFrameExtentForSlowPathCall)
addPtr(TrustedImm32(-maxFrameExtentForSlowPathCall), stackPointerRegister);
m_speculative->callOperationWithCallFrameRollbackOnException(m_codeBlock->m_isConstructor ? operationConstructArityCheck : operationCallArityCheck, GPRInfo::regT0);
if (maxFrameExtentForSlowPathCall)
addPtr(TrustedImm32(maxFrameExtentForSlowPathCall), stackPointerRegister);
branchTest32(Zero, GPRInfo::regT0).linkTo(fromArityCheck, this);
emitStoreCodeOrigin(CodeOrigin(0));
move(TrustedImmPtr(m_vm->arityCheckFailReturnThunks->returnPCsFor(*m_vm, m_codeBlock->numParameters())), GPRInfo::regT5);
loadPtr(BaseIndex(GPRInfo::regT5, GPRInfo::regT0, timesPtr()), GPRInfo::regT5);
m_callArityFixup = call();
jump(fromArityCheck);
// Generate slow path code.
m_speculative->runSlowPathGenerators();
compileExceptionHandlers();
linkOSRExits();
// Create OSR entry trampolines if necessary.
m_speculative->createOSREntries();
setEndOfCode();
}
void JIT::compileOpCall(OpcodeID opcodeID, Instruction* instruction, unsigned callLinkInfoIndex)
{
CallLinkInfo* info = m_codeBlock->addCallLinkInfo();
int callee = instruction[2].u.operand;
/* Caller always:
- Updates callFrameRegister to callee callFrame.
- Initializes ArgumentCount; CallerFrame; Callee.
For a JS call:
- Callee initializes ReturnPC; CodeBlock.
- Callee restores callFrameRegister before return.
For a non-JS call:
- Caller initializes ReturnPC; CodeBlock.
- Caller restores callFrameRegister after return.
*/
if (opcodeID == op_call_varargs || opcodeID == op_construct_varargs)
compileSetupVarargsFrame(instruction, info);
else {
int argCount = instruction[3].u.operand;
int registerOffset = -instruction[4].u.operand;
if (opcodeID == op_call && shouldEmitProfiling()) {
emitLoad(registerOffset + CallFrame::argumentOffsetIncludingThis(0), regT0, regT1);
Jump done = branch32(NotEqual, regT0, TrustedImm32(JSValue::CellTag));
loadPtr(Address(regT1, JSCell::structureIDOffset()), regT1);
storePtr(regT1, instruction[OPCODE_LENGTH(op_call) - 2].u.arrayProfile->addressOfLastSeenStructureID());
done.link(this);
}
addPtr(TrustedImm32(registerOffset * sizeof(Register) + sizeof(CallerFrameAndPC)), callFrameRegister, stackPointerRegister);
store32(TrustedImm32(argCount), Address(stackPointerRegister, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)));
} // SP holds newCallFrame + sizeof(CallerFrameAndPC), with ArgumentCount initialized.
uint32_t locationBits = CallFrame::Location::encodeAsBytecodeInstruction(instruction);
store32(TrustedImm32(locationBits), tagFor(JSStack::ArgumentCount, callFrameRegister));
emitLoad(callee, regT1, regT0); // regT1, regT0 holds callee.
store32(regT0, Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)));
store32(regT1, Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) + TagOffset - sizeof(CallerFrameAndPC)));
if (opcodeID == op_call_eval) {
compileCallEval(instruction);
return;
}
addSlowCase(branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag)));
DataLabelPtr addressOfLinkedFunctionCheck;
Jump slowCase = branchPtrWithPatch(NotEqual, regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
addSlowCase(slowCase);
ASSERT(m_callCompilationInfo.size() == callLinkInfoIndex);
info->setUpCall(CallLinkInfo::callTypeFor(opcodeID), CodeOrigin(m_bytecodeOffset), regT0);
m_callCompilationInfo.append(CallCompilationInfo());
m_callCompilationInfo[callLinkInfoIndex].hotPathBegin = addressOfLinkedFunctionCheck;
m_callCompilationInfo[callLinkInfoIndex].callLinkInfo = info;
checkStackPointerAlignment();
m_callCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall();
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
checkStackPointerAlignment();
sampleCodeBlock(m_codeBlock);
emitPutCallResult(instruction);
}
void JIT::compileOpCall(OpcodeID opcodeID, Instruction* instruction, unsigned callLinkInfoIndex)
{
int callee = instruction[2].u.operand;
/* Caller always:
- Updates callFrameRegister to callee callFrame.
- Initializes ArgumentCount; CallerFrame; Callee.
For a JS call:
- Caller initializes ScopeChain.
- Callee initializes ReturnPC; CodeBlock.
- Callee restores callFrameRegister before return.
For a non-JS call:
- Caller initializes ScopeChain; ReturnPC; CodeBlock.
- Caller restores callFrameRegister after return.
*/
COMPILE_ASSERT(OPCODE_LENGTH(op_call) == OPCODE_LENGTH(op_construct), call_and_construct_opcodes_must_be_same_length);
COMPILE_ASSERT(OPCODE_LENGTH(op_call) == OPCODE_LENGTH(op_call_varargs), call_and_call_varargs_opcodes_must_be_same_length);
COMPILE_ASSERT(OPCODE_LENGTH(op_call) == OPCODE_LENGTH(op_construct_varargs), call_and_construct_varargs_opcodes_must_be_same_length);
if (opcodeID == op_call_varargs || opcodeID == op_construct_varargs)
compileLoadVarargs(instruction);
else {
int argCount = instruction[3].u.operand;
int registerOffset = -instruction[4].u.operand;
if (opcodeID == op_call && shouldEmitProfiling()) {
emitGetVirtualRegister(registerOffset + CallFrame::argumentOffsetIncludingThis(0), regT0);
Jump done = emitJumpIfNotJSCell(regT0);
load32(Address(regT0, JSCell::structureIDOffset()), regT0);
store32(regT0, instruction[OPCODE_LENGTH(op_call) - 2].u.arrayProfile->addressOfLastSeenStructureID());
done.link(this);
}
addPtr(TrustedImm32(registerOffset * sizeof(Register) + sizeof(CallerFrameAndPC)), callFrameRegister, stackPointerRegister);
store32(TrustedImm32(argCount), Address(stackPointerRegister, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset - sizeof(CallerFrameAndPC)));
} // SP holds newCallFrame + sizeof(CallerFrameAndPC), with ArgumentCount initialized.
uint32_t bytecodeOffset = instruction - m_codeBlock->instructions().begin();
uint32_t locationBits = CallFrame::Location::encodeAsBytecodeOffset(bytecodeOffset);
store32(TrustedImm32(locationBits), Address(callFrameRegister, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + TagOffset));
emitGetVirtualRegister(callee, regT0); // regT0 holds callee.
store64(regT0, Address(stackPointerRegister, JSStack::Callee * static_cast<int>(sizeof(Register)) - sizeof(CallerFrameAndPC)));
if (opcodeID == op_call_eval) {
compileCallEval(instruction);
return;
}
DataLabelPtr addressOfLinkedFunctionCheck;
Jump slowCase = branchPtrWithPatch(NotEqual, regT0, addressOfLinkedFunctionCheck, TrustedImmPtr(0));
addSlowCase(slowCase);
ASSERT(m_callCompilationInfo.size() == callLinkInfoIndex);
CallLinkInfo* info = m_codeBlock->addCallLinkInfo();
info->callType = CallLinkInfo::callTypeFor(opcodeID);
info->codeOrigin = CodeOrigin(m_bytecodeOffset);
info->calleeGPR = regT0;
m_callCompilationInfo.append(CallCompilationInfo());
m_callCompilationInfo[callLinkInfoIndex].hotPathBegin = addressOfLinkedFunctionCheck;
m_callCompilationInfo[callLinkInfoIndex].callLinkInfo = info;
loadPtr(Address(regT0, OBJECT_OFFSETOF(JSFunction, m_scope)), regT2);
store64(regT2, Address(MacroAssembler::stackPointerRegister, JSStack::ScopeChain * sizeof(Register) - sizeof(CallerFrameAndPC)));
m_callCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall();
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
checkStackPointerAlignment();
sampleCodeBlock(m_codeBlock);
emitPutCallResult(instruction);
}
void JIT::compileOpCall(OpcodeID opcodeID, Instruction* instruction, unsigned callLinkInfoIndex)
{
int dst = instruction[1].u.operand;
int callee = instruction[2].u.operand;
int argCount = instruction[3].u.operand;
int registerOffset = instruction[4].u.operand;
// Handle eval
Jump wasEval;
if (opcodeID == op_call_eval) {
emitGetVirtualRegister(callee, X86::ecx);
compileOpCallEvalSetupArgs(instruction);
emitCTICall(Interpreter::cti_op_call_eval);
wasEval = jnePtr(X86::eax, ImmPtr(JSValuePtr::encode(JSImmediate::impossibleValue())));
}
// This plants a check for a cached JSFunction value, so we can plant a fast link to the callee.
// This deliberately leaves the callee in ecx, used when setting up the stack frame below
emitGetVirtualRegister(callee, X86::ecx);
DataLabelPtr addressOfLinkedFunctionCheck;
Jump jumpToSlow = jnePtrWithPatch(X86::ecx, addressOfLinkedFunctionCheck, ImmPtr(JSValuePtr::encode(JSImmediate::impossibleValue())));
addSlowCase(jumpToSlow);
ASSERT(differenceBetween(addressOfLinkedFunctionCheck, jumpToSlow) == patchOffsetOpCallCompareToJump);
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathBegin = addressOfLinkedFunctionCheck;
// The following is the fast case, only used whan a callee can be linked.
// In the case of OpConstruct, call out to a cti_ function to create the new object.
if (opcodeID == op_construct) {
int proto = instruction[5].u.operand;
int thisRegister = instruction[6].u.operand;
emitPutJITStubArg(X86::ecx, 1);
emitPutJITStubArgFromVirtualRegister(proto, 4, X86::eax);
emitCTICall(Interpreter::cti_op_construct_JSConstruct);
emitPutVirtualRegister(thisRegister);
emitGetVirtualRegister(callee, X86::ecx);
}
// Fast version of stack frame initialization, directly relative to edi.
// Note that this omits to set up RegisterFile::CodeBlock, which is set in the callee
storePtr(ImmPtr(JSValuePtr::encode(noValue())), Address(callFrameRegister, (registerOffset + RegisterFile::OptionalCalleeArguments) * static_cast<int>(sizeof(Register))));
storePtr(X86::ecx, Address(callFrameRegister, (registerOffset + RegisterFile::Callee) * static_cast<int>(sizeof(Register))));
loadPtr(Address(X86::ecx, FIELD_OFFSET(JSFunction, m_scopeChain) + FIELD_OFFSET(ScopeChain, m_node)), X86::edx); // newScopeChain
store32(Imm32(argCount), Address(callFrameRegister, (registerOffset + RegisterFile::ArgumentCount) * static_cast<int>(sizeof(Register))));
storePtr(callFrameRegister, Address(callFrameRegister, (registerOffset + RegisterFile::CallerFrame) * static_cast<int>(sizeof(Register))));
storePtr(X86::edx, Address(callFrameRegister, (registerOffset + RegisterFile::ScopeChain) * static_cast<int>(sizeof(Register))));
addPtr(Imm32(registerOffset * sizeof(Register)), callFrameRegister);
// Call to the callee
m_callStructureStubCompilationInfo[callLinkInfoIndex].hotPathOther = emitNakedCall(reinterpret_cast<void*>(unreachable));
if (opcodeID == op_call_eval)
wasEval.link(this);
// Put the return value in dst. In the interpreter, op_ret does this.
emitPutVirtualRegister(dst);
#if ENABLE(CODEBLOCK_SAMPLING)
storePtr(ImmPtr(m_codeBlock), m_interpreter->sampler()->codeBlockSlot());
#endif
}
