void JIT::compileSetupVarargsFrame(Instruction* instruction, CallLinkInfo* info)
{
int thisValue = instruction[3].u.operand;
int arguments = instruction[4].u.operand;
int firstFreeRegister = instruction[5].u.operand;
int firstVarArgOffset = instruction[6].u.operand;
emitLoad(arguments, regT1, regT0);
callOperation(operationSizeFrameForVarargs, regT1, regT0, -firstFreeRegister, firstVarArgOffset);
move(TrustedImm32(-firstFreeRegister), regT1);
emitSetVarargsFrame(*this, returnValueGPR, false, regT1, regT1);
addPtr(TrustedImm32(-(sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 6 * sizeof(void*)))), regT1, stackPointerRegister);
emitLoad(arguments, regT2, regT4);
callOperation(operationSetupVarargsFrame, regT1, regT2, regT4, firstVarArgOffset, regT0);
move(returnValueGPR, regT1);
// Profile the argument count.
load32(Address(regT1, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset), regT2);
load8(info->addressOfMaxNumArguments(), regT0);
Jump notBiggest = branch32(Above, regT0, regT2);
Jump notSaturated = branch32(BelowOrEqual, regT2, TrustedImm32(255));
move(TrustedImm32(255), regT2);
notSaturated.link(this);
store8(regT2, info->addressOfMaxNumArguments());
notBiggest.link(this);
// Initialize 'this'.
emitLoad(thisValue, regT2, regT0);
store32(regT0, Address(regT1, PayloadOffset + (CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))));
store32(regT2, Address(regT1, TagOffset + (CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))));
addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), regT1, stackPointerRegister);
}
void JIT::compileSetupVarargsFrame(Instruction* instruction, CallLinkInfo* info)
{
int thisValue = instruction[3].u.operand;
int arguments = instruction[4].u.operand;
int firstFreeRegister = instruction[5].u.operand;
int firstVarArgOffset = instruction[6].u.operand;
JumpList slowCase;
JumpList end;
bool canOptimize = m_codeBlock->usesArguments()
&& arguments == m_codeBlock->argumentsRegister().offset()
&& !m_codeBlock->symbolTable()->slowArguments();
if (canOptimize) {
emitGetVirtualRegister(arguments, regT0);
slowCase.append(branch64(NotEqual, regT0, TrustedImm64(JSValue::encode(JSValue()))));
move(TrustedImm32(-firstFreeRegister), regT1);
emitSetupVarargsFrameFastCase(*this, regT1, regT0, regT1, regT2, firstVarArgOffset, slowCase);
end.append(jump());
slowCase.link(this);
}
emitGetVirtualRegister(arguments, regT1);
callOperation(operationSizeFrameForVarargs, regT1, -firstFreeRegister, firstVarArgOffset);
move(TrustedImm32(-firstFreeRegister), regT1);
emitSetVarargsFrame(*this, returnValueGPR, false, regT1, regT1);
addPtr(TrustedImm32(-(sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(void*)))), regT1, stackPointerRegister);
emitGetVirtualRegister(arguments, regT2);
callOperation(operationSetupVarargsFrame, regT1, regT2, firstVarArgOffset, regT0);
move(returnValueGPR, regT1);
if (canOptimize)
end.link(this);
// Profile the argument count.
load32(Address(regT1, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset), regT2);
load8(&info->maxNumArguments, regT0);
Jump notBiggest = branch32(Above, regT0, regT2);
Jump notSaturated = branch32(BelowOrEqual, regT2, TrustedImm32(255));
move(TrustedImm32(255), regT2);
notSaturated.link(this);
store8(regT2, &info->maxNumArguments);
notBiggest.link(this);
// Initialize 'this'.
emitGetVirtualRegister(thisValue, regT0);
store64(regT0, Address(regT1, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), regT1, stackPointerRegister);
}
void JIT::compileLoadVarargs(Instruction* instruction)
{
int thisValue = instruction[3].u.operand;
int arguments = instruction[4].u.operand;
int firstFreeRegister = instruction[5].u.operand;
JumpList slowCase;
JumpList end;
bool canOptimize = m_codeBlock->usesArguments()
&& arguments == m_codeBlock->argumentsRegister().offset()
&& !m_codeBlock->symbolTable()->slowArguments();
if (canOptimize) {
emitGetVirtualRegister(arguments, regT0);
slowCase.append(branch64(NotEqual, regT0, TrustedImm64(JSValue::encode(JSValue()))));
emitGetFromCallFrameHeader32(JSStack::ArgumentCount, regT0);
slowCase.append(branch32(Above, regT0, TrustedImm32(Arguments::MaxArguments + 1)));
// regT0: argumentCountIncludingThis
move(regT0, regT1);
neg64(regT1);
add64(TrustedImm32(firstFreeRegister - JSStack::CallFrameHeaderSize), regT1);
lshift64(TrustedImm32(3), regT1);
addPtr(callFrameRegister, regT1);
// regT1: newCallFrame
slowCase.append(branchPtr(Above, AbsoluteAddress(m_vm->addressOfJSStackLimit()), regT1));
// Initialize ArgumentCount.
store32(regT0, Address(regT1, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
// Initialize 'this'.
emitGetVirtualRegister(thisValue, regT2);
store64(regT2, Address(regT1, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
// Copy arguments.
signExtend32ToPtr(regT0, regT0);
end.append(branchSub64(Zero, TrustedImm32(1), regT0));
// regT0: argumentCount
Label copyLoop = label();
load64(BaseIndex(callFrameRegister, regT0, TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))), regT2);
store64(regT2, BaseIndex(regT1, regT0, TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
branchSub64(NonZero, TrustedImm32(1), regT0).linkTo(copyLoop, this);
end.append(jump());
}
if (canOptimize)
slowCase.link(this);
emitGetVirtualRegister(thisValue, regT0);
emitGetVirtualRegister(arguments, regT1);
callOperation(operationLoadVarargs, regT0, regT1, firstFreeRegister);
move(returnValueGPR, regT1);
if (canOptimize)
end.link(this);
}
std::enable_if_t<
Op::opcodeID == op_call_varargs || Op::opcodeID == op_construct_varargs
|| Op::opcodeID == op_tail_call_varargs || Op::opcodeID == op_tail_call_forward_arguments
, void>
JIT::compileSetupFrame(const Op& bytecode, CallLinkInfo* info)
{
OpcodeID opcodeID = Op::opcodeID;
int thisValue = bytecode.m_thisValue.offset();
int arguments = bytecode.m_arguments.offset();
int firstFreeRegister = bytecode.m_firstFree.offset();
int firstVarArgOffset = bytecode.m_firstVarArg;
emitLoad(arguments, regT1, regT0);
Z_JITOperation_EJZZ sizeOperation;
if (Op::opcodeID == op_tail_call_forward_arguments)
sizeOperation = operationSizeFrameForForwardArguments;
else
sizeOperation = operationSizeFrameForVarargs;
callOperation(sizeOperation, JSValueRegs(regT1, regT0), -firstFreeRegister, firstVarArgOffset);
move(TrustedImm32(-firstFreeRegister), regT1);
emitSetVarargsFrame(*this, returnValueGPR, false, regT1, regT1);
addPtr(TrustedImm32(-(sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 6 * sizeof(void*)))), regT1, stackPointerRegister);
emitLoad(arguments, regT2, regT4);
F_JITOperation_EFJZZ setupOperation;
if (opcodeID == op_tail_call_forward_arguments)
setupOperation = operationSetupForwardArgumentsFrame;
else
setupOperation = operationSetupVarargsFrame;
callOperation(setupOperation, regT1, JSValueRegs(regT2, regT4), firstVarArgOffset, regT0);
move(returnValueGPR, regT1);
// Profile the argument count.
load32(Address(regT1, CallFrameSlot::argumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset), regT2);
load32(info->addressOfMaxNumArguments(), regT0);
Jump notBiggest = branch32(Above, regT0, regT2);
store32(regT2, info->addressOfMaxNumArguments());
notBiggest.link(this);
// Initialize 'this'.
emitLoad(thisValue, regT2, regT0);
store32(regT0, Address(regT1, PayloadOffset + (CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))));
store32(regT2, Address(regT1, TagOffset + (CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))));
addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), regT1, stackPointerRegister);
}
static void dispatch(CCallHelpers& jit, FTL::State* state, const B3::StackmapGenerationParams& params, DFG::Node* node, Box<CCallHelpers::JumpList> exceptions, CCallHelpers::JumpList from, OperationType operation, ResultType result, Arguments arguments, std::index_sequence<ArgumentsIndex...>)
{
CCallHelpers::Label done = jit.label();
params.addLatePath([=] (CCallHelpers& jit) {
AllowMacroScratchRegisterUsage allowScratch(jit);
from.link(&jit);
callOperation(
*state, params.unavailableRegisters(), jit, node->origin.semantic,
exceptions.get(), operation, extractResult(result), std::get<ArgumentsIndex>(arguments)...);
jit.jump().linkTo(done, &jit);
});
}
void JIT::compileSetupVarargsFrame(OpcodeID opcode, Instruction* instruction, CallLinkInfo* info)
{
int thisValue = instruction[3].u.operand;
int arguments = instruction[4].u.operand;
int firstFreeRegister = instruction[5].u.operand;
int firstVarArgOffset = instruction[6].u.operand;
emitGetVirtualRegister(arguments, regT1);
Z_JITOperation_EJZZ sizeOperation;
if (opcode == op_tail_call_forward_arguments)
sizeOperation = operationSizeFrameForForwardArguments;
else
sizeOperation = operationSizeFrameForVarargs;
callOperation(sizeOperation, regT1, -firstFreeRegister, firstVarArgOffset);
move(TrustedImm32(-firstFreeRegister), regT1);
emitSetVarargsFrame(*this, returnValueGPR, false, regT1, regT1);
addPtr(TrustedImm32(-(sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(void*)))), regT1, stackPointerRegister);
emitGetVirtualRegister(arguments, regT2);
F_JITOperation_EFJZZ setupOperation;
if (opcode == op_tail_call_forward_arguments)
setupOperation = operationSetupForwardArgumentsFrame;
else
setupOperation = operationSetupVarargsFrame;
callOperation(setupOperation, regT1, regT2, firstVarArgOffset, regT0);
move(returnValueGPR, regT1);
// Profile the argument count.
load32(Address(regT1, CallFrameSlot::argumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset), regT2);
load32(info->addressOfMaxNumArguments(), regT0);
Jump notBiggest = branch32(Above, regT0, regT2);
store32(regT2, info->addressOfMaxNumArguments());
notBiggest.link(this);
// Initialize 'this'.
emitGetVirtualRegister(thisValue, regT0);
store64(regT0, Address(regT1, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), regT1, stackPointerRegister);
}
void JIT::compileCallEval(Instruction* instruction)
{
addPtr(TrustedImm32(-static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC))), stackPointerRegister, regT1);
storePtr(callFrameRegister, Address(regT1, CallFrame::callerFrameOffset()));
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
callOperation(operationCallEval, regT1);
addSlowCase(branch32(Equal, regT1, TrustedImm32(JSValue::EmptyValueTag)));
sampleCodeBlock(m_codeBlock);
emitPutCallResult(instruction);
}
bool JIT::compileCallEval(const OpCallEval& bytecode)
{
addPtr(TrustedImm32(-static_cast<ptrdiff_t>(sizeof(CallerFrameAndPC))), stackPointerRegister, regT1);
storePtr(callFrameRegister, Address(regT1, CallFrame::callerFrameOffset()));
addPtr(TrustedImm32(stackPointerOffsetFor(m_codeBlock) * sizeof(Register)), callFrameRegister, stackPointerRegister);
callOperation(operationCallEval, regT1);
addSlowCase(branchIfEmpty(regT1));
sampleCodeBlock(m_codeBlock);
emitPutCallResult(bytecode);
return true;
}
string DxOperation::runThroughDxList(const string &dxName,
Site *site)
{
Assert(site);
DxList dxList;
site->dxManager()->getProxyList(&dxList);
dxList.sort(compareDxsByName);
string resultString;
bool found = false;
for (DxList::iterator index = dxList.begin();
index != dxList.end(); ++index)
{
DxProxy *dxProxy = *index;
Assert(dxProxy);
if ( (dxName == "all") || dxProxy->getName() == dxName)
{
found = true;
resultString += callOperation(dxProxy);
}
}
if (found)
{
// if the command had no output text of its own,
// return a generic acknowledgement.
if (resultString.empty())
{
resultString = "dx command sent.";
}
}
else
{
resultString = "The requested DX(s) are not connected: " + dxName;
}
return resultString;
}
void JIT::compileLoadVarargs(Instruction* instruction)
{
int thisValue = instruction[3].u.operand;
int arguments = instruction[4].u.operand;
int firstFreeRegister = instruction[5].u.operand;
JumpList slowCase;
JumpList end;
bool canOptimize = m_codeBlock->usesArguments()
&& VirtualRegister(arguments) == m_codeBlock->argumentsRegister()
&& !m_codeBlock->symbolTable()->slowArguments();
if (canOptimize) {
emitLoadTag(arguments, regT1);
slowCase.append(branch32(NotEqual, regT1, TrustedImm32(JSValue::EmptyValueTag)));
load32(payloadFor(JSStack::ArgumentCount), regT2);
slowCase.append(branch32(Above, regT2, TrustedImm32(Arguments::MaxArguments + 1)));
// regT2: argumentCountIncludingThis
move(regT2, regT3);
neg32(regT3);
add32(TrustedImm32(firstFreeRegister - JSStack::CallFrameHeaderSize), regT3);
lshift32(TrustedImm32(3), regT3);
addPtr(callFrameRegister, regT3);
// regT3: newCallFrame
slowCase.append(branchPtr(Above, AbsoluteAddress(m_vm->interpreter->stack().addressOfEnd()), regT3));
// Initialize ArgumentCount.
store32(regT2, payloadFor(JSStack::ArgumentCount, regT3));
// Initialize 'this'.
emitLoad(thisValue, regT1, regT0);
store32(regT0, Address(regT3, OBJECT_OFFSETOF(JSValue, u.asBits.payload) + (CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))));
store32(regT1, Address(regT3, OBJECT_OFFSETOF(JSValue, u.asBits.tag) + (CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))));
// Copy arguments.
end.append(branchSub32(Zero, TrustedImm32(1), regT2));
// regT2: argumentCount;
Label copyLoop = label();
load32(BaseIndex(callFrameRegister, regT2, TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload) +(CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))), regT0);
load32(BaseIndex(callFrameRegister, regT2, TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag) +(CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))), regT1);
store32(regT0, BaseIndex(regT3, regT2, TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.payload) +(CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))));
store32(regT1, BaseIndex(regT3, regT2, TimesEight, OBJECT_OFFSETOF(JSValue, u.asBits.tag) +(CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))));
branchSub32(NonZero, TrustedImm32(1), regT2).linkTo(copyLoop, this);
end.append(jump());
}
if (canOptimize)
slowCase.link(this);
emitLoad(thisValue, regT1, regT0);
emitLoad(arguments, regT3, regT2);
callOperation(operationLoadVarargs, regT1, regT0, regT3, regT2, firstFreeRegister);
move(returnValueRegister, regT3);
if (canOptimize)
end.link(this);
}
// -----------------------------------------------------------------------------
// CMmConferenceCallMessHandler::CallControlInd
// Breaks a CALL_CONTROL_IND ISI-message. Completes failed
// CreateConference, AddCall or Swap API request if there is a cause sub
// block in this response. In case that this message contains Split operation
// sub block, inform ConferenceCallTsy that GoOneToOne handling has started
// -----------------------------------------------------------------------------
//
void CMmConferenceCallMessHandler::CallControlInd(
const TIsiReceiveC& aIsiMsg )
{
TFLOGSTRING("TSY: CMmConferenceCallMessHandler::CallControlInd");
OstTrace0( TRACE_NORMAL, CMMCONFERENCECALLMESSHANDLER_CALLCONTROLIND_TD, "CMmConferenceCallMessHandler::CallControlInd" );
TUint sbStartOffSet( 0 );
// Try to find CALL_MODEM_SB_CAUSE subblock. If found something has gone wrong
if( KErrNone == aIsiMsg.FindSubBlockOffsetById(
ISI_HEADER_SIZE + SIZE_CALL_MODEM_CONTROL_IND,
CALL_MODEM_SB_CAUSE,
EIsiSubBlockTypeId8Len8,
sbStartOffSet ) )
{
// CALL_MODEM_SB_CAUSE sub block found, something has gone wrong.
// Get cause type
TUint8 causeType( aIsiMsg.Get8bit(
sbStartOffSet + CALL_MODEM_SB_CAUSE_OFFSET_CAUSETYPE ) );
// Get cause value
TUint8 causeValue( aIsiMsg.Get8bit(
sbStartOffSet + CALL_MODEM_SB_CAUSE_OFFSET_CAUSE ) );
// Get Symbian OS error value
TInt result( CMmStaticUtility::CSCauseToEpocError(
PN_MODEM_CALL,
causeType,
causeValue ) );
// If last operation was build, CreateConference or AddCall was asked
if ( CALL_MODEM_OP_CONFERENCE_BUILD == iLastOperation )
{
// Has CreateConference been requested last.
if ( iIsCreateConference )
{
// Complete with error
iMessageRouter->Complete(
EMobileConferenceCallCreateConference,
result);
// Reset flag
iIsCreateConference = EFalse;
}
else
{
// Must be response to AddCall request
iMessageRouter->Complete(
EMobileConferenceCallAddCall,
result );
}
iLastOperation = 0;
}
// if last operation was hold or retrieve
else if ( CALL_MODEM_OP_HOLD == iLastOperation ||
CALL_MODEM_OP_RETRIEVE == iLastOperation )
{
// Complete swap with error
iMessageRouter->Complete(
EMobileConferenceCallSwap,
result );
}
// No else
}
// Successful operation handling for operations
// Initiated bypassing the ETel (ghost operetion).
else if ( !iIsCreateConference && 0 == iLastOperation )
{
TUint8 callOperation( 0 );
// Get call Id (three bits (1-3) contain unique call ID)
TInt uniqueCallId( static_cast<TInt>( aIsiMsg.Get8bit(
ISI_HEADER_SIZE + CALL_MODEM_CONTROL_IND_OFFSET_CALLID )
& KMaskBits1to3 ) );
// Get call operation sub block
if ( KErrNone == aIsiMsg.FindSubBlockOffsetById(
ISI_HEADER_SIZE + CALL_MODEM_CONTROL_IND_OFFSET_OPERATION,
CALL_MODEM_SB_OPERATION,
EIsiSubBlockTypeId8Len8,
sbStartOffSet ) )
{
TInt symbianConferenceEvent( -1 );
// Get call operation
callOperation = aIsiMsg.Get8bit(
sbStartOffSet + CALL_MODEM_SB_OPERATION_OFFSET_OPERATION );
TFLOGSTRING2("TSY: CMmConferenceCallMessHandler::CallControlInd - Call operation: %d", callOperation);
OstTraceExt1( TRACE_NORMAL, DUP1_CMMCONFERENCECALLMESSHANDLER_CALLCONTROLIND_TD, "CMmConferenceCallMessHandler::CallControlInd;callOperation=%hhu", callOperation );
// Purpose of this case is to prepare handling of externally
// controlled conference call
switch ( callOperation )
{
case CALL_MODEM_OP_CONFERENCE_BUILD:
{
symbianConferenceEvent =
RMobileConferenceCall::EConferenceBuilt;
break;
}
case CALL_MODEM_OP_HOLD:
case CALL_MODEM_OP_SWAP:
{
symbianConferenceEvent =
RMobileConferenceCall::EConferenceSwapped;
break;
}
case CALL_MODEM_OP_CONFERENCE_SPLIT:
{
symbianConferenceEvent =
RMobileConferenceCall::EConferenceSplit;
break;
}
default:
{
// Nothing to do
break;
}
}
// Update the information to Conference call
if ( 0 <= symbianConferenceEvent )
{
// Create call package
CCallDataPackage callData;
// Pack the data for sending to the manager
callData.SetCallIdAndMode(
uniqueCallId,
RMobilePhone::EVoiceService );
// Pack conference event
callData.PackData( &symbianConferenceEvent );
// Pass call information to the Symbian OS layer
// complete mobile call info change
iMessageRouter->Complete(
EMobileConferenceCallNotifyConferenceEvent,
&callData,
KErrNone );
}
// No else
}
// No else
}
// No else
iIsCreateConference = EFalse;
iLastOperation = 0;
}
static void fixFunctionBasedOnStackMaps(
State& state, CodeBlock* codeBlock, JITCode* jitCode, GeneratedFunction generatedFunction,
StackMaps::RecordMap& recordMap, bool didSeeUnwindInfo)
{
Graph& graph = state.graph;
VM& vm = graph.m_vm;
StackMaps stackmaps = jitCode->stackmaps;
int localsOffset =
offsetOfStackRegion(recordMap, state.capturedStackmapID) + graph.m_nextMachineLocal;
int varargsSpillSlotsOffset;
if (state.varargsSpillSlotsStackmapID != UINT_MAX)
varargsSpillSlotsOffset = offsetOfStackRegion(recordMap, state.varargsSpillSlotsStackmapID);
else
varargsSpillSlotsOffset = 0;
for (unsigned i = graph.m_inlineVariableData.size(); i--;) {
InlineCallFrame* inlineCallFrame = graph.m_inlineVariableData[i].inlineCallFrame;
if (inlineCallFrame->argumentsRegister.isValid())
inlineCallFrame->argumentsRegister += localsOffset;
if (inlineCallFrame->argumentCountRegister.isValid())
inlineCallFrame->argumentCountRegister += localsOffset;
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
inlineCallFrame->arguments[argument] =
inlineCallFrame->arguments[argument].withLocalsOffset(localsOffset);
}
if (inlineCallFrame->isClosureCall) {
inlineCallFrame->calleeRecovery =
inlineCallFrame->calleeRecovery.withLocalsOffset(localsOffset);
}
}
if (codeBlock->usesArguments()) {
codeBlock->setArgumentsRegister(
VirtualRegister(codeBlock->argumentsRegister().offset() + localsOffset));
}
MacroAssembler::Label stackOverflowException;
{
CCallHelpers checkJIT(&vm, codeBlock);
// At this point it's perfectly fair to just blow away all state and restore the
// JS JIT view of the universe.
checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0);
checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
MacroAssembler::Call callLookupExceptionHandler = checkJIT.call();
checkJIT.jumpToExceptionHandler();
stackOverflowException = checkJIT.label();
checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0);
checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
MacroAssembler::Call callLookupExceptionHandlerFromCallerFrame = checkJIT.call();
checkJIT.jumpToExceptionHandler();
auto linkBuffer = std::make_unique<LinkBuffer>(
vm, checkJIT, codeBlock, JITCompilationMustSucceed);
linkBuffer->link(callLookupExceptionHandler, FunctionPtr(lookupExceptionHandler));
linkBuffer->link(callLookupExceptionHandlerFromCallerFrame, FunctionPtr(lookupExceptionHandlerFromCallerFrame));
state.finalizer->handleExceptionsLinkBuffer = WTF::move(linkBuffer);
}
ExitThunkGenerator exitThunkGenerator(state);
exitThunkGenerator.emitThunks();
if (exitThunkGenerator.didThings()) {
RELEASE_ASSERT(state.finalizer->osrExit.size());
RELEASE_ASSERT(didSeeUnwindInfo);
auto linkBuffer = std::make_unique<LinkBuffer>(
vm, exitThunkGenerator, codeBlock, JITCompilationMustSucceed);
RELEASE_ASSERT(state.finalizer->osrExit.size() == state.jitCode->osrExit.size());
for (unsigned i = 0; i < state.jitCode->osrExit.size(); ++i) {
OSRExitCompilationInfo& info = state.finalizer->osrExit[i];
OSRExit& exit = jitCode->osrExit[i];
if (verboseCompilationEnabled())
dataLog("Handling OSR stackmap #", exit.m_stackmapID, " for ", exit.m_codeOrigin, "\n");
auto iter = recordMap.find(exit.m_stackmapID);
if (iter == recordMap.end()) {
// It was optimized out.
continue;
}
info.m_thunkAddress = linkBuffer->locationOf(info.m_thunkLabel);
exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump);
for (unsigned j = exit.m_values.size(); j--;) {
ExitValue value = exit.m_values[j];
if (!value.isInJSStackSomehow())
continue;
if (!value.virtualRegister().isLocal())
continue;
exit.m_values[j] = value.withVirtualRegister(
VirtualRegister(value.virtualRegister().offset() + localsOffset));
}
if (verboseCompilationEnabled()) {
DumpContext context;
dataLog(" Exit values: ", inContext(exit.m_values, &context), "\n");
}
}
state.finalizer->exitThunksLinkBuffer = WTF::move(linkBuffer);
}
if (!state.getByIds.isEmpty() || !state.putByIds.isEmpty() || !state.checkIns.isEmpty()) {
CCallHelpers slowPathJIT(&vm, codeBlock);
CCallHelpers::JumpList exceptionTarget;
for (unsigned i = state.getByIds.size(); i--;) {
GetByIdDescriptor& getById = state.getByIds[i];
if (verboseCompilationEnabled())
dataLog("Handling GetById stackmap #", getById.stackmapID(), "\n");
auto iter = recordMap.find(getById.stackmapID());
if (iter == recordMap.end()) {
// It was optimized out.
continue;
}
for (unsigned i = 0; i < iter->value.size(); ++i) {
StackMaps::Record& record = iter->value[i];
RegisterSet usedRegisters = usedRegistersFor(record);
GPRReg result = record.locations[0].directGPR();
GPRReg base = record.locations[1].directGPR();
JITGetByIdGenerator gen(
codeBlock, getById.codeOrigin(), usedRegisters, JSValueRegs(base),
JSValueRegs(result), NeedToSpill);
MacroAssembler::Label begin = slowPathJIT.label();
MacroAssembler::Call call = callOperation(
state, usedRegisters, slowPathJIT, getById.codeOrigin(), &exceptionTarget,
operationGetByIdOptimize, result, gen.stubInfo(), base, getById.uid());
gen.reportSlowPathCall(begin, call);
getById.m_slowPathDone.append(slowPathJIT.jump());
getById.m_generators.append(gen);
}
}
for (unsigned i = state.putByIds.size(); i--;) {
PutByIdDescriptor& putById = state.putByIds[i];
if (verboseCompilationEnabled())
dataLog("Handling PutById stackmap #", putById.stackmapID(), "\n");
auto iter = recordMap.find(putById.stackmapID());
if (iter == recordMap.end()) {
// It was optimized out.
continue;
}
for (unsigned i = 0; i < iter->value.size(); ++i) {
StackMaps::Record& record = iter->value[i];
RegisterSet usedRegisters = usedRegistersFor(record);
GPRReg base = record.locations[0].directGPR();
GPRReg value = record.locations[1].directGPR();
JITPutByIdGenerator gen(
codeBlock, putById.codeOrigin(), usedRegisters, JSValueRegs(base),
JSValueRegs(value), GPRInfo::patchpointScratchRegister, NeedToSpill,
putById.ecmaMode(), putById.putKind());
MacroAssembler::Label begin = slowPathJIT.label();
MacroAssembler::Call call = callOperation(
state, usedRegisters, slowPathJIT, putById.codeOrigin(), &exceptionTarget,
gen.slowPathFunction(), gen.stubInfo(), value, base, putById.uid());
gen.reportSlowPathCall(begin, call);
putById.m_slowPathDone.append(slowPathJIT.jump());
putById.m_generators.append(gen);
}
}
for (unsigned i = state.checkIns.size(); i--;) {
CheckInDescriptor& checkIn = state.checkIns[i];
if (verboseCompilationEnabled())
dataLog("Handling checkIn stackmap #", checkIn.stackmapID(), "\n");
auto iter = recordMap.find(checkIn.stackmapID());
if (iter == recordMap.end()) {
// It was optimized out.
continue;
}
for (unsigned i = 0; i < iter->value.size(); ++i) {
StackMaps::Record& record = iter->value[i];
RegisterSet usedRegisters = usedRegistersFor(record);
GPRReg result = record.locations[0].directGPR();
GPRReg obj = record.locations[1].directGPR();
StructureStubInfo* stubInfo = codeBlock->addStubInfo();
stubInfo->codeOrigin = checkIn.codeOrigin();
stubInfo->patch.baseGPR = static_cast<int8_t>(obj);
stubInfo->patch.valueGPR = static_cast<int8_t>(result);
stubInfo->patch.usedRegisters = usedRegisters;
stubInfo->patch.spillMode = NeedToSpill;
MacroAssembler::Label begin = slowPathJIT.label();
MacroAssembler::Call slowCall = callOperation(
state, usedRegisters, slowPathJIT, checkIn.codeOrigin(), &exceptionTarget,
operationInOptimize, result, stubInfo, obj, checkIn.m_id);
checkIn.m_slowPathDone.append(slowPathJIT.jump());
checkIn.m_generators.append(CheckInGenerator(stubInfo, slowCall, begin));
}
}
exceptionTarget.link(&slowPathJIT);
MacroAssembler::Jump exceptionJump = slowPathJIT.jump();
state.finalizer->sideCodeLinkBuffer = std::make_unique<LinkBuffer>(vm, slowPathJIT, codeBlock, JITCompilationMustSucceed);
state.finalizer->sideCodeLinkBuffer->link(
exceptionJump, state.finalizer->handleExceptionsLinkBuffer->entrypoint());
for (unsigned i = state.getByIds.size(); i--;) {
generateICFastPath(
state, codeBlock, generatedFunction, recordMap, state.getByIds[i],
sizeOfGetById());
}
for (unsigned i = state.putByIds.size(); i--;) {
generateICFastPath(
state, codeBlock, generatedFunction, recordMap, state.putByIds[i],
sizeOfPutById());
}
for (unsigned i = state.checkIns.size(); i--;) {
generateCheckInICFastPath(
state, codeBlock, generatedFunction, recordMap, state.checkIns[i],
sizeOfIn());
}
}
adjustCallICsForStackmaps(state.jsCalls, recordMap);
for (unsigned i = state.jsCalls.size(); i--;) {
JSCall& call = state.jsCalls[i];
CCallHelpers fastPathJIT(&vm, codeBlock);
call.emit(fastPathJIT);
char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset;
LinkBuffer linkBuffer(vm, fastPathJIT, startOfIC, sizeOfCall());
if (!linkBuffer.isValid()) {
dataLog("Failed to insert inline cache for call because we thought the size would be ", sizeOfCall(), " but it ended up being ", fastPathJIT.m_assembler.codeSize(), " prior to compaction.\n");
RELEASE_ASSERT_NOT_REACHED();
}
MacroAssembler::AssemblerType_T::fillNops(
startOfIC + linkBuffer.size(), sizeOfCall() - linkBuffer.size());
call.link(vm, linkBuffer);
}
adjustCallICsForStackmaps(state.jsCallVarargses, recordMap);
for (unsigned i = state.jsCallVarargses.size(); i--;) {
JSCallVarargs& call = state.jsCallVarargses[i];
CCallHelpers fastPathJIT(&vm, codeBlock);
call.emit(fastPathJIT, graph, varargsSpillSlotsOffset);
char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset;
size_t sizeOfIC = sizeOfICFor(call.node());
LinkBuffer linkBuffer(vm, fastPathJIT, startOfIC, sizeOfIC);
if (!linkBuffer.isValid()) {
dataLog("Failed to insert inline cache for varargs call (specifically, ", Graph::opName(call.node()->op()), ") because we thought the size would be ", sizeOfIC, " but it ended up being ", fastPathJIT.m_assembler.codeSize(), " prior to compaction.\n");
RELEASE_ASSERT_NOT_REACHED();
}
MacroAssembler::AssemblerType_T::fillNops(
startOfIC + linkBuffer.size(), sizeOfIC - linkBuffer.size());
call.link(vm, linkBuffer, state.finalizer->handleExceptionsLinkBuffer->entrypoint());
}
RepatchBuffer repatchBuffer(codeBlock);
auto iter = recordMap.find(state.handleStackOverflowExceptionStackmapID);
// It's sort of remotely possible that we won't have an in-band exception handling
// path, for some kinds of functions.
if (iter != recordMap.end()) {
for (unsigned i = iter->value.size(); i--;) {
StackMaps::Record& record = iter->value[i];
CodeLocationLabel source = CodeLocationLabel(
bitwise_cast<char*>(generatedFunction) + record.instructionOffset);
RELEASE_ASSERT(stackOverflowException.isSet());
repatchBuffer.replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->locationOf(stackOverflowException));
}
}
iter = recordMap.find(state.handleExceptionStackmapID);
// It's sort of remotely possible that we won't have an in-band exception handling
// path, for some kinds of functions.
if (iter != recordMap.end()) {
for (unsigned i = iter->value.size(); i--;) {
StackMaps::Record& record = iter->value[i];
CodeLocationLabel source = CodeLocationLabel(
bitwise_cast<char*>(generatedFunction) + record.instructionOffset);
repatchBuffer.replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->entrypoint());
}
}
for (unsigned exitIndex = 0; exitIndex < jitCode->osrExit.size(); ++exitIndex) {
OSRExitCompilationInfo& info = state.finalizer->osrExit[exitIndex];
OSRExit& exit = jitCode->osrExit[exitIndex];
iter = recordMap.find(exit.m_stackmapID);
Vector<const void*> codeAddresses;
if (iter != recordMap.end()) {
for (unsigned i = iter->value.size(); i--;) {
StackMaps::Record& record = iter->value[i];
CodeLocationLabel source = CodeLocationLabel(
bitwise_cast<char*>(generatedFunction) + record.instructionOffset);
codeAddresses.append(bitwise_cast<char*>(generatedFunction) + record.instructionOffset + MacroAssembler::maxJumpReplacementSize());
if (info.m_isInvalidationPoint)
jitCode->common.jumpReplacements.append(JumpReplacement(source, info.m_thunkAddress));
else
repatchBuffer.replaceWithJump(source, info.m_thunkAddress);
}
}
if (graph.compilation())
graph.compilation()->addOSRExitSite(codeAddresses);
}
}
static void fixFunctionBasedOnStackMaps(
State& state, CodeBlock* codeBlock, JITCode* jitCode, GeneratedFunction generatedFunction,
StackMaps::RecordMap& recordMap)
{
Graph& graph = state.graph;
VM& vm = graph.m_vm;
StackMaps stackmaps = jitCode->stackmaps;
int localsOffset = offsetOfStackRegion(recordMap, state.capturedStackmapID) + graph.m_nextMachineLocal;
int varargsSpillSlotsOffset = offsetOfStackRegion(recordMap, state.varargsSpillSlotsStackmapID);
for (unsigned i = graph.m_inlineVariableData.size(); i--;) {
InlineCallFrame* inlineCallFrame = graph.m_inlineVariableData[i].inlineCallFrame;
if (inlineCallFrame->argumentCountRegister.isValid())
inlineCallFrame->argumentCountRegister += localsOffset;
for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) {
inlineCallFrame->arguments[argument] =
inlineCallFrame->arguments[argument].withLocalsOffset(localsOffset);
}
if (inlineCallFrame->isClosureCall) {
inlineCallFrame->calleeRecovery =
inlineCallFrame->calleeRecovery.withLocalsOffset(localsOffset);
}
if (graph.hasDebuggerEnabled())
codeBlock->setScopeRegister(codeBlock->scopeRegister() + localsOffset);
}
MacroAssembler::Label stackOverflowException;
{
CCallHelpers checkJIT(&vm, codeBlock);
// At this point it's perfectly fair to just blow away all state and restore the
// JS JIT view of the universe.
checkJIT.copyCalleeSavesToVMCalleeSavesBuffer();
checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0);
checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
MacroAssembler::Call callLookupExceptionHandler = checkJIT.call();
checkJIT.jumpToExceptionHandler();
stackOverflowException = checkJIT.label();
checkJIT.copyCalleeSavesToVMCalleeSavesBuffer();
checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0);
checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1);
MacroAssembler::Call callLookupExceptionHandlerFromCallerFrame = checkJIT.call();
checkJIT.jumpToExceptionHandler();
auto linkBuffer = std::make_unique<LinkBuffer>(
vm, checkJIT, codeBlock, JITCompilationCanFail);
if (linkBuffer->didFailToAllocate()) {
state.allocationFailed = true;
return;
}
linkBuffer->link(callLookupExceptionHandler, FunctionPtr(lookupExceptionHandler));
linkBuffer->link(callLookupExceptionHandlerFromCallerFrame, FunctionPtr(lookupExceptionHandlerFromCallerFrame));
state.finalizer->handleExceptionsLinkBuffer = WTF::move(linkBuffer);
}
ExitThunkGenerator exitThunkGenerator(state);
exitThunkGenerator.emitThunks();
if (exitThunkGenerator.didThings()) {
RELEASE_ASSERT(state.finalizer->osrExit.size());
auto linkBuffer = std::make_unique<LinkBuffer>(
vm, exitThunkGenerator, codeBlock, JITCompilationCanFail);
if (linkBuffer->didFailToAllocate()) {
state.allocationFailed = true;
return;
}
RELEASE_ASSERT(state.finalizer->osrExit.size() == state.jitCode->osrExit.size());
for (unsigned i = 0; i < state.jitCode->osrExit.size(); ++i) {
OSRExitCompilationInfo& info = state.finalizer->osrExit[i];
OSRExit& exit = jitCode->osrExit[i];
if (verboseCompilationEnabled())
dataLog("Handling OSR stackmap #", exit.m_stackmapID, " for ", exit.m_codeOrigin, "\n");
auto iter = recordMap.find(exit.m_stackmapID);
if (iter == recordMap.end()) {
// It was optimized out.
continue;
}
info.m_thunkAddress = linkBuffer->locationOf(info.m_thunkLabel);
exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump);
for (unsigned j = exit.m_values.size(); j--;)
exit.m_values[j] = exit.m_values[j].withLocalsOffset(localsOffset);
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations)
materialization->accountForLocalsOffset(localsOffset);
if (verboseCompilationEnabled()) {
DumpContext context;
dataLog(" Exit values: ", inContext(exit.m_values, &context), "\n");
if (!exit.m_materializations.isEmpty()) {
dataLog(" Materializations: \n");
for (ExitTimeObjectMaterialization* materialization : exit.m_materializations)
dataLog(" Materialize(", pointerDump(materialization), ")\n");
}
}
}
state.finalizer->exitThunksLinkBuffer = WTF::move(linkBuffer);
}
if (!state.getByIds.isEmpty() || !state.putByIds.isEmpty() || !state.checkIns.isEmpty()) {
CCallHelpers slowPathJIT(&vm, codeBlock);
CCallHelpers::JumpList exceptionTarget;
for (unsigned i = state.getByIds.size(); i--;) {
GetByIdDescriptor& getById = state.getByIds[i];
if (verboseCompilationEnabled())
dataLog("Handling GetById stackmap #", getById.stackmapID(), "\n");
auto iter = recordMap.find(getById.stackmapID());
if (iter == recordMap.end()) {
// It was optimized out.
continue;
}
CodeOrigin codeOrigin = state.jitCode->common.codeOrigins[getById.callSiteIndex().bits()];
for (unsigned i = 0; i < iter->value.size(); ++i) {
StackMaps::Record& record = iter->value[i];
RegisterSet usedRegisters = usedRegistersFor(record);
GPRReg result = record.locations[0].directGPR();
GPRReg base = record.locations[1].directGPR();
JITGetByIdGenerator gen(
codeBlock, codeOrigin, getById.callSiteIndex(), usedRegisters, JSValueRegs(base),
JSValueRegs(result), NeedToSpill);
MacroAssembler::Label begin = slowPathJIT.label();
MacroAssembler::Call call = callOperation(
state, usedRegisters, slowPathJIT, codeOrigin, &exceptionTarget,
operationGetByIdOptimize, result, gen.stubInfo(), base, getById.uid());
gen.reportSlowPathCall(begin, call);
getById.m_slowPathDone.append(slowPathJIT.jump());
getById.m_generators.append(gen);
}
}
for (unsigned i = state.putByIds.size(); i--;) {
PutByIdDescriptor& putById = state.putByIds[i];
if (verboseCompilationEnabled())
dataLog("Handling PutById stackmap #", putById.stackmapID(), "\n");
auto iter = recordMap.find(putById.stackmapID());
if (iter == recordMap.end()) {
// It was optimized out.
continue;
}
CodeOrigin codeOrigin = state.jitCode->common.codeOrigins[putById.callSiteIndex().bits()];
for (unsigned i = 0; i < iter->value.size(); ++i) {
StackMaps::Record& record = iter->value[i];
RegisterSet usedRegisters = usedRegistersFor(record);
GPRReg base = record.locations[0].directGPR();
GPRReg value = record.locations[1].directGPR();
JITPutByIdGenerator gen(
codeBlock, codeOrigin, putById.callSiteIndex(), usedRegisters, JSValueRegs(base),
JSValueRegs(value), GPRInfo::patchpointScratchRegister, NeedToSpill,
putById.ecmaMode(), putById.putKind());
MacroAssembler::Label begin = slowPathJIT.label();
MacroAssembler::Call call = callOperation(
state, usedRegisters, slowPathJIT, codeOrigin, &exceptionTarget,
gen.slowPathFunction(), gen.stubInfo(), value, base, putById.uid());
gen.reportSlowPathCall(begin, call);
putById.m_slowPathDone.append(slowPathJIT.jump());
putById.m_generators.append(gen);
}
}
for (unsigned i = state.checkIns.size(); i--;) {
CheckInDescriptor& checkIn = state.checkIns[i];
if (verboseCompilationEnabled())
dataLog("Handling checkIn stackmap #", checkIn.stackmapID(), "\n");
auto iter = recordMap.find(checkIn.stackmapID());
if (iter == recordMap.end()) {
// It was optimized out.
continue;
}
CodeOrigin codeOrigin = state.jitCode->common.codeOrigins[checkIn.callSiteIndex().bits()];
for (unsigned i = 0; i < iter->value.size(); ++i) {
StackMaps::Record& record = iter->value[i];
RegisterSet usedRegisters = usedRegistersFor(record);
GPRReg result = record.locations[0].directGPR();
GPRReg obj = record.locations[1].directGPR();
StructureStubInfo* stubInfo = codeBlock->addStubInfo(AccessType::In);
stubInfo->codeOrigin = codeOrigin;
stubInfo->callSiteIndex = checkIn.callSiteIndex();
stubInfo->patch.baseGPR = static_cast<int8_t>(obj);
stubInfo->patch.valueGPR = static_cast<int8_t>(result);
stubInfo->patch.usedRegisters = usedRegisters;
stubInfo->patch.spillMode = NeedToSpill;
MacroAssembler::Label begin = slowPathJIT.label();
MacroAssembler::Call slowCall = callOperation(
state, usedRegisters, slowPathJIT, codeOrigin, &exceptionTarget,
operationInOptimize, result, stubInfo, obj, checkIn.m_uid);
checkIn.m_slowPathDone.append(slowPathJIT.jump());
checkIn.m_generators.append(CheckInGenerator(stubInfo, slowCall, begin));
}
}
exceptionTarget.link(&slowPathJIT);
MacroAssembler::Jump exceptionJump = slowPathJIT.jump();
state.finalizer->sideCodeLinkBuffer = std::make_unique<LinkBuffer>(vm, slowPathJIT, codeBlock, JITCompilationCanFail);
if (state.finalizer->sideCodeLinkBuffer->didFailToAllocate()) {
state.allocationFailed = true;
return;
}
state.finalizer->sideCodeLinkBuffer->link(
exceptionJump, state.finalizer->handleExceptionsLinkBuffer->entrypoint());
for (unsigned i = state.getByIds.size(); i--;) {
generateICFastPath(
state, codeBlock, generatedFunction, recordMap, state.getByIds[i],
sizeOfGetById());
}
for (unsigned i = state.putByIds.size(); i--;) {
generateICFastPath(
state, codeBlock, generatedFunction, recordMap, state.putByIds[i],
sizeOfPutById());
}
for (unsigned i = state.checkIns.size(); i--;) {
generateCheckInICFastPath(
state, codeBlock, generatedFunction, recordMap, state.checkIns[i],
sizeOfIn());
}
}
adjustCallICsForStackmaps(state.jsCalls, recordMap);
for (unsigned i = state.jsCalls.size(); i--;) {
JSCall& call = state.jsCalls[i];
CCallHelpers fastPathJIT(&vm, codeBlock);
call.emit(fastPathJIT, state.jitCode->stackmaps.stackSizeForLocals());
char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset;
generateInlineIfPossibleOutOfLineIfNot(state, vm, codeBlock, fastPathJIT, startOfIC, sizeOfCall(), "JSCall inline cache", [&] (LinkBuffer& linkBuffer, CCallHelpers&, bool) {
call.link(vm, linkBuffer);
});
}
adjustCallICsForStackmaps(state.jsCallVarargses, recordMap);
for (unsigned i = state.jsCallVarargses.size(); i--;) {
JSCallVarargs& call = state.jsCallVarargses[i];
CCallHelpers fastPathJIT(&vm, codeBlock);
call.emit(fastPathJIT, varargsSpillSlotsOffset);
char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset;
size_t sizeOfIC = sizeOfICFor(call.node());
generateInlineIfPossibleOutOfLineIfNot(state, vm, codeBlock, fastPathJIT, startOfIC, sizeOfIC, "varargs call inline cache", [&] (LinkBuffer& linkBuffer, CCallHelpers&, bool) {
call.link(vm, linkBuffer, state.finalizer->handleExceptionsLinkBuffer->entrypoint());
});
}
adjustCallICsForStackmaps(state.jsTailCalls, recordMap);
for (unsigned i = state.jsTailCalls.size(); i--;) {
JSTailCall& call = state.jsTailCalls[i];
CCallHelpers fastPathJIT(&vm, codeBlock);
call.emit(*state.jitCode.get(), fastPathJIT);
char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset;
size_t sizeOfIC = call.estimatedSize();
generateInlineIfPossibleOutOfLineIfNot(state, vm, codeBlock, fastPathJIT, startOfIC, sizeOfIC, "tail call inline cache", [&] (LinkBuffer& linkBuffer, CCallHelpers&, bool) {
call.link(vm, linkBuffer);
});
}
auto iter = recordMap.find(state.handleStackOverflowExceptionStackmapID);
// It's sort of remotely possible that we won't have an in-band exception handling
// path, for some kinds of functions.
if (iter != recordMap.end()) {
for (unsigned i = iter->value.size(); i--;) {
StackMaps::Record& record = iter->value[i];
CodeLocationLabel source = CodeLocationLabel(
bitwise_cast<char*>(generatedFunction) + record.instructionOffset);
RELEASE_ASSERT(stackOverflowException.isSet());
MacroAssembler::replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->locationOf(stackOverflowException));
}
}
iter = recordMap.find(state.handleExceptionStackmapID);
// It's sort of remotely possible that we won't have an in-band exception handling
// path, for some kinds of functions.
if (iter != recordMap.end()) {
for (unsigned i = iter->value.size(); i--;) {
StackMaps::Record& record = iter->value[i];
CodeLocationLabel source = CodeLocationLabel(
bitwise_cast<char*>(generatedFunction) + record.instructionOffset);
MacroAssembler::replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->entrypoint());
}
}
for (unsigned exitIndex = 0; exitIndex < jitCode->osrExit.size(); ++exitIndex) {
OSRExitCompilationInfo& info = state.finalizer->osrExit[exitIndex];
OSRExit& exit = jitCode->osrExit[exitIndex];
iter = recordMap.find(exit.m_stackmapID);
Vector<const void*> codeAddresses;
if (iter != recordMap.end()) {
for (unsigned i = iter->value.size(); i--;) {
StackMaps::Record& record = iter->value[i];
CodeLocationLabel source = CodeLocationLabel(
bitwise_cast<char*>(generatedFunction) + record.instructionOffset);
codeAddresses.append(bitwise_cast<char*>(generatedFunction) + record.instructionOffset + MacroAssembler::maxJumpReplacementSize());
if (info.m_isInvalidationPoint)
jitCode->common.jumpReplacements.append(JumpReplacement(source, info.m_thunkAddress));
else
MacroAssembler::replaceWithJump(source, info.m_thunkAddress);
}
}
if (graph.compilation())
graph.compilation()->addOSRExitSite(codeAddresses);
}
}
void JIT::compileLoadVarargs(Instruction* instruction)
{
int thisValue = instruction[3].u.operand;
int arguments = instruction[4].u.operand;
int firstFreeRegister = instruction[5].u.operand;
int firstVarArgOffset = instruction[6].u.operand;
JumpList slowCase;
JumpList end;
bool canOptimize = m_codeBlock->usesArguments()
&& arguments == m_codeBlock->argumentsRegister().offset()
&& !m_codeBlock->symbolTable()->slowArguments();
if (canOptimize) {
emitGetVirtualRegister(arguments, regT0);
slowCase.append(branch64(NotEqual, regT0, TrustedImm64(JSValue::encode(JSValue()))));
emitGetFromCallFrameHeader32(JSStack::ArgumentCount, regT0);
if (firstVarArgOffset) {
Jump sufficientArguments = branch32(GreaterThan, regT0, TrustedImm32(firstVarArgOffset + 1));
move(TrustedImm32(1), regT0);
Jump endVarArgs = jump();
sufficientArguments.link(this);
sub32(TrustedImm32(firstVarArgOffset), regT0);
endVarArgs.link(this);
}
slowCase.append(branch32(Above, regT0, TrustedImm32(Arguments::MaxArguments + 1)));
// regT0: argumentCountIncludingThis
move(regT0, regT1);
add64(TrustedImm32(-firstFreeRegister + JSStack::CallFrameHeaderSize), regT1);
// regT1 now has the required frame size in Register units
// Round regT1 to next multiple of stackAlignmentRegisters()
add64(TrustedImm32(stackAlignmentRegisters() - 1), regT1);
and64(TrustedImm32(~(stackAlignmentRegisters() - 1)), regT1);
neg64(regT1);
lshift64(TrustedImm32(3), regT1);
addPtr(callFrameRegister, regT1);
// regT1: newCallFrame
slowCase.append(branchPtr(Above, AbsoluteAddress(m_vm->addressOfStackLimit()), regT1));
// Initialize ArgumentCount.
store32(regT0, Address(regT1, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
// Initialize 'this'.
emitGetVirtualRegister(thisValue, regT2);
store64(regT2, Address(regT1, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
// Copy arguments.
signExtend32ToPtr(regT0, regT0);
end.append(branchSub64(Zero, TrustedImm32(1), regT0));
// regT0: argumentCount
Label copyLoop = label();
load64(BaseIndex(callFrameRegister, regT0, TimesEight, (CallFrame::thisArgumentOffset() + firstVarArgOffset) * static_cast<int>(sizeof(Register))), regT2);
store64(regT2, BaseIndex(regT1, regT0, TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
branchSub64(NonZero, TrustedImm32(1), regT0).linkTo(copyLoop, this);
end.append(jump());
}
if (canOptimize)
slowCase.link(this);
emitGetVirtualRegister(arguments, regT1);
callOperation(operationSizeFrameForVarargs, regT1, firstFreeRegister, firstVarArgOffset);
move(returnValueGPR, stackPointerRegister);
emitGetVirtualRegister(thisValue, regT1);
emitGetVirtualRegister(arguments, regT2);
callOperation(operationLoadVarargs, returnValueGPR, regT1, regT2, firstVarArgOffset);
move(returnValueGPR, regT1);
if (canOptimize)
end.link(this);
addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), regT1, stackPointerRegister);
}
