void emitSetVarargsFrame(CCallHelpers& jit, GPRReg lengthGPR, bool lengthIncludesThis, GPRReg numUsedSlotsGPR, GPRReg resultGPR)
{
jit.move(numUsedSlotsGPR, resultGPR);
// We really want to make sure the size of the new call frame is a multiple of
// stackAlignmentRegisters(), however it is easier to accomplish this by
// rounding numUsedSlotsGPR to the next multiple of stackAlignmentRegisters().
// Together with the rounding below, we will assure that the new call frame is
// located on a stackAlignmentRegisters() boundary and a multiple of
// stackAlignmentRegisters() in size.
jit.addPtr(CCallHelpers::TrustedImm32(stackAlignmentRegisters() - 1), resultGPR);
jit.andPtr(CCallHelpers::TrustedImm32(~(stackAlignmentRegisters() - 1)), resultGPR);
jit.addPtr(lengthGPR, resultGPR);
jit.addPtr(CCallHelpers::TrustedImm32(JSStack::CallFrameHeaderSize + (lengthIncludesThis? 0 : 1)), resultGPR);
// resultGPR now has the required frame size in Register units
// Round resultGPR to next multiple of stackAlignmentRegisters()
jit.addPtr(CCallHelpers::TrustedImm32(stackAlignmentRegisters() - 1), resultGPR);
jit.andPtr(CCallHelpers::TrustedImm32(~(stackAlignmentRegisters() - 1)), resultGPR);
// Now resultGPR has the right stack frame offset in Register units.
jit.negPtr(resultGPR);
jit.lshiftPtr(CCallHelpers::Imm32(3), resultGPR);
jit.addPtr(GPRInfo::callFrameRegister, resultGPR);
}
void emitSetVarargsFrame(CCallHelpers& jit, GPRReg lengthGPR, bool lengthIncludesThis, GPRReg numUsedSlotsGPR, GPRReg resultGPR)
{
jit.move(numUsedSlotsGPR, resultGPR);
jit.addPtr(lengthGPR, resultGPR);
jit.addPtr(CCallHelpers::TrustedImm32(JSStack::CallFrameHeaderSize + (lengthIncludesThis? 0 : 1)), resultGPR);
// resultGPR now has the required frame size in Register units
// Round resultGPR to next multiple of stackAlignmentRegisters()
jit.addPtr(CCallHelpers::TrustedImm32(stackAlignmentRegisters() - 1), resultGPR);
jit.andPtr(CCallHelpers::TrustedImm32(~(stackAlignmentRegisters() - 1)), resultGPR);
// Now resultGPR has the right stack frame offset in Register units.
jit.negPtr(resultGPR);
jit.lshiftPtr(CCallHelpers::Imm32(3), resultGPR);
jit.addPtr(GPRInfo::callFrameRegister, resultGPR);
}
ptrdiff_t registerPreservationOffset()
{
unsigned numberOfCalleeSaves = registersToPreserve().numberOfSetRegisters();
// Need to preserve the old return PC.
unsigned numberOfValuesToSave = numberOfCalleeSaves + 1;
// Alignment. Preserve the same alignment invariants that the caller imposed.
unsigned numberOfNewStackSlots =
WTF::roundUpToMultipleOf(stackAlignmentRegisters(), numberOfValuesToSave);
return sizeof(Register) * numberOfNewStackSlots;
}
void JIT::compileLoadVarargs(Instruction* instruction)
{
int thisValue = instruction[3].u.operand;
int arguments = instruction[4].u.operand;
int firstFreeRegister = instruction[5].u.operand;
int firstVarArgOffset = instruction[6].u.operand;
JumpList slowCase;
JumpList end;
bool canOptimize = m_codeBlock->usesArguments()
&& arguments == m_codeBlock->argumentsRegister().offset()
&& !m_codeBlock->symbolTable()->slowArguments();
if (canOptimize) {
emitGetVirtualRegister(arguments, regT0);
slowCase.append(branch64(NotEqual, regT0, TrustedImm64(JSValue::encode(JSValue()))));
emitGetFromCallFrameHeader32(JSStack::ArgumentCount, regT0);
if (firstVarArgOffset) {
Jump sufficientArguments = branch32(GreaterThan, regT0, TrustedImm32(firstVarArgOffset + 1));
move(TrustedImm32(1), regT0);
Jump endVarArgs = jump();
sufficientArguments.link(this);
sub32(TrustedImm32(firstVarArgOffset), regT0);
endVarArgs.link(this);
}
slowCase.append(branch32(Above, regT0, TrustedImm32(Arguments::MaxArguments + 1)));
// regT0: argumentCountIncludingThis
move(regT0, regT1);
add64(TrustedImm32(-firstFreeRegister + JSStack::CallFrameHeaderSize), regT1);
// regT1 now has the required frame size in Register units
// Round regT1 to next multiple of stackAlignmentRegisters()
add64(TrustedImm32(stackAlignmentRegisters() - 1), regT1);
and64(TrustedImm32(~(stackAlignmentRegisters() - 1)), regT1);
neg64(regT1);
lshift64(TrustedImm32(3), regT1);
addPtr(callFrameRegister, regT1);
// regT1: newCallFrame
slowCase.append(branchPtr(Above, AbsoluteAddress(m_vm->addressOfStackLimit()), regT1));
// Initialize ArgumentCount.
store32(regT0, Address(regT1, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload)));
// Initialize 'this'.
emitGetVirtualRegister(thisValue, regT2);
store64(regT2, Address(regT1, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
// Copy arguments.
signExtend32ToPtr(regT0, regT0);
end.append(branchSub64(Zero, TrustedImm32(1), regT0));
// regT0: argumentCount
Label copyLoop = label();
load64(BaseIndex(callFrameRegister, regT0, TimesEight, (CallFrame::thisArgumentOffset() + firstVarArgOffset) * static_cast<int>(sizeof(Register))), regT2);
store64(regT2, BaseIndex(regT1, regT0, TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register))));
branchSub64(NonZero, TrustedImm32(1), regT0).linkTo(copyLoop, this);
end.append(jump());
}
if (canOptimize)
slowCase.link(this);
emitGetVirtualRegister(arguments, regT1);
callOperation(operationSizeFrameForVarargs, regT1, firstFreeRegister, firstVarArgOffset);
move(returnValueGPR, stackPointerRegister);
emitGetVirtualRegister(thisValue, regT1);
emitGetVirtualRegister(arguments, regT2);
callOperation(operationLoadVarargs, returnValueGPR, regT1, regT2, firstVarArgOffset);
move(returnValueGPR, regT1);
if (canOptimize)
end.link(this);
addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), regT1, stackPointerRegister);
}
MacroAssemblerCodeRef arityFixupGenerator(VM* vm)
{
JSInterfaceJIT jit(vm);
// We enter with fixup count in argumentGPR0
// We have the guarantee that a0, a1, a2, t3, t4 and t5 (or t0 for Windows) are all distinct :-)
#if USE(JSVALUE64)
#if OS(WINDOWS)
const GPRReg extraTemp = JSInterfaceJIT::regT0;
#else
const GPRReg extraTemp = JSInterfaceJIT::regT5;
#endif
# if CPU(X86_64)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
// Check to see if we have extra slots we can use
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
JSInterfaceJIT::Label fillExtraSlots(jit.label());
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight));
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
noExtraSlot.link(&jit);
jit.neg64(JSInterfaceJIT::argumentGPR0);
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load64(JSInterfaceJIT::regT3, extraTemp);
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
// Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::argumentGPR0, extraTemp);
jit.lshift64(JSInterfaceJIT::TrustedImm32(3), extraTemp);
jit.addPtr(extraTemp, JSInterfaceJIT::callFrameRegister);
jit.addPtr(extraTemp, JSInterfaceJIT::stackPointerRegister);
done.link(&jit);
# if CPU(X86_64)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#else
# if CPU(X86)
jit.pop(JSInterfaceJIT::regT4);
# endif
jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3);
jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2);
jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2);
// Check to see if we have extra slots we can use
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1);
jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1);
JSInterfaceJIT::Label fillExtraSlots(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, TagOffset));
jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2);
jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit);
jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0);
JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0);
noExtraSlot.link(&jit);
jit.neg32(JSInterfaceJIT::argumentGPR0);
// Move current frame down argumentGPR0 number of slots
JSInterfaceJIT::Label copyLoop(jit.label());
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, PayloadOffset), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, TagOffset), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit);
// Fill in argumentGPR0 missing arg slots with undefined
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2);
JSInterfaceJIT::Label fillUndefinedLoop(jit.label());
jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset));
jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5);
jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset));
jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3);
jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit);
// Adjust call frame register and stack pointer to account for missing args
jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::regT5);
jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT5);
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::callFrameRegister);
jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::stackPointerRegister);
done.link(&jit);
# if CPU(X86)
jit.push(JSInterfaceJIT::regT4);
# endif
jit.ret();
#endif
LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID);
return FINALIZE_CODE(patchBuffer, ("fixup arity"));
}
CodeLocationLabel* ArityCheckFailReturnThunks::returnPCsFor(
VM& vm, unsigned numExpectedArgumentsIncludingThis)
{
ASSERT(numExpectedArgumentsIncludingThis >= 1);
numExpectedArgumentsIncludingThis = WTF::roundUpToMultipleOf(
stackAlignmentRegisters(), numExpectedArgumentsIncludingThis);
{
ConcurrentJITLocker locker(m_lock);
if (numExpectedArgumentsIncludingThis < m_nextSize)
return m_returnPCArrays.last().get();
}
ASSERT(!isCompilationThread());
numExpectedArgumentsIncludingThis = std::max(numExpectedArgumentsIncludingThis, m_nextSize * 2);
AssemblyHelpers jit(&vm, 0);
Vector<AssemblyHelpers::Label> labels;
for (unsigned size = m_nextSize; size <= numExpectedArgumentsIncludingThis; size += stackAlignmentRegisters()) {
labels.append(jit.label());
jit.load32(
AssemblyHelpers::Address(
AssemblyHelpers::stackPointerRegister,
(JSStack::ArgumentCount - JSStack::CallerFrameAndPCSize) * sizeof(Register) +
PayloadOffset),
GPRInfo::regT4);
jit.add32(
AssemblyHelpers::TrustedImm32(
JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + size - 1),
GPRInfo::regT4, GPRInfo::regT2);
jit.lshift32(AssemblyHelpers::TrustedImm32(3), GPRInfo::regT2);
jit.addPtr(AssemblyHelpers::stackPointerRegister, GPRInfo::regT2);
jit.loadPtr(GPRInfo::regT2, GPRInfo::regT2);
jit.addPtr(
AssemblyHelpers::TrustedImm32(size * sizeof(Register)),
AssemblyHelpers::stackPointerRegister);
// Thunks like ours want to use the return PC to figure out where things
// were saved. So, we pay it forward.
jit.store32(
GPRInfo::regT4,
AssemblyHelpers::Address(
AssemblyHelpers::stackPointerRegister,
(JSStack::ArgumentCount - JSStack::CallerFrameAndPCSize) * sizeof(Register) +
PayloadOffset));
jit.jump(GPRInfo::regT2);
}
LinkBuffer linkBuffer(vm, &jit, GLOBAL_THUNK_ID);
unsigned returnPCsSize = numExpectedArgumentsIncludingThis / stackAlignmentRegisters() + 1;
std::unique_ptr<CodeLocationLabel[]> returnPCs =
std::make_unique<CodeLocationLabel[]>(returnPCsSize);
for (unsigned size = 0; size <= numExpectedArgumentsIncludingThis; size += stackAlignmentRegisters()) {
unsigned index = size / stackAlignmentRegisters();
RELEASE_ASSERT(index < returnPCsSize);
if (size < m_nextSize)
returnPCs[index] = m_returnPCArrays.last()[index];
else
returnPCs[index] = linkBuffer.locationOf(labels[(size - m_nextSize) / stackAlignmentRegisters()]);
}
CodeLocationLabel* result = returnPCs.get();
{
ConcurrentJITLocker locker(m_lock);
m_returnPCArrays.append(std::move(returnPCs));
m_refs.append(FINALIZE_CODE(linkBuffer, ("Arity check fail return thunks for up to numArgs = %u", numExpectedArgumentsIncludingThis)));
m_nextSize = numExpectedArgumentsIncludingThis + stackAlignmentRegisters();
}
return result;
}
CodeLocationLabel ArityCheckFailReturnThunks::returnPCFor(VM& vm, unsigned slotsToAdd)
{
return returnPCsFor(vm, slotsToAdd)[slotsToAdd / stackAlignmentRegisters()];
}
