void emitSetVarargsFrame(CCallHelpers& jit, GPRReg lengthGPR, bool lengthIncludesThis, GPRReg numUsedSlotsGPR, GPRReg resultGPR) { jit.move(numUsedSlotsGPR, resultGPR); // We really want to make sure the size of the new call frame is a multiple of // stackAlignmentRegisters(), however it is easier to accomplish this by // rounding numUsedSlotsGPR to the next multiple of stackAlignmentRegisters(). // Together with the rounding below, we will assure that the new call frame is // located on a stackAlignmentRegisters() boundary and a multiple of // stackAlignmentRegisters() in size. jit.addPtr(CCallHelpers::TrustedImm32(stackAlignmentRegisters() - 1), resultGPR); jit.andPtr(CCallHelpers::TrustedImm32(~(stackAlignmentRegisters() - 1)), resultGPR); jit.addPtr(lengthGPR, resultGPR); jit.addPtr(CCallHelpers::TrustedImm32(JSStack::CallFrameHeaderSize + (lengthIncludesThis? 0 : 1)), resultGPR); // resultGPR now has the required frame size in Register units // Round resultGPR to next multiple of stackAlignmentRegisters() jit.addPtr(CCallHelpers::TrustedImm32(stackAlignmentRegisters() - 1), resultGPR); jit.andPtr(CCallHelpers::TrustedImm32(~(stackAlignmentRegisters() - 1)), resultGPR); // Now resultGPR has the right stack frame offset in Register units. jit.negPtr(resultGPR); jit.lshiftPtr(CCallHelpers::Imm32(3), resultGPR); jit.addPtr(GPRInfo::callFrameRegister, resultGPR); }
void emitSetVarargsFrame(CCallHelpers& jit, GPRReg lengthGPR, bool lengthIncludesThis, GPRReg numUsedSlotsGPR, GPRReg resultGPR) { jit.move(numUsedSlotsGPR, resultGPR); jit.addPtr(lengthGPR, resultGPR); jit.addPtr(CCallHelpers::TrustedImm32(JSStack::CallFrameHeaderSize + (lengthIncludesThis? 0 : 1)), resultGPR); // resultGPR now has the required frame size in Register units // Round resultGPR to next multiple of stackAlignmentRegisters() jit.addPtr(CCallHelpers::TrustedImm32(stackAlignmentRegisters() - 1), resultGPR); jit.andPtr(CCallHelpers::TrustedImm32(~(stackAlignmentRegisters() - 1)), resultGPR); // Now resultGPR has the right stack frame offset in Register units. jit.negPtr(resultGPR); jit.lshiftPtr(CCallHelpers::Imm32(3), resultGPR); jit.addPtr(GPRInfo::callFrameRegister, resultGPR); }
ptrdiff_t registerPreservationOffset() { unsigned numberOfCalleeSaves = registersToPreserve().numberOfSetRegisters(); // Need to preserve the old return PC. unsigned numberOfValuesToSave = numberOfCalleeSaves + 1; // Alignment. Preserve the same alignment invariants that the caller imposed. unsigned numberOfNewStackSlots = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), numberOfValuesToSave); return sizeof(Register) * numberOfNewStackSlots; }
void JIT::compileLoadVarargs(Instruction* instruction) { int thisValue = instruction[3].u.operand; int arguments = instruction[4].u.operand; int firstFreeRegister = instruction[5].u.operand; int firstVarArgOffset = instruction[6].u.operand; JumpList slowCase; JumpList end; bool canOptimize = m_codeBlock->usesArguments() && arguments == m_codeBlock->argumentsRegister().offset() && !m_codeBlock->symbolTable()->slowArguments(); if (canOptimize) { emitGetVirtualRegister(arguments, regT0); slowCase.append(branch64(NotEqual, regT0, TrustedImm64(JSValue::encode(JSValue())))); emitGetFromCallFrameHeader32(JSStack::ArgumentCount, regT0); if (firstVarArgOffset) { Jump sufficientArguments = branch32(GreaterThan, regT0, TrustedImm32(firstVarArgOffset + 1)); move(TrustedImm32(1), regT0); Jump endVarArgs = jump(); sufficientArguments.link(this); sub32(TrustedImm32(firstVarArgOffset), regT0); endVarArgs.link(this); } slowCase.append(branch32(Above, regT0, TrustedImm32(Arguments::MaxArguments + 1))); // regT0: argumentCountIncludingThis move(regT0, regT1); add64(TrustedImm32(-firstFreeRegister + JSStack::CallFrameHeaderSize), regT1); // regT1 now has the required frame size in Register units // Round regT1 to next multiple of stackAlignmentRegisters() add64(TrustedImm32(stackAlignmentRegisters() - 1), regT1); and64(TrustedImm32(~(stackAlignmentRegisters() - 1)), regT1); neg64(regT1); lshift64(TrustedImm32(3), regT1); addPtr(callFrameRegister, regT1); // regT1: newCallFrame slowCase.append(branchPtr(Above, AbsoluteAddress(m_vm->addressOfStackLimit()), regT1)); // Initialize ArgumentCount. store32(regT0, Address(regT1, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.payload))); // Initialize 'this'. emitGetVirtualRegister(thisValue, regT2); store64(regT2, Address(regT1, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))); // Copy arguments. signExtend32ToPtr(regT0, regT0); end.append(branchSub64(Zero, TrustedImm32(1), regT0)); // regT0: argumentCount Label copyLoop = label(); load64(BaseIndex(callFrameRegister, regT0, TimesEight, (CallFrame::thisArgumentOffset() + firstVarArgOffset) * static_cast<int>(sizeof(Register))), regT2); store64(regT2, BaseIndex(regT1, regT0, TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))); branchSub64(NonZero, TrustedImm32(1), regT0).linkTo(copyLoop, this); end.append(jump()); } if (canOptimize) slowCase.link(this); emitGetVirtualRegister(arguments, regT1); callOperation(operationSizeFrameForVarargs, regT1, firstFreeRegister, firstVarArgOffset); move(returnValueGPR, stackPointerRegister); emitGetVirtualRegister(thisValue, regT1); emitGetVirtualRegister(arguments, regT2); callOperation(operationLoadVarargs, returnValueGPR, regT1, regT2, firstVarArgOffset); move(returnValueGPR, regT1); if (canOptimize) end.link(this); addPtr(TrustedImm32(sizeof(CallerFrameAndPC)), regT1, stackPointerRegister); }
MacroAssemblerCodeRef arityFixupGenerator(VM* vm) { JSInterfaceJIT jit(vm); // We enter with fixup count in argumentGPR0 // We have the guarantee that a0, a1, a2, t3, t4 and t5 (or t0 for Windows) are all distinct :-) #if USE(JSVALUE64) #if OS(WINDOWS) const GPRReg extraTemp = JSInterfaceJIT::regT0; #else const GPRReg extraTemp = JSInterfaceJIT::regT5; #endif # if CPU(X86_64) jit.pop(JSInterfaceJIT::regT4); # endif jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3); jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2); jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2); // Check to see if we have extra slots we can use jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1); jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1); JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1); jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp); JSInterfaceJIT::Label fillExtraSlots(jit.label()); jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight)); jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2); jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit); jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0); JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0); noExtraSlot.link(&jit); jit.neg64(JSInterfaceJIT::argumentGPR0); // Move current frame down argumentGPR0 number of slots JSInterfaceJIT::Label copyLoop(jit.label()); jit.load64(JSInterfaceJIT::regT3, extraTemp); jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3); jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit); // Fill in argumentGPR0 missing arg slots with undefined jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2); jit.move(JSInterfaceJIT::TrustedImm64(ValueUndefined), extraTemp); JSInterfaceJIT::Label fillUndefinedLoop(jit.label()); jit.store64(extraTemp, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3); jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit); // Adjust call frame register and stack pointer to account for missing args jit.move(JSInterfaceJIT::argumentGPR0, extraTemp); jit.lshift64(JSInterfaceJIT::TrustedImm32(3), extraTemp); jit.addPtr(extraTemp, JSInterfaceJIT::callFrameRegister); jit.addPtr(extraTemp, JSInterfaceJIT::stackPointerRegister); done.link(&jit); # if CPU(X86_64) jit.push(JSInterfaceJIT::regT4); # endif jit.ret(); #else # if CPU(X86) jit.pop(JSInterfaceJIT::regT4); # endif jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT3); jit.load32(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister, JSStack::ArgumentCount * sizeof(Register)), JSInterfaceJIT::argumentGPR2); jit.add32(JSInterfaceJIT::TrustedImm32(JSStack::CallFrameHeaderSize), JSInterfaceJIT::argumentGPR2); // Check to see if we have extra slots we can use jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR1); jit.and32(JSInterfaceJIT::TrustedImm32(stackAlignmentRegisters() - 1), JSInterfaceJIT::argumentGPR1); JSInterfaceJIT::Jump noExtraSlot = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR1); JSInterfaceJIT::Label fillExtraSlots(jit.label()); jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5); jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, PayloadOffset)); jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5); jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR2, JSInterfaceJIT::TimesEight, TagOffset)); jit.add32(JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2); jit.branchSub32(JSInterfaceJIT::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR1).linkTo(fillExtraSlots, &jit); jit.and32(JSInterfaceJIT::TrustedImm32(-stackAlignmentRegisters()), JSInterfaceJIT::argumentGPR0); JSInterfaceJIT::Jump done = jit.branchTest32(MacroAssembler::Zero, JSInterfaceJIT::argumentGPR0); noExtraSlot.link(&jit); jit.neg32(JSInterfaceJIT::argumentGPR0); // Move current frame down argumentGPR0 number of slots JSInterfaceJIT::Label copyLoop(jit.label()); jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, PayloadOffset), JSInterfaceJIT::regT5); jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset)); jit.load32(MacroAssembler::Address(JSInterfaceJIT::regT3, TagOffset), JSInterfaceJIT::regT5); jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3); jit.branchSub32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(copyLoop, &jit); // Fill in argumentGPR0 missing arg slots with undefined jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::argumentGPR2); JSInterfaceJIT::Label fillUndefinedLoop(jit.label()); jit.move(JSInterfaceJIT::TrustedImm32(0), JSInterfaceJIT::regT5); jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, PayloadOffset)); jit.move(JSInterfaceJIT::TrustedImm32(JSValue::UndefinedTag), JSInterfaceJIT::regT5); jit.store32(JSInterfaceJIT::regT5, MacroAssembler::BaseIndex(JSInterfaceJIT::regT3, JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::TimesEight, TagOffset)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::regT3); jit.branchAdd32(MacroAssembler::NonZero, JSInterfaceJIT::TrustedImm32(1), JSInterfaceJIT::argumentGPR2).linkTo(fillUndefinedLoop, &jit); // Adjust call frame register and stack pointer to account for missing args jit.move(JSInterfaceJIT::argumentGPR0, JSInterfaceJIT::regT5); jit.lshift32(JSInterfaceJIT::TrustedImm32(3), JSInterfaceJIT::regT5); jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::callFrameRegister); jit.addPtr(JSInterfaceJIT::regT5, JSInterfaceJIT::stackPointerRegister); done.link(&jit); # if CPU(X86) jit.push(JSInterfaceJIT::regT4); # endif jit.ret(); #endif LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID); return FINALIZE_CODE(patchBuffer, ("fixup arity")); }
CodeLocationLabel* ArityCheckFailReturnThunks::returnPCsFor( VM& vm, unsigned numExpectedArgumentsIncludingThis) { ASSERT(numExpectedArgumentsIncludingThis >= 1); numExpectedArgumentsIncludingThis = WTF::roundUpToMultipleOf( stackAlignmentRegisters(), numExpectedArgumentsIncludingThis); { ConcurrentJITLocker locker(m_lock); if (numExpectedArgumentsIncludingThis < m_nextSize) return m_returnPCArrays.last().get(); } ASSERT(!isCompilationThread()); numExpectedArgumentsIncludingThis = std::max(numExpectedArgumentsIncludingThis, m_nextSize * 2); AssemblyHelpers jit(&vm, 0); Vector<AssemblyHelpers::Label> labels; for (unsigned size = m_nextSize; size <= numExpectedArgumentsIncludingThis; size += stackAlignmentRegisters()) { labels.append(jit.label()); jit.load32( AssemblyHelpers::Address( AssemblyHelpers::stackPointerRegister, (JSStack::ArgumentCount - JSStack::CallerFrameAndPCSize) * sizeof(Register) + PayloadOffset), GPRInfo::regT4); jit.add32( AssemblyHelpers::TrustedImm32( JSStack::CallFrameHeaderSize - JSStack::CallerFrameAndPCSize + size - 1), GPRInfo::regT4, GPRInfo::regT2); jit.lshift32(AssemblyHelpers::TrustedImm32(3), GPRInfo::regT2); jit.addPtr(AssemblyHelpers::stackPointerRegister, GPRInfo::regT2); jit.loadPtr(GPRInfo::regT2, GPRInfo::regT2); jit.addPtr( AssemblyHelpers::TrustedImm32(size * sizeof(Register)), AssemblyHelpers::stackPointerRegister); // Thunks like ours want to use the return PC to figure out where things // were saved. So, we pay it forward. jit.store32( GPRInfo::regT4, AssemblyHelpers::Address( AssemblyHelpers::stackPointerRegister, (JSStack::ArgumentCount - JSStack::CallerFrameAndPCSize) * sizeof(Register) + PayloadOffset)); jit.jump(GPRInfo::regT2); } LinkBuffer linkBuffer(vm, &jit, GLOBAL_THUNK_ID); unsigned returnPCsSize = numExpectedArgumentsIncludingThis / stackAlignmentRegisters() + 1; std::unique_ptr<CodeLocationLabel[]> returnPCs = std::make_unique<CodeLocationLabel[]>(returnPCsSize); for (unsigned size = 0; size <= numExpectedArgumentsIncludingThis; size += stackAlignmentRegisters()) { unsigned index = size / stackAlignmentRegisters(); RELEASE_ASSERT(index < returnPCsSize); if (size < m_nextSize) returnPCs[index] = m_returnPCArrays.last()[index]; else returnPCs[index] = linkBuffer.locationOf(labels[(size - m_nextSize) / stackAlignmentRegisters()]); } CodeLocationLabel* result = returnPCs.get(); { ConcurrentJITLocker locker(m_lock); m_returnPCArrays.append(std::move(returnPCs)); m_refs.append(FINALIZE_CODE(linkBuffer, ("Arity check fail return thunks for up to numArgs = %u", numExpectedArgumentsIncludingThis))); m_nextSize = numExpectedArgumentsIncludingThis + stackAlignmentRegisters(); } return result; }
CodeLocationLabel ArityCheckFailReturnThunks::returnPCFor(VM& vm, unsigned slotsToAdd) { return returnPCsFor(vm, slotsToAdd)[slotsToAdd / stackAlignmentRegisters()]; }