// Generate a stub that is called via the internal ABI derived from the
// signature of the import and calls into a compatible JIT function,
// having boxed all the ABI arguments into the JIT stack frame layout.
static bool
GenerateJitExitStub(ModuleGenerator& mg, unsigned importIndex, bool usesHeap,
Label* throwLabel, ProfilingOffsets* offsets)
{
MacroAssembler& masm = mg.masm();
const Sig& sig = *mg.import(importIndex).sig;
masm.setFramePushed(0);
// JIT calls use the following stack layout (sp grows to the left):
// | retaddr | descriptor | callee | argc | this | arg1..N |
// After the JIT frame, the global register (if present) is saved since the
// JIT's ABI does not preserve non-volatile regs. Also, unlike most ABIs,
// the JIT ABI requires that sp be JitStackAlignment-aligned *after* pushing
// the return address.
static_assert(AsmJSStackAlignment >= JitStackAlignment, "subsumes");
unsigned sizeOfRetAddr = sizeof(void*);
unsigned jitFrameBytes = 3 * sizeof(void*) + (1 + sig.args().length()) * sizeof(Value);
unsigned totalJitFrameBytes = sizeOfRetAddr + jitFrameBytes + MaybeSavedGlobalReg;
unsigned jitFramePushed = StackDecrementForCall(masm, JitStackAlignment, totalJitFrameBytes) -
sizeOfRetAddr;
GenerateExitPrologue(masm, jitFramePushed, ExitReason::ImportJit, offsets);
// 1. Descriptor
size_t argOffset = 0;
uint32_t descriptor = MakeFrameDescriptor(jitFramePushed, JitFrame_Entry);
masm.storePtr(ImmWord(uintptr_t(descriptor)), Address(masm.getStackPointer(), argOffset));
argOffset += sizeof(size_t);
// 2. Callee
Register callee = ABIArgGenerator::NonArgReturnReg0; // live until call
Register scratch = ABIArgGenerator::NonArgReturnReg1; // repeatedly clobbered
// 2.1. Get ExitDatum
unsigned globalDataOffset = mg.import(importIndex).globalDataOffset;
#if defined(JS_CODEGEN_X64)
masm.append(AsmJSGlobalAccess(masm.leaRipRelative(callee), globalDataOffset));
#elif defined(JS_CODEGEN_X86)
masm.append(AsmJSGlobalAccess(masm.movlWithPatch(Imm32(0), callee), globalDataOffset));
#elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \
defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
masm.computeEffectiveAddress(Address(GlobalReg, globalDataOffset - AsmJSGlobalRegBias), callee);
#endif
// 2.2. Get callee
masm.loadPtr(Address(callee, Module::OffsetOfImportExitFun), callee);
// 2.3. Save callee
masm.storePtr(callee, Address(masm.getStackPointer(), argOffset));
argOffset += sizeof(size_t);
// 2.4. Load callee executable entry point
masm.loadPtr(Address(callee, JSFunction::offsetOfNativeOrScript()), callee);
masm.loadBaselineOrIonNoArgCheck(callee, callee, nullptr);
// 3. Argc
unsigned argc = sig.args().length();
masm.storePtr(ImmWord(uintptr_t(argc)), Address(masm.getStackPointer(), argOffset));
argOffset += sizeof(size_t);
// 4. |this| value
masm.storeValue(UndefinedValue(), Address(masm.getStackPointer(), argOffset));
argOffset += sizeof(Value);
// 5. Fill the arguments
unsigned offsetToCallerStackArgs = jitFramePushed + sizeof(AsmJSFrame);
FillArgumentArray(masm, sig.args(), argOffset, offsetToCallerStackArgs, scratch);
argOffset += sig.args().length() * sizeof(Value);
MOZ_ASSERT(argOffset == jitFrameBytes);
// 6. Jit code will clobber all registers, even non-volatiles. GlobalReg and
// HeapReg are removed from the general register set for asm.js code, so
// these will not have been saved by the caller like all other registers,
// so they must be explicitly preserved. Only save GlobalReg since
// HeapReg can be reloaded (from global data) after the call.
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
static_assert(MaybeSavedGlobalReg == sizeof(void*), "stack frame accounting");
masm.storePtr(GlobalReg, Address(masm.getStackPointer(), jitFrameBytes));
#endif
{
// Enable Activation.
//
// This sequence requires four registers, and needs to preserve the 'callee'
// register, so there are five live registers.
MOZ_ASSERT(callee == AsmJSIonExitRegCallee);
Register reg0 = AsmJSIonExitRegE0;
Register reg1 = AsmJSIonExitRegE1;
Register reg2 = AsmJSIonExitRegE2;
Register reg3 = AsmJSIonExitRegE3;
// The following is inlined:
// JSContext* cx = activation->cx();
// Activation* act = cx->runtime()->activation();
// act.active_ = true;
// act.prevJitTop_ = cx->runtime()->jitTop;
// act.prevJitJSContext_ = cx->runtime()->jitJSContext;
// cx->runtime()->jitJSContext = cx;
// act.prevJitActivation_ = cx->runtime()->jitActivation;
// cx->runtime()->jitActivation = act;
// act.prevProfilingActivation_ = cx->runtime()->profilingActivation;
// cx->runtime()->profilingActivation_ = act;
// On the ARM store8() uses the secondScratchReg (lr) as a temp.
size_t offsetOfActivation = JSRuntime::offsetOfActivation();
size_t offsetOfJitTop = offsetof(JSRuntime, jitTop);
size_t offsetOfJitJSContext = offsetof(JSRuntime, jitJSContext);
size_t offsetOfJitActivation = offsetof(JSRuntime, jitActivation);
size_t offsetOfProfilingActivation = JSRuntime::offsetOfProfilingActivation();
masm.loadWasmActivation(reg0);
masm.loadPtr(Address(reg0, WasmActivation::offsetOfContext()), reg3);
masm.loadPtr(Address(reg3, JSContext::offsetOfRuntime()), reg0);
masm.loadPtr(Address(reg0, offsetOfActivation), reg1);
// act.active_ = true;
masm.store8(Imm32(1), Address(reg1, JitActivation::offsetOfActiveUint8()));
// act.prevJitTop_ = cx->runtime()->jitTop;
masm.loadPtr(Address(reg0, offsetOfJitTop), reg2);
masm.storePtr(reg2, Address(reg1, JitActivation::offsetOfPrevJitTop()));
// act.prevJitJSContext_ = cx->runtime()->jitJSContext;
masm.loadPtr(Address(reg0, offsetOfJitJSContext), reg2);
masm.storePtr(reg2, Address(reg1, JitActivation::offsetOfPrevJitJSContext()));
// cx->runtime()->jitJSContext = cx;
masm.storePtr(reg3, Address(reg0, offsetOfJitJSContext));
// act.prevJitActivation_ = cx->runtime()->jitActivation;
masm.loadPtr(Address(reg0, offsetOfJitActivation), reg2);
masm.storePtr(reg2, Address(reg1, JitActivation::offsetOfPrevJitActivation()));
// cx->runtime()->jitActivation = act;
masm.storePtr(reg1, Address(reg0, offsetOfJitActivation));
// act.prevProfilingActivation_ = cx->runtime()->profilingActivation;
masm.loadPtr(Address(reg0, offsetOfProfilingActivation), reg2);
masm.storePtr(reg2, Address(reg1, Activation::offsetOfPrevProfiling()));
// cx->runtime()->profilingActivation_ = act;
masm.storePtr(reg1, Address(reg0, offsetOfProfilingActivation));
}
AssertStackAlignment(masm, JitStackAlignment, sizeOfRetAddr);
masm.callJitNoProfiler(callee);
AssertStackAlignment(masm, JitStackAlignment, sizeOfRetAddr);
{
// Disable Activation.
//
// This sequence needs three registers, and must preserve the JSReturnReg_Data and
// JSReturnReg_Type, so there are five live registers.
MOZ_ASSERT(JSReturnReg_Data == AsmJSIonExitRegReturnData);
MOZ_ASSERT(JSReturnReg_Type == AsmJSIonExitRegReturnType);
Register reg0 = AsmJSIonExitRegD0;
Register reg1 = AsmJSIonExitRegD1;
Register reg2 = AsmJSIonExitRegD2;
// The following is inlined:
// rt->profilingActivation = prevProfilingActivation_;
// rt->activation()->active_ = false;
// rt->jitTop = prevJitTop_;
// rt->jitJSContext = prevJitJSContext_;
// rt->jitActivation = prevJitActivation_;
// On the ARM store8() uses the secondScratchReg (lr) as a temp.
size_t offsetOfActivation = JSRuntime::offsetOfActivation();
size_t offsetOfJitTop = offsetof(JSRuntime, jitTop);
size_t offsetOfJitJSContext = offsetof(JSRuntime, jitJSContext);
size_t offsetOfJitActivation = offsetof(JSRuntime, jitActivation);
size_t offsetOfProfilingActivation = JSRuntime::offsetOfProfilingActivation();
masm.movePtr(SymbolicAddress::Runtime, reg0);
masm.loadPtr(Address(reg0, offsetOfActivation), reg1);
// rt->jitTop = prevJitTop_;
masm.loadPtr(Address(reg1, JitActivation::offsetOfPrevJitTop()), reg2);
masm.storePtr(reg2, Address(reg0, offsetOfJitTop));
// rt->profilingActivation = rt->activation()->prevProfiling_;
masm.loadPtr(Address(reg1, Activation::offsetOfPrevProfiling()), reg2);
masm.storePtr(reg2, Address(reg0, offsetOfProfilingActivation));
// rt->activation()->active_ = false;
masm.store8(Imm32(0), Address(reg1, JitActivation::offsetOfActiveUint8()));
// rt->jitJSContext = prevJitJSContext_;
masm.loadPtr(Address(reg1, JitActivation::offsetOfPrevJitJSContext()), reg2);
masm.storePtr(reg2, Address(reg0, offsetOfJitJSContext));
// rt->jitActivation = prevJitActivation_;
masm.loadPtr(Address(reg1, JitActivation::offsetOfPrevJitActivation()), reg2);
masm.storePtr(reg2, Address(reg0, offsetOfJitActivation));
}
// Reload the global register since JIT code can clobber any register.
#if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64)
static_assert(MaybeSavedGlobalReg == sizeof(void*), "stack frame accounting");
masm.loadPtr(Address(masm.getStackPointer(), jitFrameBytes), GlobalReg);
#endif
// As explained above, the frame was aligned for the JIT ABI such that
// (sp + sizeof(void*)) % JitStackAlignment == 0
// But now we possibly want to call one of several different C++ functions,
// so subtract the sizeof(void*) so that sp is aligned for an ABI call.
static_assert(ABIStackAlignment <= JitStackAlignment, "subsumes");
masm.reserveStack(sizeOfRetAddr);
unsigned nativeFramePushed = masm.framePushed();
AssertStackAlignment(masm, ABIStackAlignment);
masm.branchTestMagic(Assembler::Equal, JSReturnOperand, throwLabel);
Label oolConvert;
switch (sig.ret()) {
case ExprType::Void:
break;
case ExprType::I32:
masm.convertValueToInt32(JSReturnOperand, ReturnDoubleReg, ReturnReg, &oolConvert,
/* -0 check */ false);
break;
case ExprType::I64:
MOZ_CRASH("no int64 in asm.js");
case ExprType::F32:
MOZ_CRASH("Float shouldn't be returned from an import");
case ExprType::F64:
masm.convertValueToDouble(JSReturnOperand, ReturnDoubleReg, &oolConvert);
break;
case ExprType::I32x4:
case ExprType::F32x4:
case ExprType::B32x4:
MOZ_CRASH("SIMD types shouldn't be returned from an import");
}
Label done;
masm.bind(&done);
// Ion code does not respect system callee-saved register conventions so
// reload the heap register.
if (usesHeap)
masm.loadAsmJSHeapRegisterFromGlobalData();
GenerateExitEpilogue(masm, masm.framePushed(), ExitReason::ImportJit, offsets);
if (oolConvert.used()) {
masm.bind(&oolConvert);
masm.setFramePushed(nativeFramePushed);
// Coercion calls use the following stack layout (sp grows to the left):
// | args | padding | Value argv[1] | padding | exit AsmJSFrame |
MIRTypeVector coerceArgTypes;
JS_ALWAYS_TRUE(coerceArgTypes.append(MIRType_Pointer));
unsigned offsetToCoerceArgv = AlignBytes(StackArgBytes(coerceArgTypes), sizeof(Value));
MOZ_ASSERT(nativeFramePushed >= offsetToCoerceArgv + sizeof(Value));
AssertStackAlignment(masm, ABIStackAlignment);
// Store return value into argv[0]
masm.storeValue(JSReturnOperand, Address(masm.getStackPointer(), offsetToCoerceArgv));
// argument 0: argv
ABIArgMIRTypeIter i(coerceArgTypes);
Address argv(masm.getStackPointer(), offsetToCoerceArgv);
if (i->kind() == ABIArg::GPR) {
masm.computeEffectiveAddress(argv, i->gpr());
} else {
masm.computeEffectiveAddress(argv, scratch);
masm.storePtr(scratch, Address(masm.getStackPointer(), i->offsetFromArgBase()));
}
i++;
MOZ_ASSERT(i.done());
// Call coercion function
AssertStackAlignment(masm, ABIStackAlignment);
switch (sig.ret()) {
case ExprType::I32:
masm.call(SymbolicAddress::CoerceInPlace_ToInt32);
masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
masm.unboxInt32(Address(masm.getStackPointer(), offsetToCoerceArgv), ReturnReg);
break;
case ExprType::F64:
masm.call(SymbolicAddress::CoerceInPlace_ToNumber);
masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
masm.loadDouble(Address(masm.getStackPointer(), offsetToCoerceArgv), ReturnDoubleReg);
break;
default:
MOZ_CRASH("Unsupported convert type");
}
masm.jump(&done);
masm.setFramePushed(0);
}
MOZ_ASSERT(masm.framePushed() == 0);
if (masm.oom())
return false;
offsets->end = masm.currentOffset();
return true;
}
// Generate a stub that is called via the internal ABI derived from the
// signature of the import and calls into an appropriate InvokeImport C++
// function, having boxed all the ABI arguments into a homogeneous Value array.
static bool
GenerateInterpExitStub(ModuleGenerator& mg, unsigned importIndex, Label* throwLabel,
ProfilingOffsets* offsets)
{
MacroAssembler& masm = mg.masm();
const Sig& sig = *mg.import(importIndex).sig;
masm.setFramePushed(0);
// Argument types for InvokeImport_*:
static const MIRType typeArray[] = { MIRType_Pointer, // ImportExit
MIRType_Int32, // argc
MIRType_Pointer }; // argv
MIRTypeVector invokeArgTypes;
MOZ_ALWAYS_TRUE(invokeArgTypes.append(typeArray, ArrayLength(typeArray)));
// At the point of the call, the stack layout shall be (sp grows to the left):
// | stack args | padding | Value argv[] | padding | retaddr | caller stack args |
// The padding between stack args and argv ensures that argv is aligned. The
// padding between argv and retaddr ensures that sp is aligned.
unsigned argOffset = AlignBytes(StackArgBytes(invokeArgTypes), sizeof(double));
unsigned argBytes = Max<size_t>(1, sig.args().length()) * sizeof(Value);
unsigned framePushed = StackDecrementForCall(masm, ABIStackAlignment, argOffset + argBytes);
GenerateExitPrologue(masm, framePushed, ExitReason::ImportInterp, offsets);
// Fill the argument array.
unsigned offsetToCallerStackArgs = sizeof(AsmJSFrame) + masm.framePushed();
Register scratch = ABIArgGenerator::NonArgReturnReg0;
FillArgumentArray(masm, sig.args(), argOffset, offsetToCallerStackArgs, scratch);
// Prepare the arguments for the call to InvokeImport_*.
ABIArgMIRTypeIter i(invokeArgTypes);
// argument 0: importIndex
if (i->kind() == ABIArg::GPR)
masm.mov(ImmWord(importIndex), i->gpr());
else
masm.store32(Imm32(importIndex), Address(masm.getStackPointer(), i->offsetFromArgBase()));
i++;
// argument 1: argc
unsigned argc = sig.args().length();
if (i->kind() == ABIArg::GPR)
masm.mov(ImmWord(argc), i->gpr());
else
masm.store32(Imm32(argc), Address(masm.getStackPointer(), i->offsetFromArgBase()));
i++;
// argument 2: argv
Address argv(masm.getStackPointer(), argOffset);
if (i->kind() == ABIArg::GPR) {
masm.computeEffectiveAddress(argv, i->gpr());
} else {
masm.computeEffectiveAddress(argv, scratch);
masm.storePtr(scratch, Address(masm.getStackPointer(), i->offsetFromArgBase()));
}
i++;
MOZ_ASSERT(i.done());
// Make the call, test whether it succeeded, and extract the return value.
AssertStackAlignment(masm, ABIStackAlignment);
switch (sig.ret()) {
case ExprType::Void:
masm.call(SymbolicAddress::InvokeImport_Void);
masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
break;
case ExprType::I32:
masm.call(SymbolicAddress::InvokeImport_I32);
masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
masm.unboxInt32(argv, ReturnReg);
break;
case ExprType::I64:
MOZ_CRASH("no int64 in asm.js");
case ExprType::F32:
MOZ_CRASH("Float32 shouldn't be returned from a FFI");
case ExprType::F64:
masm.call(SymbolicAddress::InvokeImport_F64);
masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel);
masm.loadDouble(argv, ReturnDoubleReg);
break;
case ExprType::I32x4:
case ExprType::F32x4:
case ExprType::B32x4:
MOZ_CRASH("SIMD types shouldn't be returned from a FFI");
}
GenerateExitEpilogue(masm, framePushed, ExitReason::ImportInterp, offsets);
if (masm.oom())
return false;
offsets->end = masm.currentOffset();
return true;
}
