// Generate a stub that is called via the internal ABI derived from the // signature of the import and calls into a compatible JIT function, // having boxed all the ABI arguments into the JIT stack frame layout. static bool GenerateJitExitStub(ModuleGenerator& mg, unsigned importIndex, bool usesHeap, Label* throwLabel, ProfilingOffsets* offsets) { MacroAssembler& masm = mg.masm(); const Sig& sig = *mg.import(importIndex).sig; masm.setFramePushed(0); // JIT calls use the following stack layout (sp grows to the left): // | retaddr | descriptor | callee | argc | this | arg1..N | // After the JIT frame, the global register (if present) is saved since the // JIT's ABI does not preserve non-volatile regs. Also, unlike most ABIs, // the JIT ABI requires that sp be JitStackAlignment-aligned *after* pushing // the return address. static_assert(AsmJSStackAlignment >= JitStackAlignment, "subsumes"); unsigned sizeOfRetAddr = sizeof(void*); unsigned jitFrameBytes = 3 * sizeof(void*) + (1 + sig.args().length()) * sizeof(Value); unsigned totalJitFrameBytes = sizeOfRetAddr + jitFrameBytes + MaybeSavedGlobalReg; unsigned jitFramePushed = StackDecrementForCall(masm, JitStackAlignment, totalJitFrameBytes) - sizeOfRetAddr; GenerateExitPrologue(masm, jitFramePushed, ExitReason::ImportJit, offsets); // 1. Descriptor size_t argOffset = 0; uint32_t descriptor = MakeFrameDescriptor(jitFramePushed, JitFrame_Entry); masm.storePtr(ImmWord(uintptr_t(descriptor)), Address(masm.getStackPointer(), argOffset)); argOffset += sizeof(size_t); // 2. Callee Register callee = ABIArgGenerator::NonArgReturnReg0; // live until call Register scratch = ABIArgGenerator::NonArgReturnReg1; // repeatedly clobbered // 2.1. Get ExitDatum unsigned globalDataOffset = mg.import(importIndex).globalDataOffset; #if defined(JS_CODEGEN_X64) masm.append(AsmJSGlobalAccess(masm.leaRipRelative(callee), globalDataOffset)); #elif defined(JS_CODEGEN_X86) masm.append(AsmJSGlobalAccess(masm.movlWithPatch(Imm32(0), callee), globalDataOffset)); #elif defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_ARM64) || \ defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) masm.computeEffectiveAddress(Address(GlobalReg, globalDataOffset - AsmJSGlobalRegBias), callee); #endif // 2.2. Get callee masm.loadPtr(Address(callee, Module::OffsetOfImportExitFun), callee); // 2.3. Save callee masm.storePtr(callee, Address(masm.getStackPointer(), argOffset)); argOffset += sizeof(size_t); // 2.4. Load callee executable entry point masm.loadPtr(Address(callee, JSFunction::offsetOfNativeOrScript()), callee); masm.loadBaselineOrIonNoArgCheck(callee, callee, nullptr); // 3. Argc unsigned argc = sig.args().length(); masm.storePtr(ImmWord(uintptr_t(argc)), Address(masm.getStackPointer(), argOffset)); argOffset += sizeof(size_t); // 4. |this| value masm.storeValue(UndefinedValue(), Address(masm.getStackPointer(), argOffset)); argOffset += sizeof(Value); // 5. Fill the arguments unsigned offsetToCallerStackArgs = jitFramePushed + sizeof(AsmJSFrame); FillArgumentArray(masm, sig.args(), argOffset, offsetToCallerStackArgs, scratch); argOffset += sig.args().length() * sizeof(Value); MOZ_ASSERT(argOffset == jitFrameBytes); // 6. Jit code will clobber all registers, even non-volatiles. GlobalReg and // HeapReg are removed from the general register set for asm.js code, so // these will not have been saved by the caller like all other registers, // so they must be explicitly preserved. Only save GlobalReg since // HeapReg can be reloaded (from global data) after the call. #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) static_assert(MaybeSavedGlobalReg == sizeof(void*), "stack frame accounting"); masm.storePtr(GlobalReg, Address(masm.getStackPointer(), jitFrameBytes)); #endif { // Enable Activation. // // This sequence requires four registers, and needs to preserve the 'callee' // register, so there are five live registers. MOZ_ASSERT(callee == AsmJSIonExitRegCallee); Register reg0 = AsmJSIonExitRegE0; Register reg1 = AsmJSIonExitRegE1; Register reg2 = AsmJSIonExitRegE2; Register reg3 = AsmJSIonExitRegE3; // The following is inlined: // JSContext* cx = activation->cx(); // Activation* act = cx->runtime()->activation(); // act.active_ = true; // act.prevJitTop_ = cx->runtime()->jitTop; // act.prevJitJSContext_ = cx->runtime()->jitJSContext; // cx->runtime()->jitJSContext = cx; // act.prevJitActivation_ = cx->runtime()->jitActivation; // cx->runtime()->jitActivation = act; // act.prevProfilingActivation_ = cx->runtime()->profilingActivation; // cx->runtime()->profilingActivation_ = act; // On the ARM store8() uses the secondScratchReg (lr) as a temp. size_t offsetOfActivation = JSRuntime::offsetOfActivation(); size_t offsetOfJitTop = offsetof(JSRuntime, jitTop); size_t offsetOfJitJSContext = offsetof(JSRuntime, jitJSContext); size_t offsetOfJitActivation = offsetof(JSRuntime, jitActivation); size_t offsetOfProfilingActivation = JSRuntime::offsetOfProfilingActivation(); masm.loadWasmActivation(reg0); masm.loadPtr(Address(reg0, WasmActivation::offsetOfContext()), reg3); masm.loadPtr(Address(reg3, JSContext::offsetOfRuntime()), reg0); masm.loadPtr(Address(reg0, offsetOfActivation), reg1); // act.active_ = true; masm.store8(Imm32(1), Address(reg1, JitActivation::offsetOfActiveUint8())); // act.prevJitTop_ = cx->runtime()->jitTop; masm.loadPtr(Address(reg0, offsetOfJitTop), reg2); masm.storePtr(reg2, Address(reg1, JitActivation::offsetOfPrevJitTop())); // act.prevJitJSContext_ = cx->runtime()->jitJSContext; masm.loadPtr(Address(reg0, offsetOfJitJSContext), reg2); masm.storePtr(reg2, Address(reg1, JitActivation::offsetOfPrevJitJSContext())); // cx->runtime()->jitJSContext = cx; masm.storePtr(reg3, Address(reg0, offsetOfJitJSContext)); // act.prevJitActivation_ = cx->runtime()->jitActivation; masm.loadPtr(Address(reg0, offsetOfJitActivation), reg2); masm.storePtr(reg2, Address(reg1, JitActivation::offsetOfPrevJitActivation())); // cx->runtime()->jitActivation = act; masm.storePtr(reg1, Address(reg0, offsetOfJitActivation)); // act.prevProfilingActivation_ = cx->runtime()->profilingActivation; masm.loadPtr(Address(reg0, offsetOfProfilingActivation), reg2); masm.storePtr(reg2, Address(reg1, Activation::offsetOfPrevProfiling())); // cx->runtime()->profilingActivation_ = act; masm.storePtr(reg1, Address(reg0, offsetOfProfilingActivation)); } AssertStackAlignment(masm, JitStackAlignment, sizeOfRetAddr); masm.callJitNoProfiler(callee); AssertStackAlignment(masm, JitStackAlignment, sizeOfRetAddr); { // Disable Activation. // // This sequence needs three registers, and must preserve the JSReturnReg_Data and // JSReturnReg_Type, so there are five live registers. MOZ_ASSERT(JSReturnReg_Data == AsmJSIonExitRegReturnData); MOZ_ASSERT(JSReturnReg_Type == AsmJSIonExitRegReturnType); Register reg0 = AsmJSIonExitRegD0; Register reg1 = AsmJSIonExitRegD1; Register reg2 = AsmJSIonExitRegD2; // The following is inlined: // rt->profilingActivation = prevProfilingActivation_; // rt->activation()->active_ = false; // rt->jitTop = prevJitTop_; // rt->jitJSContext = prevJitJSContext_; // rt->jitActivation = prevJitActivation_; // On the ARM store8() uses the secondScratchReg (lr) as a temp. size_t offsetOfActivation = JSRuntime::offsetOfActivation(); size_t offsetOfJitTop = offsetof(JSRuntime, jitTop); size_t offsetOfJitJSContext = offsetof(JSRuntime, jitJSContext); size_t offsetOfJitActivation = offsetof(JSRuntime, jitActivation); size_t offsetOfProfilingActivation = JSRuntime::offsetOfProfilingActivation(); masm.movePtr(SymbolicAddress::Runtime, reg0); masm.loadPtr(Address(reg0, offsetOfActivation), reg1); // rt->jitTop = prevJitTop_; masm.loadPtr(Address(reg1, JitActivation::offsetOfPrevJitTop()), reg2); masm.storePtr(reg2, Address(reg0, offsetOfJitTop)); // rt->profilingActivation = rt->activation()->prevProfiling_; masm.loadPtr(Address(reg1, Activation::offsetOfPrevProfiling()), reg2); masm.storePtr(reg2, Address(reg0, offsetOfProfilingActivation)); // rt->activation()->active_ = false; masm.store8(Imm32(0), Address(reg1, JitActivation::offsetOfActiveUint8())); // rt->jitJSContext = prevJitJSContext_; masm.loadPtr(Address(reg1, JitActivation::offsetOfPrevJitJSContext()), reg2); masm.storePtr(reg2, Address(reg0, offsetOfJitJSContext)); // rt->jitActivation = prevJitActivation_; masm.loadPtr(Address(reg1, JitActivation::offsetOfPrevJitActivation()), reg2); masm.storePtr(reg2, Address(reg0, offsetOfJitActivation)); } // Reload the global register since JIT code can clobber any register. #if defined(JS_CODEGEN_ARM) || defined(JS_CODEGEN_MIPS32) || defined(JS_CODEGEN_MIPS64) static_assert(MaybeSavedGlobalReg == sizeof(void*), "stack frame accounting"); masm.loadPtr(Address(masm.getStackPointer(), jitFrameBytes), GlobalReg); #endif // As explained above, the frame was aligned for the JIT ABI such that // (sp + sizeof(void*)) % JitStackAlignment == 0 // But now we possibly want to call one of several different C++ functions, // so subtract the sizeof(void*) so that sp is aligned for an ABI call. static_assert(ABIStackAlignment <= JitStackAlignment, "subsumes"); masm.reserveStack(sizeOfRetAddr); unsigned nativeFramePushed = masm.framePushed(); AssertStackAlignment(masm, ABIStackAlignment); masm.branchTestMagic(Assembler::Equal, JSReturnOperand, throwLabel); Label oolConvert; switch (sig.ret()) { case ExprType::Void: break; case ExprType::I32: masm.convertValueToInt32(JSReturnOperand, ReturnDoubleReg, ReturnReg, &oolConvert, /* -0 check */ false); break; case ExprType::I64: MOZ_CRASH("no int64 in asm.js"); case ExprType::F32: MOZ_CRASH("Float shouldn't be returned from an import"); case ExprType::F64: masm.convertValueToDouble(JSReturnOperand, ReturnDoubleReg, &oolConvert); break; case ExprType::I32x4: case ExprType::F32x4: case ExprType::B32x4: MOZ_CRASH("SIMD types shouldn't be returned from an import"); } Label done; masm.bind(&done); // Ion code does not respect system callee-saved register conventions so // reload the heap register. if (usesHeap) masm.loadAsmJSHeapRegisterFromGlobalData(); GenerateExitEpilogue(masm, masm.framePushed(), ExitReason::ImportJit, offsets); if (oolConvert.used()) { masm.bind(&oolConvert); masm.setFramePushed(nativeFramePushed); // Coercion calls use the following stack layout (sp grows to the left): // | args | padding | Value argv[1] | padding | exit AsmJSFrame | MIRTypeVector coerceArgTypes; JS_ALWAYS_TRUE(coerceArgTypes.append(MIRType_Pointer)); unsigned offsetToCoerceArgv = AlignBytes(StackArgBytes(coerceArgTypes), sizeof(Value)); MOZ_ASSERT(nativeFramePushed >= offsetToCoerceArgv + sizeof(Value)); AssertStackAlignment(masm, ABIStackAlignment); // Store return value into argv[0] masm.storeValue(JSReturnOperand, Address(masm.getStackPointer(), offsetToCoerceArgv)); // argument 0: argv ABIArgMIRTypeIter i(coerceArgTypes); Address argv(masm.getStackPointer(), offsetToCoerceArgv); if (i->kind() == ABIArg::GPR) { masm.computeEffectiveAddress(argv, i->gpr()); } else { masm.computeEffectiveAddress(argv, scratch); masm.storePtr(scratch, Address(masm.getStackPointer(), i->offsetFromArgBase())); } i++; MOZ_ASSERT(i.done()); // Call coercion function AssertStackAlignment(masm, ABIStackAlignment); switch (sig.ret()) { case ExprType::I32: masm.call(SymbolicAddress::CoerceInPlace_ToInt32); masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); masm.unboxInt32(Address(masm.getStackPointer(), offsetToCoerceArgv), ReturnReg); break; case ExprType::F64: masm.call(SymbolicAddress::CoerceInPlace_ToNumber); masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); masm.loadDouble(Address(masm.getStackPointer(), offsetToCoerceArgv), ReturnDoubleReg); break; default: MOZ_CRASH("Unsupported convert type"); } masm.jump(&done); masm.setFramePushed(0); } MOZ_ASSERT(masm.framePushed() == 0); if (masm.oom()) return false; offsets->end = masm.currentOffset(); return true; }
// Generate a stub that is called via the internal ABI derived from the // signature of the import and calls into an appropriate InvokeImport C++ // function, having boxed all the ABI arguments into a homogeneous Value array. static bool GenerateInterpExitStub(ModuleGenerator& mg, unsigned importIndex, Label* throwLabel, ProfilingOffsets* offsets) { MacroAssembler& masm = mg.masm(); const Sig& sig = *mg.import(importIndex).sig; masm.setFramePushed(0); // Argument types for InvokeImport_*: static const MIRType typeArray[] = { MIRType_Pointer, // ImportExit MIRType_Int32, // argc MIRType_Pointer }; // argv MIRTypeVector invokeArgTypes; MOZ_ALWAYS_TRUE(invokeArgTypes.append(typeArray, ArrayLength(typeArray))); // At the point of the call, the stack layout shall be (sp grows to the left): // | stack args | padding | Value argv[] | padding | retaddr | caller stack args | // The padding between stack args and argv ensures that argv is aligned. The // padding between argv and retaddr ensures that sp is aligned. unsigned argOffset = AlignBytes(StackArgBytes(invokeArgTypes), sizeof(double)); unsigned argBytes = Max<size_t>(1, sig.args().length()) * sizeof(Value); unsigned framePushed = StackDecrementForCall(masm, ABIStackAlignment, argOffset + argBytes); GenerateExitPrologue(masm, framePushed, ExitReason::ImportInterp, offsets); // Fill the argument array. unsigned offsetToCallerStackArgs = sizeof(AsmJSFrame) + masm.framePushed(); Register scratch = ABIArgGenerator::NonArgReturnReg0; FillArgumentArray(masm, sig.args(), argOffset, offsetToCallerStackArgs, scratch); // Prepare the arguments for the call to InvokeImport_*. ABIArgMIRTypeIter i(invokeArgTypes); // argument 0: importIndex if (i->kind() == ABIArg::GPR) masm.mov(ImmWord(importIndex), i->gpr()); else masm.store32(Imm32(importIndex), Address(masm.getStackPointer(), i->offsetFromArgBase())); i++; // argument 1: argc unsigned argc = sig.args().length(); if (i->kind() == ABIArg::GPR) masm.mov(ImmWord(argc), i->gpr()); else masm.store32(Imm32(argc), Address(masm.getStackPointer(), i->offsetFromArgBase())); i++; // argument 2: argv Address argv(masm.getStackPointer(), argOffset); if (i->kind() == ABIArg::GPR) { masm.computeEffectiveAddress(argv, i->gpr()); } else { masm.computeEffectiveAddress(argv, scratch); masm.storePtr(scratch, Address(masm.getStackPointer(), i->offsetFromArgBase())); } i++; MOZ_ASSERT(i.done()); // Make the call, test whether it succeeded, and extract the return value. AssertStackAlignment(masm, ABIStackAlignment); switch (sig.ret()) { case ExprType::Void: masm.call(SymbolicAddress::InvokeImport_Void); masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); break; case ExprType::I32: masm.call(SymbolicAddress::InvokeImport_I32); masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); masm.unboxInt32(argv, ReturnReg); break; case ExprType::I64: MOZ_CRASH("no int64 in asm.js"); case ExprType::F32: MOZ_CRASH("Float32 shouldn't be returned from a FFI"); case ExprType::F64: masm.call(SymbolicAddress::InvokeImport_F64); masm.branchTest32(Assembler::Zero, ReturnReg, ReturnReg, throwLabel); masm.loadDouble(argv, ReturnDoubleReg); break; case ExprType::I32x4: case ExprType::F32x4: case ExprType::B32x4: MOZ_CRASH("SIMD types shouldn't be returned from a FFI"); } GenerateExitEpilogue(masm, framePushed, ExitReason::ImportInterp, offsets); if (masm.oom()) return false; offsets->end = masm.currentOffset(); return true; }