CCallHelpers::JumpList generateImpl(AccessGenerationState& state, const RegisterSet& usedRegistersBySnippet, CCallHelpers& jit, std::index_sequence<ArgumentsIndex...>) { CCallHelpers::JumpList exceptions; // We spill (1) the used registers by IC and (2) the used registers by Snippet. AccessGenerationState::SpillState spillState = state.preserveLiveRegistersToStackForCall(usedRegistersBySnippet); jit.store32( CCallHelpers::TrustedImm32(state.callSiteIndexForExceptionHandlingOrOriginal().bits()), CCallHelpers::tagFor(static_cast<VirtualRegister>(CallFrameSlot::argumentCount))); jit.makeSpaceOnStackForCCall(); jit.setupArguments<FunctionType>(std::get<ArgumentsIndex>(m_arguments)...); CCallHelpers::Call operationCall = jit.call(OperationPtrTag); auto function = m_function; jit.addLinkTask([=] (LinkBuffer& linkBuffer) { linkBuffer.link(operationCall, FunctionPtr<OperationPtrTag>(function)); }); jit.setupResults(m_result); jit.reclaimSpaceOnStackForCCall(); CCallHelpers::Jump noException = jit.emitExceptionCheck(state.m_vm, CCallHelpers::InvertedExceptionCheck); state.restoreLiveRegistersFromStackForCallWithThrownException(spillState); exceptions.append(jit.jump()); noException.link(&jit); RegisterSet dontRestore; dontRestore.set(m_result); state.restoreLiveRegistersFromStackForCall(spillState, dontRestore); return exceptions; }
void BasicBlockLocation::emitExecuteCode(CCallHelpers& jit, MacroAssembler::RegisterID scratch) const { static_assert(sizeof(size_t) == 4, "Assuming size_t is 32 bits on 32 bit platforms."); jit.load32(&m_executionCount, scratch); CCallHelpers::Jump done = jit.branchAdd32(CCallHelpers::Zero, scratch, CCallHelpers::TrustedImm32(1), scratch); jit.store32(scratch, bitwise_cast<void*>(&m_executionCount)); done.link(&jit); }
inline void emitPointerValidation(CCallHelpers& jit, GPRReg pointerGPR) { #if !ASSERT_DISABLED CCallHelpers::Jump isNonZero = jit.branchTestPtr(CCallHelpers::NonZero, pointerGPR); jit.breakpoint(); isNonZero.link(&jit); jit.push(pointerGPR); jit.load8(pointerGPR, pointerGPR); jit.pop(pointerGPR); #else UNUSED_PARAM(jit); UNUSED_PARAM(pointerGPR); #endif }
void ArithProfile::emitObserveResult(CCallHelpers& jit, JSValueRegs regs, TagRegistersMode mode) { if (!shouldEmitSetDouble() && !shouldEmitSetNonNumber()) return; CCallHelpers::Jump isInt32 = jit.branchIfInt32(regs, mode); CCallHelpers::Jump notDouble = jit.branchIfNotDoubleKnownNotInt32(regs, mode); emitSetDouble(jit); CCallHelpers::Jump done = jit.jump(); notDouble.link(&jit); emitSetNonNumber(jit); done.link(&jit); isInt32.link(&jit); }
static void slowPathFor( CCallHelpers& jit, VM* vm, Sprt_JITOperation_ECli slowPathFunction) { jit.emitFunctionPrologue(); jit.storePtr(GPRInfo::callFrameRegister, &vm->topCallFrame); #if OS(WINDOWS) && CPU(X86_64) // Windows X86_64 needs some space pointed to by arg0 for return types larger than 64 bits. // Other argument values are shift by 1. Use space on the stack for our two return values. // Moving the stack down maxFrameExtentForSlowPathCall bytes gives us room for our 3 arguments // and space for the 16 byte return area. jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister); jit.move(GPRInfo::regT2, GPRInfo::argumentGPR2); jit.addPtr(CCallHelpers::TrustedImm32(32), CCallHelpers::stackPointerRegister, GPRInfo::argumentGPR0); jit.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0); emitPointerValidation(jit, GPRInfo::nonArgGPR0); jit.call(GPRInfo::nonArgGPR0); jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR, 8), GPRInfo::returnValueGPR2); jit.loadPtr(CCallHelpers::Address(GPRInfo::returnValueGPR), GPRInfo::returnValueGPR); jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister); #else if (maxFrameExtentForSlowPathCall) jit.addPtr(CCallHelpers::TrustedImm32(-maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister); jit.setupArgumentsWithExecState(GPRInfo::regT2); jit.move(CCallHelpers::TrustedImmPtr(bitwise_cast<void*>(slowPathFunction)), GPRInfo::nonArgGPR0); emitPointerValidation(jit, GPRInfo::nonArgGPR0); jit.call(GPRInfo::nonArgGPR0); if (maxFrameExtentForSlowPathCall) jit.addPtr(CCallHelpers::TrustedImm32(maxFrameExtentForSlowPathCall), CCallHelpers::stackPointerRegister); #endif // This slow call will return the address of one of the following: // 1) Exception throwing thunk. // 2) Host call return value returner thingy. // 3) The function to call. // The second return value GPR will hold a non-zero value for tail calls. emitPointerValidation(jit, GPRInfo::returnValueGPR); jit.emitFunctionEpilogue(); RELEASE_ASSERT(reinterpret_cast<void*>(KeepTheFrame) == reinterpret_cast<void*>(0)); CCallHelpers::Jump doNotTrash = jit.branchTestPtr(CCallHelpers::Zero, GPRInfo::returnValueGPR2); jit.preserveReturnAddressAfterCall(GPRInfo::nonPreservedNonReturnGPR); jit.prepareForTailCallSlow(GPRInfo::returnValueGPR); doNotTrash.link(&jit); jit.jump(GPRInfo::returnValueGPR); }
void JSCallBase::emit(CCallHelpers& jit) { m_callLinkInfo = jit.codeBlock()->addCallLinkInfo(); CCallHelpers::Jump slowPath = jit.branchPtrWithPatch( CCallHelpers::NotEqual, GPRInfo::regT0, m_targetToCheck, CCallHelpers::TrustedImmPtr(0)); m_fastCall = jit.nearCall(); CCallHelpers::Jump done = jit.jump(); slowPath.link(&jit); jit.move(CCallHelpers::TrustedImmPtr(m_callLinkInfo), GPRInfo::regT2); m_slowCall = jit.nearCall(); done.link(&jit); }
void JSCallBase::emit(CCallHelpers& jit, State& /*state*/, int32_t osrExitFromGenericUnwindStackSpillSlot) { RELEASE_ASSERT(!!m_callSiteIndex); #if FTL_USES_B3 UNUSED_PARAM(osrExitFromGenericUnwindStackSpillSlot); #else // FTL_USES_B3 if (m_correspondingGenericUnwindOSRExit) m_correspondingGenericUnwindOSRExit->spillRegistersToSpillSlot(jit, osrExitFromGenericUnwindStackSpillSlot); #endif // FTL_USES_B3 jit.store32(CCallHelpers::TrustedImm32(m_callSiteIndex.bits()), CCallHelpers::tagFor(static_cast<VirtualRegister>(JSStack::ArgumentCount))); m_callLinkInfo = jit.codeBlock()->addCallLinkInfo(); if (CallLinkInfo::callModeFor(m_type) == CallMode::Tail) jit.emitRestoreCalleeSaves(); CCallHelpers::Jump slowPath = jit.branchPtrWithPatch( CCallHelpers::NotEqual, GPRInfo::regT0, m_targetToCheck, CCallHelpers::TrustedImmPtr(0)); CCallHelpers::Jump done; if (CallLinkInfo::callModeFor(m_type) == CallMode::Tail) { jit.prepareForTailCallSlow(); m_fastCall = jit.nearTailCall(); } else { m_fastCall = jit.nearCall(); done = jit.jump(); } slowPath.link(&jit); jit.move(CCallHelpers::TrustedImmPtr(m_callLinkInfo), GPRInfo::regT2); m_slowCall = jit.nearCall(); if (CallLinkInfo::callModeFor(m_type) == CallMode::Tail) jit.abortWithReason(JITDidReturnFromTailCall); else done.link(&jit); m_callLinkInfo->setUpCall(m_type, m_semanticeOrigin, GPRInfo::regT0); }
CCallHelpers::Jump CheckSpecial::generate(Inst& inst, CCallHelpers& jit, GenerationContext& context) { CCallHelpers::Jump fail = hiddenBranch(inst).generate(jit, context); ASSERT(fail.isSet()); Value* value = inst.origin; Vector<ValueRep> reps; if (isCheckMath(value->opcode())) { if (value->opcode() == CheckMul) reps.append(ValueRep()); else if (value->opcode() == CheckSub && value->child(0)->isInt(0)) reps.append(ValueRep::constant(0)); else reps.append(repForArg(*context.code, inst.args[3])); reps.append(repForArg(*context.code, inst.args[2])); } else { ASSERT(value->opcode() == Check); reps.append(ValueRep::constant(1)); } appendRepsImpl(context, m_numCheckArgs + 1, inst, reps); context.latePaths.append( createSharedTask<GenerationContext::LatePathFunction>( [=] (CCallHelpers& jit, GenerationContext&) { fail.link(&jit); Stackmap* stackmap = value->stackmap(); ASSERT(stackmap); Stackmap::GenerationParams params; params.value = value; params.stackmap = stackmap; params.reps = reps; params.usedRegisters = stackmap->m_usedRegisters; stackmap->m_generator->run(jit, params); })); return CCallHelpers::Jump(); // As far as Air thinks, we are not a terminal. }
bool JITNegGenerator::generateFastPath(CCallHelpers& jit, CCallHelpers::JumpList& endJumpList, CCallHelpers::JumpList& slowPathJumpList, const ArithProfile* arithProfile, bool shouldEmitProfiling) { ASSERT(m_scratchGPR != m_src.payloadGPR()); ASSERT(m_scratchGPR != m_result.payloadGPR()); ASSERT(m_scratchGPR != InvalidGPRReg); #if USE(JSVALUE32_64) ASSERT(m_scratchGPR != m_src.tagGPR()); ASSERT(m_scratchGPR != m_result.tagGPR()); #endif jit.moveValueRegs(m_src, m_result); CCallHelpers::Jump srcNotInt = jit.branchIfNotInt32(m_src); // -0 should produce a double, and hence cannot be negated as an int. // The negative int32 0x80000000 doesn't have a positive int32 representation, and hence cannot be negated as an int. slowPathJumpList.append(jit.branchTest32(CCallHelpers::Zero, m_src.payloadGPR(), CCallHelpers::TrustedImm32(0x7fffffff))); jit.neg32(m_result.payloadGPR()); #if USE(JSVALUE64) jit.boxInt32(m_result.payloadGPR(), m_result); #endif endJumpList.append(jit.jump()); srcNotInt.link(&jit); slowPathJumpList.append(jit.branchIfNotNumber(m_src, m_scratchGPR)); // For a double, all we need to do is to invert the sign bit. #if USE(JSVALUE64) jit.move(CCallHelpers::TrustedImm64((int64_t)(1ull << 63)), m_scratchGPR); jit.xor64(m_scratchGPR, m_result.payloadGPR()); #else jit.xor32(CCallHelpers::TrustedImm32(1 << 31), m_result.tagGPR()); #endif // The flags of ArithNegate are basic in DFG. // We only need to know if we ever produced a number. if (shouldEmitProfiling && arithProfile && !arithProfile->lhsObservedType().sawNumber() && !arithProfile->didObserveDouble()) arithProfile->emitSetDouble(jit); return true; }
void emitSetupVarargsFrameFastCase(CCallHelpers& jit, GPRReg numUsedSlotsGPR, GPRReg scratchGPR1, GPRReg scratchGPR2, GPRReg scratchGPR3, ValueRecovery argCountRecovery, VirtualRegister firstArgumentReg, unsigned firstVarArgOffset, CCallHelpers::JumpList& slowCase) { CCallHelpers::JumpList end; if (argCountRecovery.isConstant()) { // FIXME: We could constant-fold a lot of the computation below in this case. // https://bugs.webkit.org/show_bug.cgi?id=141486 jit.move(CCallHelpers::TrustedImm32(argCountRecovery.constant().asInt32()), scratchGPR1); } else jit.load32(CCallHelpers::payloadFor(argCountRecovery.virtualRegister()), scratchGPR1); if (firstVarArgOffset) { CCallHelpers::Jump sufficientArguments = jit.branch32(CCallHelpers::GreaterThan, scratchGPR1, CCallHelpers::TrustedImm32(firstVarArgOffset + 1)); jit.move(CCallHelpers::TrustedImm32(1), scratchGPR1); CCallHelpers::Jump endVarArgs = jit.jump(); sufficientArguments.link(&jit); jit.sub32(CCallHelpers::TrustedImm32(firstVarArgOffset), scratchGPR1); endVarArgs.link(&jit); } slowCase.append(jit.branch32(CCallHelpers::Above, scratchGPR1, CCallHelpers::TrustedImm32(maxArguments + 1))); emitSetVarargsFrame(jit, scratchGPR1, true, numUsedSlotsGPR, scratchGPR2); slowCase.append(jit.branchPtr(CCallHelpers::Above, CCallHelpers::AbsoluteAddress(jit.vm()->addressOfStackLimit()), scratchGPR2)); // Initialize ArgumentCount. jit.store32(scratchGPR1, CCallHelpers::Address(scratchGPR2, JSStack::ArgumentCount * static_cast<int>(sizeof(Register)) + PayloadOffset)); // Copy arguments. jit.signExtend32ToPtr(scratchGPR1, scratchGPR1); CCallHelpers::Jump done = jit.branchSubPtr(CCallHelpers::Zero, CCallHelpers::TrustedImm32(1), scratchGPR1); // scratchGPR1: argumentCount CCallHelpers::Label copyLoop = jit.label(); int argOffset = (firstArgumentReg.offset() - 1 + firstVarArgOffset) * static_cast<int>(sizeof(Register)); #if USE(JSVALUE64) jit.load64(CCallHelpers::BaseIndex(GPRInfo::callFrameRegister, scratchGPR1, CCallHelpers::TimesEight, argOffset), scratchGPR3); jit.store64(scratchGPR3, CCallHelpers::BaseIndex(scratchGPR2, scratchGPR1, CCallHelpers::TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)))); #else // USE(JSVALUE64), so this begins the 32-bit case jit.load32(CCallHelpers::BaseIndex(GPRInfo::callFrameRegister, scratchGPR1, CCallHelpers::TimesEight, argOffset + TagOffset), scratchGPR3); jit.store32(scratchGPR3, CCallHelpers::BaseIndex(scratchGPR2, scratchGPR1, CCallHelpers::TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)) + TagOffset)); jit.load32(CCallHelpers::BaseIndex(GPRInfo::callFrameRegister, scratchGPR1, CCallHelpers::TimesEight, argOffset + PayloadOffset), scratchGPR3); jit.store32(scratchGPR3, CCallHelpers::BaseIndex(scratchGPR2, scratchGPR1, CCallHelpers::TimesEight, CallFrame::thisArgumentOffset() * static_cast<int>(sizeof(Register)) + PayloadOffset)); #endif // USE(JSVALUE64), end of 32-bit case jit.branchSubPtr(CCallHelpers::NonZero, CCallHelpers::TrustedImm32(1), scratchGPR1).linkTo(copyLoop, &jit); done.link(&jit); }
void JITRightShiftGenerator::generateFastPath(CCallHelpers& jit) { ASSERT(m_scratchGPR != InvalidGPRReg); ASSERT(m_scratchGPR != m_left.payloadGPR()); ASSERT(m_scratchGPR != m_right.payloadGPR()); #if USE(JSVALUE32_64) ASSERT(m_scratchGPR != m_left.tagGPR()); ASSERT(m_scratchGPR != m_right.tagGPR()); ASSERT(m_scratchFPR != InvalidFPRReg); #endif ASSERT(!m_leftOperand.isConstInt32() || !m_rightOperand.isConstInt32()); m_didEmitFastPath = true; if (m_rightOperand.isConstInt32()) { // Try to do (intVar >> intConstant). CCallHelpers::Jump notInt = jit.branchIfNotInt32(m_left); jit.moveValueRegs(m_left, m_result); int32_t shiftAmount = m_rightOperand.asConstInt32() & 0x1f; if (shiftAmount) { if (m_shiftType == SignedShift) jit.rshift32(CCallHelpers::Imm32(shiftAmount), m_result.payloadGPR()); else jit.urshift32(CCallHelpers::Imm32(shiftAmount), m_result.payloadGPR()); #if USE(JSVALUE64) jit.or64(GPRInfo::tagTypeNumberRegister, m_result.payloadGPR()); #endif } if (jit.supportsFloatingPointTruncate()) { m_endJumpList.append(jit.jump()); // Terminate the above case before emitting more code. // Try to do (doubleVar >> intConstant). notInt.link(&jit); m_slowPathJumpList.append(jit.branchIfNotNumber(m_left, m_scratchGPR)); jit.unboxDoubleNonDestructive(m_left, m_leftFPR, m_scratchGPR, m_scratchFPR); m_slowPathJumpList.append(jit.branchTruncateDoubleToInt32(m_leftFPR, m_scratchGPR)); if (shiftAmount) { if (m_shiftType == SignedShift) jit.rshift32(CCallHelpers::Imm32(shiftAmount), m_scratchGPR); else jit.urshift32(CCallHelpers::Imm32(shiftAmount), m_scratchGPR); } jit.boxInt32(m_scratchGPR, m_result); } else m_slowPathJumpList.append(notInt); } else { // Try to do (intConstant >> intVar) or (intVar >> intVar). m_slowPathJumpList.append(jit.branchIfNotInt32(m_right)); CCallHelpers::Jump notInt; if (m_leftOperand.isConstInt32()) { #if USE(JSVALUE32_64) jit.move(m_right.tagGPR(), m_result.tagGPR()); #endif jit.move(CCallHelpers::Imm32(m_leftOperand.asConstInt32()), m_result.payloadGPR()); } else { notInt = jit.branchIfNotInt32(m_left); jit.moveValueRegs(m_left, m_result); } if (m_shiftType == SignedShift) jit.rshift32(m_right.payloadGPR(), m_result.payloadGPR()); else jit.urshift32(m_right.payloadGPR(), m_result.payloadGPR()); #if USE(JSVALUE64) jit.or64(GPRInfo::tagTypeNumberRegister, m_result.payloadGPR()); #endif if (m_leftOperand.isConstInt32()) return; if (jit.supportsFloatingPointTruncate()) { m_endJumpList.append(jit.jump()); // Terminate the above case before emitting more code. // Try to do (doubleVar >> intVar). notInt.link(&jit); m_slowPathJumpList.append(jit.branchIfNotNumber(m_left, m_scratchGPR)); jit.unboxDoubleNonDestructive(m_left, m_leftFPR, m_scratchGPR, m_scratchFPR); m_slowPathJumpList.append(jit.branchTruncateDoubleToInt32(m_leftFPR, m_scratchGPR)); if (m_shiftType == SignedShift) jit.rshift32(m_right.payloadGPR(), m_scratchGPR); else jit.urshift32(m_right.payloadGPR(), m_scratchGPR); jit.boxInt32(m_scratchGPR, m_result); } else m_slowPathJumpList.append(notInt); } }
CCallHelpers::Jump CheckSpecial::generate(Inst& inst, CCallHelpers& jit, GenerationContext& context) { CCallHelpers::Jump fail = hiddenBranch(inst).generate(jit, context); ASSERT(fail.isSet()); StackmapValue* value = inst.origin->as<StackmapValue>(); ASSERT(value); Vector<ValueRep> reps = repsImpl(context, numB3Args(inst), m_numCheckArgs + 1, inst); // Set aside the args that are relevant to undoing the operation. This is because we don't want to // capture all of inst in the closure below. Vector<Arg, 3> args; for (unsigned i = 0; i < m_numCheckArgs; ++i) args.append(inst.args[1 + i]); context.latePaths.append( createSharedTask<GenerationContext::LatePathFunction>( [=] (CCallHelpers& jit, GenerationContext& context) { fail.link(&jit); // If necessary, undo the operation. switch (m_checkKind.opcode) { case BranchAdd32: if ((m_numCheckArgs == 4 && args[1] == args[2] && args[2] == args[3]) || (m_numCheckArgs == 3 && args[1] == args[2])) { // This is ugly, but that's fine - we won't have to do this very often. ASSERT(args[1].isGPR()); GPRReg valueGPR = args[1].gpr(); GPRReg scratchGPR = CCallHelpers::selectScratchGPR(valueGPR); jit.pushToSave(scratchGPR); jit.setCarry(scratchGPR); jit.lshift32(CCallHelpers::TrustedImm32(31), scratchGPR); jit.urshift32(CCallHelpers::TrustedImm32(1), valueGPR); jit.or32(scratchGPR, valueGPR); jit.popToRestore(scratchGPR); break; } if (m_numCheckArgs == 4) { if (args[1] == args[3]) Inst(Sub32, nullptr, args[2], args[3]).generate(jit, context); else if (args[2] == args[3]) Inst(Sub32, nullptr, args[1], args[3]).generate(jit, context); } else if (m_numCheckArgs == 3) Inst(Sub32, nullptr, args[1], args[2]).generate(jit, context); break; case BranchAdd64: if ((m_numCheckArgs == 4 && args[1] == args[2] && args[2] == args[3]) || (m_numCheckArgs == 3 && args[1] == args[2])) { // This is ugly, but that's fine - we won't have to do this very often. ASSERT(args[1].isGPR()); GPRReg valueGPR = args[1].gpr(); GPRReg scratchGPR = CCallHelpers::selectScratchGPR(valueGPR); jit.pushToSave(scratchGPR); jit.setCarry(scratchGPR); jit.lshift64(CCallHelpers::TrustedImm32(63), scratchGPR); jit.urshift64(CCallHelpers::TrustedImm32(1), valueGPR); jit.or64(scratchGPR, valueGPR); jit.popToRestore(scratchGPR); break; } if (m_numCheckArgs == 4) { if (args[1] == args[3]) Inst(Sub64, nullptr, args[2], args[3]).generate(jit, context); else if (args[2] == args[3]) Inst(Sub64, nullptr, args[1], args[3]).generate(jit, context); } else if (m_numCheckArgs == 3) Inst(Sub64, nullptr, args[1], args[2]).generate(jit, context); break; case BranchSub32: Inst(Add32, nullptr, args[1], args[2]).generate(jit, context); break; case BranchSub64: Inst(Add64, nullptr, args[1], args[2]).generate(jit, context); break; case BranchNeg32: Inst(Neg32, nullptr, args[1]).generate(jit, context); break; case BranchNeg64: Inst(Neg64, nullptr, args[1]).generate(jit, context); break; default: break; } value->m_generator->run(jit, StackmapGenerationParams(value, reps, context)); })); return CCallHelpers::Jump(); // As far as Air thinks, we are not a terminal. }
void generate(Code& code, CCallHelpers& jit) { TimingScope timingScope("Air::generate"); // We don't expect the incoming code to have predecessors computed. code.resetReachability(); if (shouldValidateIR()) validate(code); // If we're doing super verbose dumping, the phase scope of any phase will already do a dump. if (shouldDumpIR() && !shouldDumpIRAtEachPhase()) { dataLog("Initial air:\n"); dataLog(code); } // This is where we run our optimizations and transformations. // FIXME: Add Air optimizations. // https://bugs.webkit.org/show_bug.cgi?id=150456 eliminateDeadCode(code); // This is where we would have a real register allocator. Then, we could use spillEverything() // in place of the register allocator only for testing. // FIXME: https://bugs.webkit.org/show_bug.cgi?id=150457 spillEverything(code); // Prior to this point the prologue and epilogue is implicit. This makes it explicit. It also // does things like identify which callee-saves we're using and saves them. handleCalleeSaves(code); // This turns all Stack and CallArg Args into Addr args that use the frame pointer. It does // this by first-fit allocating stack slots. It should be pretty darn close to optimal, so we // shouldn't have to worry about this very much. allocateStack(code); // If we coalesced moves then we can unbreak critical edges. This is the main reason for this // phase. simplifyCFG(code); // FIXME: We should really have a code layout optimization here. // https://bugs.webkit.org/show_bug.cgi?id=150478 reportUsedRegisters(code); if (shouldValidateIR()) validate(code); // Do a final dump of Air. Note that we have to do this even if we are doing per-phase dumping, // since the final generation is not a phase. if (shouldDumpIR()) { dataLog("Air after ", code.lastPhaseName(), ", before generation:\n"); dataLog(code); } TimingScope codeGenTimingScope("Air::generate backend"); // And now, we generate code. jit.emitFunctionPrologue(); jit.addPtr(CCallHelpers::TrustedImm32(-code.frameSize()), MacroAssembler::stackPointerRegister); GenerationContext context; context.code = &code; IndexMap<BasicBlock, CCallHelpers::Label> blockLabels(code.size()); IndexMap<BasicBlock, CCallHelpers::JumpList> blockJumps(code.size()); auto link = [&] (CCallHelpers::Jump jump, BasicBlock* target) { if (blockLabels[target].isSet()) { jump.linkTo(blockLabels[target], &jit); return; } blockJumps[target].append(jump); }; for (BasicBlock* block : code) { blockJumps[block].link(&jit); ASSERT(block->size() >= 1); for (unsigned i = 0; i < block->size() - 1; ++i) { CCallHelpers::Jump jump = block->at(i).generate(jit, context); ASSERT_UNUSED(jump, !jump.isSet()); } if (block->last().opcode == Jump && block->successorBlock(0) == code.findNextBlock(block)) continue; if (block->last().opcode == Ret) { // We currently don't represent the full prologue/epilogue in Air, so we need to // have this override. jit.emitFunctionEpilogue(); jit.ret(); continue; } CCallHelpers::Jump jump = block->last().generate(jit, context); for (Inst& inst : *block) jump = inst.generate(jit, context); switch (block->numSuccessors()) { case 0: ASSERT(!jump.isSet()); break; case 1: link(jump, block->successorBlock(0)); break; case 2: link(jump, block->successorBlock(0)); if (block->successorBlock(1) != code.findNextBlock(block)) link(jit.jump(), block->successorBlock(1)); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } } for (auto& latePath : context.latePaths) latePath->run(jit, context); }
void JSTailCall::emit(JITCode& jitCode, CCallHelpers& jit) { StackMaps::Record* record { nullptr }; for (unsigned i = jitCode.stackmaps.records.size(); i--;) { record = &jitCode.stackmaps.records[i]; if (record->patchpointID == m_stackmapID) break; } RELEASE_ASSERT(record->patchpointID == m_stackmapID); m_callLinkInfo = jit.codeBlock()->addCallLinkInfo(); CallFrameShuffleData shuffleData; // The callee was the first passed argument, and must be in a GPR because // we used the "anyregcc" calling convention auto calleeLocation = FTL::Location::forStackmaps(nullptr, record->locations[0]); GPRReg calleeGPR = calleeLocation.directGPR(); shuffleData.callee = ValueRecovery::inGPR(calleeGPR, DataFormatJS); // The tag type number was the second argument, if there was one auto tagTypeNumberLocation = FTL::Location::forStackmaps(&jitCode.stackmaps, record->locations[1]); if (tagTypeNumberLocation.isGPR() && !tagTypeNumberLocation.addend()) shuffleData.tagTypeNumber = tagTypeNumberLocation.directGPR(); shuffleData.args.grow(numArguments()); HashMap<Reg, Vector<std::pair<ValueRecovery*, int32_t>>> withAddend; size_t numAddends { 0 }; for (size_t i = 0; i < numArguments(); ++i) { shuffleData.args[i] = recoveryFor(m_arguments[i], *record, jitCode.stackmaps); if (FTL::Location addend = getRegisterWithAddend(m_arguments[i], *record, jitCode.stackmaps)) { withAddend.add( addend.reg(), Vector<std::pair<ValueRecovery*, int32_t>>()).iterator->value.append( std::make_pair(&shuffleData.args[i], addend.addend())); numAddends++; } } numAddends = WTF::roundUpToMultipleOf(stackAlignmentRegisters(), numAddends); shuffleData.numLocals = static_cast<int64_t>(jitCode.stackmaps.stackSizeForLocals()) / sizeof(void*) + numAddends; ASSERT(!numAddends == withAddend.isEmpty()); if (!withAddend.isEmpty()) { jit.subPtr(MacroAssembler::TrustedImm32(numAddends * sizeof(void*)), MacroAssembler::stackPointerRegister); VirtualRegister spillBase { 1 - static_cast<int>(shuffleData.numLocals) }; for (auto entry : withAddend) { for (auto pair : entry.value) { ASSERT(numAddends > 0); VirtualRegister spillSlot { spillBase + --numAddends }; ASSERT(entry.key.isGPR()); jit.addPtr(MacroAssembler::TrustedImm32(pair.second), entry.key.gpr()); jit.storePtr(entry.key.gpr(), CCallHelpers::addressFor(spillSlot)); jit.subPtr(MacroAssembler::TrustedImm32(pair.second), entry.key.gpr()); *pair.first = ValueRecovery::displacedInJSStack(spillSlot, pair.first->dataFormat()); } } ASSERT(numAddends < stackAlignmentRegisters()); } shuffleData.args.resize(numArguments()); for (size_t i = 0; i < numArguments(); ++i) shuffleData.args[i] = recoveryFor(m_arguments[i], *record, jitCode.stackmaps); shuffleData.setupCalleeSaveRegisters(jit.codeBlock()); CCallHelpers::Jump slowPath = jit.branchPtrWithPatch( CCallHelpers::NotEqual, calleeGPR, m_targetToCheck, CCallHelpers::TrustedImmPtr(0)); m_callLinkInfo->setFrameShuffleData(shuffleData); CallFrameShuffler(jit, shuffleData).prepareForTailCall(); m_fastCall = jit.nearTailCall(); slowPath.link(&jit); CallFrameShuffler slowPathShuffler(jit, shuffleData); slowPathShuffler.setCalleeJSValueRegs(JSValueRegs { GPRInfo::regT0 }); slowPathShuffler.prepareForSlowPath(); jit.move(CCallHelpers::TrustedImmPtr(m_callLinkInfo), GPRInfo::regT2); m_slowCall = jit.nearCall(); jit.abortWithReason(JITDidReturnFromTailCall); m_callLinkInfo->setUpCall(m_type, m_semanticeOrigin, calleeGPR); }
void generate(Code& code, CCallHelpers& jit) { TimingScope timingScope("Air::generate"); DisallowMacroScratchRegisterUsage disallowScratch(jit); // And now, we generate code. jit.emitFunctionPrologue(); if (code.frameSize()) jit.addPtr(CCallHelpers::TrustedImm32(-code.frameSize()), MacroAssembler::stackPointerRegister); GenerationContext context; context.code = &code; IndexMap<BasicBlock, CCallHelpers::Label> blockLabels(code.size()); IndexMap<BasicBlock, CCallHelpers::JumpList> blockJumps(code.size()); auto link = [&] (CCallHelpers::Jump jump, BasicBlock* target) { if (blockLabels[target].isSet()) { jump.linkTo(blockLabels[target], &jit); return; } blockJumps[target].append(jump); }; for (BasicBlock* block : code) { blockJumps[block].link(&jit); blockLabels[block] = jit.label(); ASSERT(block->size() >= 1); for (unsigned i = 0; i < block->size() - 1; ++i) { CCallHelpers::Jump jump = block->at(i).generate(jit, context); ASSERT_UNUSED(jump, !jump.isSet()); } if (block->last().opcode == Jump && block->successorBlock(0) == code.findNextBlock(block)) continue; if (block->last().opcode == Ret) { // We currently don't represent the full prologue/epilogue in Air, so we need to // have this override. if (code.frameSize()) jit.emitFunctionEpilogue(); else jit.emitFunctionEpilogueWithEmptyFrame(); jit.ret(); continue; } CCallHelpers::Jump jump = block->last().generate(jit, context); switch (block->numSuccessors()) { case 0: ASSERT(!jump.isSet()); break; case 1: link(jump, block->successorBlock(0)); break; case 2: link(jump, block->successorBlock(0)); if (block->successorBlock(1) != code.findNextBlock(block)) link(jit.jump(), block->successorBlock(1)); break; default: RELEASE_ASSERT_NOT_REACHED(); break; } } for (auto& latePath : context.latePaths) latePath->run(jit, context); }
void JSCallVarargs::emit(CCallHelpers& jit, State& state, int32_t spillSlotsOffset, int32_t osrExitFromGenericUnwindSpillSlots) { // We are passed three pieces of information: // - The callee. // - The arguments object, if it's not a forwarding call. // - The "this" value, if it's a constructor call. CallVarargsData* data = m_node->callVarargsData(); GPRReg calleeGPR = GPRInfo::argumentGPR0; GPRReg argumentsGPR = InvalidGPRReg; GPRReg thisGPR = InvalidGPRReg; bool forwarding = false; switch (m_node->op()) { case CallVarargs: case TailCallVarargs: case TailCallVarargsInlinedCaller: case ConstructVarargs: argumentsGPR = GPRInfo::argumentGPR1; thisGPR = GPRInfo::argumentGPR2; break; case CallForwardVarargs: case TailCallForwardVarargs: case TailCallForwardVarargsInlinedCaller: case ConstructForwardVarargs: thisGPR = GPRInfo::argumentGPR1; forwarding = true; break; default: RELEASE_ASSERT_NOT_REACHED(); break; } const unsigned calleeSpillSlot = 0; const unsigned argumentsSpillSlot = 1; const unsigned thisSpillSlot = 2; const unsigned stackPointerSpillSlot = 3; // Get some scratch registers. RegisterSet usedRegisters; usedRegisters.merge(RegisterSet::stackRegisters()); usedRegisters.merge(RegisterSet::reservedHardwareRegisters()); usedRegisters.merge(RegisterSet::calleeSaveRegisters()); usedRegisters.set(calleeGPR); if (argumentsGPR != InvalidGPRReg) usedRegisters.set(argumentsGPR); ASSERT(thisGPR); usedRegisters.set(thisGPR); ScratchRegisterAllocator allocator(usedRegisters); GPRReg scratchGPR1 = allocator.allocateScratchGPR(); GPRReg scratchGPR2 = allocator.allocateScratchGPR(); GPRReg scratchGPR3 = allocator.allocateScratchGPR(); RELEASE_ASSERT(!allocator.numberOfReusedRegisters()); auto computeUsedStack = [&] (GPRReg targetGPR, unsigned extra) { if (isARM64()) { // Have to do this the weird way because $sp on ARM64 means zero when used in a subtraction. jit.move(CCallHelpers::stackPointerRegister, targetGPR); jit.negPtr(targetGPR); jit.addPtr(GPRInfo::callFrameRegister, targetGPR); } else { jit.move(GPRInfo::callFrameRegister, targetGPR); jit.subPtr(CCallHelpers::stackPointerRegister, targetGPR); } if (extra) jit.subPtr(CCallHelpers::TrustedImm32(extra), targetGPR); jit.urshiftPtr(CCallHelpers::Imm32(3), targetGPR); }; auto callWithExceptionCheck = [&] (void* callee) { jit.move(CCallHelpers::TrustedImmPtr(callee), GPRInfo::nonPreservedNonArgumentGPR); jit.call(GPRInfo::nonPreservedNonArgumentGPR); m_exceptions.append(jit.emitExceptionCheck(AssemblyHelpers::NormalExceptionCheck, AssemblyHelpers::FarJumpWidth)); }; if (isARM64()) { jit.move(CCallHelpers::stackPointerRegister, scratchGPR1); jit.storePtr(scratchGPR1, CCallHelpers::addressFor(spillSlotsOffset + stackPointerSpillSlot)); } else jit.storePtr(CCallHelpers::stackPointerRegister, CCallHelpers::addressFor(spillSlotsOffset + stackPointerSpillSlot)); unsigned extraStack = sizeof(CallerFrameAndPC) + WTF::roundUpToMultipleOf(stackAlignmentBytes(), 5 * sizeof(void*)); if (forwarding) { CCallHelpers::JumpList slowCase; computeUsedStack(scratchGPR2, 0); emitSetupVarargsFrameFastCase(jit, scratchGPR2, scratchGPR1, scratchGPR2, scratchGPR3, m_node->child2()->origin.semantic.inlineCallFrame, data->firstVarArgOffset, slowCase); CCallHelpers::Jump done = jit.jump(); slowCase.link(&jit); jit.subPtr(CCallHelpers::TrustedImm32(extraStack), CCallHelpers::stackPointerRegister); jit.setupArgumentsExecState(); callWithExceptionCheck(bitwise_cast<void*>(operationThrowStackOverflowForVarargs)); jit.abortWithReason(DFGVarargsThrowingPathDidNotThrow); done.link(&jit); jit.move(calleeGPR, GPRInfo::regT0); } else { // Gotta spill the callee, arguments, and this because we will need them later and we will have some // calls that clobber them. jit.store64(calleeGPR, CCallHelpers::addressFor(spillSlotsOffset + calleeSpillSlot)); jit.store64(argumentsGPR, CCallHelpers::addressFor(spillSlotsOffset + argumentsSpillSlot)); jit.store64(thisGPR, CCallHelpers::addressFor(spillSlotsOffset + thisSpillSlot)); computeUsedStack(scratchGPR1, 0); jit.subPtr(CCallHelpers::TrustedImm32(extraStack), CCallHelpers::stackPointerRegister); jit.setupArgumentsWithExecState(argumentsGPR, scratchGPR1, CCallHelpers::TrustedImm32(data->firstVarArgOffset)); callWithExceptionCheck(bitwise_cast<void*>(operationSizeFrameForVarargs)); jit.move(GPRInfo::returnValueGPR, scratchGPR1); computeUsedStack(scratchGPR2, extraStack); jit.load64(CCallHelpers::addressFor(spillSlotsOffset + argumentsSpillSlot), argumentsGPR); emitSetVarargsFrame(jit, scratchGPR1, false, scratchGPR2, scratchGPR2); jit.addPtr(CCallHelpers::TrustedImm32(-extraStack), scratchGPR2, CCallHelpers::stackPointerRegister); jit.setupArgumentsWithExecState(scratchGPR2, argumentsGPR, CCallHelpers::TrustedImm32(data->firstVarArgOffset), scratchGPR1); callWithExceptionCheck(bitwise_cast<void*>(operationSetupVarargsFrame)); jit.move(GPRInfo::returnValueGPR, scratchGPR2); jit.load64(CCallHelpers::addressFor(spillSlotsOffset + thisSpillSlot), thisGPR); jit.load64(CCallHelpers::addressFor(spillSlotsOffset + calleeSpillSlot), GPRInfo::regT0); } jit.addPtr(CCallHelpers::TrustedImm32(sizeof(CallerFrameAndPC)), scratchGPR2, CCallHelpers::stackPointerRegister); jit.store64(thisGPR, CCallHelpers::calleeArgumentSlot(0)); // Henceforth we make the call. The base FTL call machinery expects the callee in regT0 and for the // stack frame to already be set up, which it is. jit.store64(GPRInfo::regT0, CCallHelpers::calleeFrameSlot(JSStack::Callee)); m_callBase.emit(jit, state, osrExitFromGenericUnwindSpillSlots); // Undo the damage we've done. if (isARM64()) { GPRReg scratchGPRAtReturn = CCallHelpers::selectScratchGPR(GPRInfo::returnValueGPR); jit.loadPtr(CCallHelpers::addressFor(spillSlotsOffset + stackPointerSpillSlot), scratchGPRAtReturn); jit.move(scratchGPRAtReturn, CCallHelpers::stackPointerRegister); } else jit.loadPtr(CCallHelpers::addressFor(spillSlotsOffset + stackPointerSpillSlot), CCallHelpers::stackPointerRegister); }
void AccessCase::emitIntrinsicGetter(AccessGenerationState& state) { CCallHelpers& jit = *state.jit; JSValueRegs valueRegs = state.valueRegs; GPRReg baseGPR = state.baseGPR; GPRReg valueGPR = valueRegs.payloadGPR(); switch (intrinsic()) { case TypedArrayLengthIntrinsic: { jit.load32(MacroAssembler::Address(state.baseGPR, JSArrayBufferView::offsetOfLength()), valueGPR); jit.boxInt32(valueGPR, valueRegs, CCallHelpers::DoNotHaveTagRegisters); state.succeed(); return; } case TypedArrayByteLengthIntrinsic: { TypedArrayType type = structure()->classInfo()->typedArrayStorageType; jit.load32(MacroAssembler::Address(state.baseGPR, JSArrayBufferView::offsetOfLength()), valueGPR); if (elementSize(type) > 1) { // We can use a bitshift here since we TypedArrays cannot have byteLength that overflows an int32. jit.lshift32(valueGPR, Imm32(logElementSize(type)), valueGPR); } jit.boxInt32(valueGPR, valueRegs, CCallHelpers::DoNotHaveTagRegisters); state.succeed(); return; } case TypedArrayByteOffsetIntrinsic: { GPRReg scratchGPR = state.scratchGPR; CCallHelpers::Jump emptyByteOffset = jit.branch32( MacroAssembler::NotEqual, MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfMode()), TrustedImm32(WastefulTypedArray)); jit.loadPtr(MacroAssembler::Address(baseGPR, JSObject::butterflyOffset()), scratchGPR); jit.loadPtr(MacroAssembler::Address(baseGPR, JSArrayBufferView::offsetOfVector()), valueGPR); jit.loadPtr(MacroAssembler::Address(scratchGPR, Butterfly::offsetOfArrayBuffer()), scratchGPR); jit.loadPtr(MacroAssembler::Address(scratchGPR, ArrayBuffer::offsetOfData()), scratchGPR); jit.subPtr(scratchGPR, valueGPR); CCallHelpers::Jump done = jit.jump(); emptyByteOffset.link(&jit); jit.move(TrustedImmPtr(0), valueGPR); done.link(&jit); jit.boxInt32(valueGPR, valueRegs, CCallHelpers::DoNotHaveTagRegisters); state.succeed(); return; } default: break; } RELEASE_ASSERT_NOT_REACHED(); }