MacroAssemblerCodeRef stringLengthTrampolineGenerator(VM* vm) { JSInterfaceJIT jit; #if USE(JSVALUE64) // Check eax is a string JSInterfaceJIT::Jump failureCases1 = jit.emitJumpIfNotJSCell(JSInterfaceJIT::regT0); JSInterfaceJIT::Jump failureCases2 = jit.branchPtr( JSInterfaceJIT::NotEqual, JSInterfaceJIT::Address( JSInterfaceJIT::regT0, JSCell::structureOffset()), JSInterfaceJIT::TrustedImmPtr(vm->stringStructure.get())); // Checks out okay! - get the length from the Ustring. jit.load32( JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSString::offsetOfLength()), JSInterfaceJIT::regT0); JSInterfaceJIT::Jump failureCases3 = jit.branch32( JSInterfaceJIT::LessThan, JSInterfaceJIT::regT0, JSInterfaceJIT::TrustedImm32(0)); // regT0 contains a 64 bit value (is positive, is zero extended) so we don't need sign extend here. jit.emitFastArithIntToImmNoCheck(JSInterfaceJIT::regT0, JSInterfaceJIT::regT0); #else // USE(JSVALUE64) // regT0 holds payload, regT1 holds tag JSInterfaceJIT::Jump failureCases1 = jit.branch32( JSInterfaceJIT::NotEqual, JSInterfaceJIT::regT1, JSInterfaceJIT::TrustedImm32(JSValue::CellTag)); JSInterfaceJIT::Jump failureCases2 = jit.branchPtr( JSInterfaceJIT::NotEqual, JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSCell::structureOffset()), JSInterfaceJIT::TrustedImmPtr(vm->stringStructure.get())); // Checks out okay! - get the length from the Ustring. jit.load32( JSInterfaceJIT::Address(JSInterfaceJIT::regT0, JSString::offsetOfLength()), JSInterfaceJIT::regT2); JSInterfaceJIT::Jump failureCases3 = jit.branch32( JSInterfaceJIT::Above, JSInterfaceJIT::regT2, JSInterfaceJIT::TrustedImm32(INT_MAX)); jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::regT0); jit.move(JSInterfaceJIT::TrustedImm32(JSValue::Int32Tag), JSInterfaceJIT::regT1); #endif // USE(JSVALUE64) jit.ret(); JSInterfaceJIT::Call failureCases1Call = jit.makeTailRecursiveCall(failureCases1); JSInterfaceJIT::Call failureCases2Call = jit.makeTailRecursiveCall(failureCases2); JSInterfaceJIT::Call failureCases3Call = jit.makeTailRecursiveCall(failureCases3); LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID); patchBuffer.link(failureCases1Call, FunctionPtr(cti_op_get_by_id_string_fail)); patchBuffer.link(failureCases2Call, FunctionPtr(cti_op_get_by_id_string_fail)); patchBuffer.link(failureCases3Call, FunctionPtr(cti_op_get_by_id_string_fail)); return FINALIZE_CODE(patchBuffer, ("string length trampoline")); }
/*static*/ FunctionPtr Function::DeserializeNativeImpl(const std::vector<Variable>& inputs, const std::wstring& name, const Dictionary& dict) { static const vector<std::wstring> s_requiredDictionaryKeys = { userDefinedStateKey, udfModuleNameKey, udfFactoryMethodNameKey, opKey }; ValidateDictionary<PrimitiveFunction>(dict, s_requiredDictionaryKeys, s_nativeUDFTypeValue, s_serializationVersion); auto state = dict[userDefinedStateKey].Value<Dictionary>(); auto opName = dict[opKey].Value<wstring>(); auto moduleName = dict[udfModuleNameKey].Value<wstring>(); auto methodName = dict[udfFactoryMethodNameKey].Value<wstring>(); FunctionPtr udf = nullptr; auto callback = Function::GetUDFDeserializeCallback(opName); if (callback != nullptr) { udf = callback->operator()(inputs, name, state); } else { Microsoft::MSR::CNTK::Plugin plugin; auto factoryMethod = (UserFunctionFactoryMethodType)(plugin.Load(moduleName, Microsoft::MSR::CNTK::ToLegacyString(Microsoft::MSR::CNTK::ToUTF8(methodName)), /*isCNTKPlugin =*/ false)); udf = FunctionPtr(factoryMethod(inputs.data(), inputs.size(), &state, name.c_str())); } if (udf == nullptr) { RuntimeError("Unable to reconstruct the native UserFunction with op name '%S'", opName.c_str()); } s_deserializedUDFsRegistry[opName] = { moduleName, methodName }; return udf; }
FunctionPtr Findable::findFunction_downward( Scope* stable, const std::string& function_name ) { for( std::vector<SymbolPtr>::iterator I = stable->m_childs.begin(), B = stable->m_childs.end() ; I != B ; I ++ ) { if( (*I)->name() == function_name && (*I)->is(Symbol::T_SCOPE) ) { ScopePtr scope_symbol = DYNA_CAST( Scope, *I ); if( scope_symbol->getScopeType() == Scope::T_FUNCTION ) { FunctionPtr func_symbol = DYNA_CAST( Function, *I ); return func_symbol; } } if( (*I)->name() == "" && (*I)->is( Symbol::T_SCOPE) ) { ScopePtr pkg_symbol = DYNA_CAST( Scope, *I ); if( pkg_symbol->getScopeType() == Scope::T_PACKAGE ) { FunctionPtr found = findFunction_downward( pkg_symbol.get() , function_name ); if( found ) return found; } } } return FunctionPtr(); }
void registerFunction(const string& name, RetT(*f)(ArgT0, ArgT1)) { TypeSystem* ts = TypeSystem::instance(); IType *retT = CType2ScriptType<RetT>::get(ts); vector<IType*> argT = {CType2ScriptType<ArgT0>::get(ts), CType2ScriptType<ArgT1>::get(ts)}; IType *ftype = ts->getFunc(retT, argT); SymbolTableManager::instance()->global()->addSymbol(name, ftype); CodeManager::instance()->registerFunction(name, FunctionPtr(new CFunction2<RetT, ArgT0, ArgT1>(f))); }
FunctionPtr Findable::findFunction( Scope* stable, const std::string& function_name ) { FunctionPtr found = findFunction_downward(stable, function_name ); if( found ) return found; if( stable->m_inherit ) return findFunction( stable->m_inherit, function_name ); if( stable->m_parent ) return findFunction( stable->m_parent, function_name ); return FunctionPtr(); }
void registerVarLengFunction(const string& name, CVarlengFunction::FuncT f) { TypeSystem* ts = TypeSystem::instance(); IType *retT = CType2ScriptType<int>::get(ts); vector<IType*> argT = {CType2ScriptType<const char*>::get(ts)}; IType *ftype = ts->getFunc(retT, argT); dynamic_cast<FunctionType*>(ftype)->isVarLengOfArg = true; SymbolTableManager::instance()->global()->addSymbol(name, ftype); CodeManager::instance()->registerFunction(name, FunctionPtr(new CVarlengFunction(f))); }
FunctionPtr Findable::findConstructor( ScopePtr stable, const std::vector<std::string>& names ) { ScopePtr pkg_info = stable; for( int idx = 0 ; idx < names.size() - 2 ; idx ++ ) { pkg_info = findPackage( pkg_info, names[idx] ); if( pkg_info == NULL ) { std::cerr << " findFunction can't find pkg "<<names[idx]<<std::endl; return FunctionPtr(); } } ScopePtr class_type = findClassType_downward( pkg_info.get(), names[names.size()-1] ); FunctionPtr found = findFunction_downward( class_type.get(), names[names.size()-1] ); if( found ) return found; return FunctionPtr(); }
void CompilerCore::register_function_header (VertexAdaptor <meta_op_function> function_header, DataStream &os) { const string &function_name = function_header->name().as <op_func_name>()->str_val; FunctionSetPtr function_set = get_function_set (fs_function, function_name, true); kphp_assert (function_set.not_null()); { AutoLocker <FunctionSetPtr> locker (function_set); kphp_error_return ( function_set->header.is_null(), dl_pstr ("Several headers for one function [%s] are found", function_name.c_str()) ); function_set->header = function_header; } require_function_set (function_set, FunctionPtr(), os); }
void JIT::privateCompileClosureCall(CallLinkInfo* callLinkInfo, CodeBlock* calleeCodeBlock, Structure* expectedStructure, ExecutableBase* expectedExecutable, MacroAssemblerCodePtr codePtr) { JumpList slowCases; slowCases.append(branch32(NotEqual, regT1, TrustedImm32(JSValue::CellTag))); slowCases.append(branchPtr(NotEqual, Address(regT0, JSCell::structureOffset()), TrustedImmPtr(expectedStructure))); slowCases.append(branchPtr(NotEqual, Address(regT0, JSFunction::offsetOfExecutable()), TrustedImmPtr(expectedExecutable))); loadPtr(Address(regT0, JSFunction::offsetOfScopeChain()), regT1); emitPutCellToCallFrameHeader(regT1, JSStack::ScopeChain); Call call = nearCall(); Jump done = jump(); slowCases.link(this); move(TrustedImmPtr(callLinkInfo->callReturnLocation.executableAddress()), regT2); restoreReturnAddressBeforeReturn(regT2); Jump slow = jump(); LinkBuffer patchBuffer(*m_vm, this, m_codeBlock); patchBuffer.link(call, FunctionPtr(codePtr.executableAddress())); patchBuffer.link(done, callLinkInfo->hotPathOther.labelAtOffset(0)); patchBuffer.link(slow, CodeLocationLabel(m_vm->getCTIStub(virtualCallThunkGenerator).code())); RefPtr<ClosureCallStubRoutine> stubRoutine = adoptRef(new ClosureCallStubRoutine( FINALIZE_CODE( patchBuffer, ("Baseline closure call stub for %s, return point %p, target %p (%s)", toCString(*m_codeBlock).data(), callLinkInfo->hotPathOther.labelAtOffset(0).executableAddress(), codePtr.executableAddress(), toCString(pointerDump(calleeCodeBlock)).data())), *m_vm, m_codeBlock->ownerExecutable(), expectedStructure, expectedExecutable, callLinkInfo->codeOrigin)); RepatchBuffer repatchBuffer(m_codeBlock); repatchBuffer.replaceWithJump( RepatchBuffer::startOfBranchPtrWithPatchOnRegister(callLinkInfo->hotPathBegin), CodeLocationLabel(stubRoutine->code().code())); repatchBuffer.relink(callLinkInfo->callReturnLocation, m_vm->getCTIStub(virtualCallThunkGenerator).code()); callLinkInfo->stub = stubRoutine.release(); }
ExceptionHandler genericUnwind(VM* vm, ExecState* callFrame, JSValue exceptionValue) { RELEASE_ASSERT(exceptionValue); HandlerInfo* handler = vm->interpreter->unwind(callFrame, exceptionValue); // This may update callFrame. void* catchRoutine; Instruction* catchPCForInterpreter = 0; if (handler) { catchPCForInterpreter = &callFrame->codeBlock()->instructions()[handler->target]; catchRoutine = ExecutableBase::catchRoutineFor(handler, catchPCForInterpreter); } else catchRoutine = FunctionPtr(LLInt::getCodePtr(ctiOpThrowNotCaught)).value(); vm->callFrameForThrow = callFrame; vm->targetMachinePCForThrow = catchRoutine; vm->targetInterpreterPCForThrow = catchPCForInterpreter; RELEASE_ASSERT(catchRoutine); ExceptionHandler exceptionHandler = { callFrame, catchRoutine}; return exceptionHandler; }
void JITCompiler::linkFunction() { // === Link === OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(*m_vm, this, m_codeBlock, JITCompilationCanFail)); if (linkBuffer->didFailToAllocate()) { m_graph.m_plan.finalizer = adoptPtr(new FailedFinalizer(m_graph.m_plan)); return; } link(*linkBuffer); m_speculative->linkOSREntries(*linkBuffer); m_jitCode->shrinkToFit(); codeBlock()->shrinkToFit(CodeBlock::LateShrink); linkBuffer->link(m_callArityFixup, FunctionPtr((m_vm->getCTIStub(arityFixup)).code().executableAddress())); disassemble(*linkBuffer); m_graph.m_plan.finalizer = adoptPtr(new JITFinalizer( m_graph.m_plan, m_jitCode.release(), linkBuffer.release(), m_arityCheck)); }
void JITCompiler::linkFunction() { // === Link === OwnPtr<LinkBuffer> linkBuffer = adoptPtr(new LinkBuffer(*m_vm, this, m_codeBlock, JITCompilationCanFail)); if (linkBuffer->didFailToAllocate()) { m_graph.m_plan.finalizer = adoptPtr(new FailedFinalizer(m_graph.m_plan)); return; } link(*linkBuffer); m_speculative->linkOSREntries(*linkBuffer); m_jitCode->shrinkToFit(); codeBlock()->shrinkToFit(CodeBlock::LateShrink); // FIXME: switch the stack check & arity check over to DFGOpertaion style calls, not JIT stubs. linkBuffer->link(m_callStackCheck, cti_stack_check); linkBuffer->link(m_callArityCheck, m_codeBlock->m_isConstructor ? cti_op_construct_arityCheck : cti_op_call_arityCheck); linkBuffer->link(m_callArityFixup, FunctionPtr((m_vm->getCTIStub(arityFixup)).code().executableAddress())); disassemble(*linkBuffer); m_graph.m_plan.finalizer = adoptPtr(new JITFinalizer( m_graph.m_plan, m_jitCode.release(), linkBuffer.release(), m_arityCheck)); }
ExceptionHandler genericThrow(JSGlobalData* globalData, ExecState* callFrame, JSValue exceptionValue, unsigned vPCIndex) { ASSERT(exceptionValue); globalData->exception = JSValue(); HandlerInfo* handler = globalData->interpreter->throwException(callFrame, exceptionValue, vPCIndex); // This may update callFrame & exceptionValue! globalData->exception = exceptionValue; void* catchRoutine; Instruction* catchPCForInterpreter = 0; if (handler) { catchPCForInterpreter = &callFrame->codeBlock()->instructions()[handler->target]; catchRoutine = ExecutableBase::catchRoutineFor(handler, catchPCForInterpreter); } else catchRoutine = FunctionPtr(LLInt::getCodePtr(ctiOpThrowNotCaught)).value(); globalData->callFrameForThrow = callFrame; globalData->targetMachinePCForThrow = catchRoutine; globalData->targetInterpreterPCForThrow = catchPCForInterpreter; ASSERT(catchRoutine); ExceptionHandler exceptionHandler = { catchRoutine, callFrame }; return exceptionHandler; }
void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. m_jitCode->common.frameRegisterCount = m_graph.frameRegisterCount(); m_jitCode->common.requiredRegisterCountForExit = m_graph.requiredRegisterCountForExit(); if (!m_graph.m_inlineCallFrames->isEmpty()) m_jitCode->common.inlineCallFrames = m_graph.m_inlineCallFrames.release(); m_jitCode->common.machineCaptureStart = m_graph.m_machineCaptureStart; m_jitCode->common.slowArguments = std::move(m_graph.m_slowArguments); BitVector usedJumpTables; for (unsigned i = m_graph.m_switchData.size(); i--;) { SwitchData& data = m_graph.m_switchData[i]; if (!data.didUseJumpTable) continue; if (data.kind == SwitchString) continue; RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar); usedJumpTables.set(data.switchTableIndex); SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]); table.ctiOffsets.grow(table.branchOffsets.size()); for (unsigned j = table.ctiOffsets.size(); j--;) table.ctiOffsets[j] = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; table.ctiOffsets[myCase.value.switchLookupValue() - table.min] = linkBuffer.locationOf(m_blockHeads[myCase.target->index]); } } for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) { if (usedJumpTables.get(i)) continue; m_codeBlock->switchJumpTable(i).clear(); } // NOTE: we cannot clear string switch tables because (1) we're running concurrently // and we cannot deref StringImpl's and (2) it would be weird to deref those // StringImpl's since we refer to them. for (unsigned i = m_graph.m_switchData.size(); i--;) { SwitchData& data = m_graph.m_switchData[i]; if (!data.didUseJumpTable) continue; if (data.kind != SwitchString) continue; StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]); StringJumpTable::StringOffsetTable::iterator iter; StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end(); for (iter = table.offsetTable.begin(); iter != end; ++iter) iter->value.ctiOffset = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; iter = table.offsetTable.find(myCase.value.stringImpl()); RELEASE_ASSERT(iter != end); iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target->index]); } } // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); for (unsigned i = m_getByIds.size(); i--;) m_getByIds[i].finalize(linkBuffer); for (unsigned i = m_putByIds.size(); i--;) m_putByIds[i].finalize(linkBuffer); for (unsigned i = 0; i < m_ins.size(); ++i) { StructureStubInfo& info = *m_ins[i].m_stubInfo; CodeLocationLabel jump = linkBuffer.locationOf(m_ins[i].m_jump); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call()); info.hotPathBegin = jump; info.callReturnLocation = callReturnLocation; info.patch.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label())); } m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size()); for (unsigned i = 0; i < m_jsCalls.size(); ++i) { CallLinkInfo& info = m_codeBlock->callLinkInfo(i); info.callType = m_jsCalls[i].m_callType; info.isDFG = true; info.codeOrigin = m_jsCalls[i].m_codeOrigin; linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_vm->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress())); info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall); info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck); info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall); info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee); } MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) { OSRExit& exit = m_jitCode->osrExit[i]; OSRExitCompilationInfo& info = m_exitCompilationInfo[i]; linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target); exit.correctJump(linkBuffer); if (info.m_replacementSource.isSet()) { m_jitCode->common.jumpReplacements.append(JumpReplacement( linkBuffer.locationOf(info.m_replacementSource), linkBuffer.locationOf(info.m_replacementDestination))); } } if (m_graph.compilation()) { ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size()); for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) { Vector<Label>& labels = m_exitSiteLabels[i]; Vector<const void*> addresses; for (unsigned j = 0; j < labels.size(); ++j) addresses.append(linkBuffer.locationOf(labels[j]).executableAddress()); m_graph.compilation()->addOSRExitSite(addresses); } } else ASSERT(!m_exitSiteLabels.size()); m_jitCode->common.compilation = m_graph.compilation(); }
MacroAssemblerCodeRef virtualConstructGenerator(VM* vm) { return virtualForGenerator(vm, FunctionPtr(cti_op_construct_jitCompile), FunctionPtr(cti_op_construct_NotJSConstruct), "construct", CodeForConstruct); }
MacroAssemblerCodeRef virtualCallGenerator(VM* vm) { return virtualForGenerator(vm, FunctionPtr(cti_op_call_jitCompile), FunctionPtr(cti_op_call_NotJSFunction), "call", CodeForCall); }
MacroAssemblerCodeRef linkConstructGenerator(VM* vm) { return linkForGenerator(vm, FunctionPtr(cti_vm_lazyLinkConstruct), FunctionPtr(cti_op_construct_NotJSConstruct), "construct"); }
Result evaluation_helper(util::seq<S...>) const { // call the function with the unpacked tuple return FunctionPtr(std::get<S>(args)...); }
static void fixFunctionBasedOnStackMaps( State& state, CodeBlock* codeBlock, JITCode* jitCode, GeneratedFunction generatedFunction, StackMaps::RecordMap& recordMap) { Graph& graph = state.graph; VM& vm = graph.m_vm; StackMaps stackmaps = jitCode->stackmaps; int localsOffset = offsetOfStackRegion(recordMap, state.capturedStackmapID) + graph.m_nextMachineLocal; int varargsSpillSlotsOffset = offsetOfStackRegion(recordMap, state.varargsSpillSlotsStackmapID); for (unsigned i = graph.m_inlineVariableData.size(); i--;) { InlineCallFrame* inlineCallFrame = graph.m_inlineVariableData[i].inlineCallFrame; if (inlineCallFrame->argumentCountRegister.isValid()) inlineCallFrame->argumentCountRegister += localsOffset; for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) { inlineCallFrame->arguments[argument] = inlineCallFrame->arguments[argument].withLocalsOffset(localsOffset); } if (inlineCallFrame->isClosureCall) { inlineCallFrame->calleeRecovery = inlineCallFrame->calleeRecovery.withLocalsOffset(localsOffset); } if (graph.hasDebuggerEnabled()) codeBlock->setScopeRegister(codeBlock->scopeRegister() + localsOffset); } MacroAssembler::Label stackOverflowException; { CCallHelpers checkJIT(&vm, codeBlock); // At this point it's perfectly fair to just blow away all state and restore the // JS JIT view of the universe. checkJIT.copyCalleeSavesToVMCalleeSavesBuffer(); checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0); checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); MacroAssembler::Call callLookupExceptionHandler = checkJIT.call(); checkJIT.jumpToExceptionHandler(); stackOverflowException = checkJIT.label(); checkJIT.copyCalleeSavesToVMCalleeSavesBuffer(); checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0); checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); MacroAssembler::Call callLookupExceptionHandlerFromCallerFrame = checkJIT.call(); checkJIT.jumpToExceptionHandler(); auto linkBuffer = std::make_unique<LinkBuffer>( vm, checkJIT, codeBlock, JITCompilationCanFail); if (linkBuffer->didFailToAllocate()) { state.allocationFailed = true; return; } linkBuffer->link(callLookupExceptionHandler, FunctionPtr(lookupExceptionHandler)); linkBuffer->link(callLookupExceptionHandlerFromCallerFrame, FunctionPtr(lookupExceptionHandlerFromCallerFrame)); state.finalizer->handleExceptionsLinkBuffer = WTF::move(linkBuffer); } ExitThunkGenerator exitThunkGenerator(state); exitThunkGenerator.emitThunks(); if (exitThunkGenerator.didThings()) { RELEASE_ASSERT(state.finalizer->osrExit.size()); auto linkBuffer = std::make_unique<LinkBuffer>( vm, exitThunkGenerator, codeBlock, JITCompilationCanFail); if (linkBuffer->didFailToAllocate()) { state.allocationFailed = true; return; } RELEASE_ASSERT(state.finalizer->osrExit.size() == state.jitCode->osrExit.size()); for (unsigned i = 0; i < state.jitCode->osrExit.size(); ++i) { OSRExitCompilationInfo& info = state.finalizer->osrExit[i]; OSRExit& exit = jitCode->osrExit[i]; if (verboseCompilationEnabled()) dataLog("Handling OSR stackmap #", exit.m_stackmapID, " for ", exit.m_codeOrigin, "\n"); auto iter = recordMap.find(exit.m_stackmapID); if (iter == recordMap.end()) { // It was optimized out. continue; } info.m_thunkAddress = linkBuffer->locationOf(info.m_thunkLabel); exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump); for (unsigned j = exit.m_values.size(); j--;) exit.m_values[j] = exit.m_values[j].withLocalsOffset(localsOffset); for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) materialization->accountForLocalsOffset(localsOffset); if (verboseCompilationEnabled()) { DumpContext context; dataLog(" Exit values: ", inContext(exit.m_values, &context), "\n"); if (!exit.m_materializations.isEmpty()) { dataLog(" Materializations: \n"); for (ExitTimeObjectMaterialization* materialization : exit.m_materializations) dataLog(" Materialize(", pointerDump(materialization), ")\n"); } } } state.finalizer->exitThunksLinkBuffer = WTF::move(linkBuffer); } if (!state.getByIds.isEmpty() || !state.putByIds.isEmpty() || !state.checkIns.isEmpty()) { CCallHelpers slowPathJIT(&vm, codeBlock); CCallHelpers::JumpList exceptionTarget; for (unsigned i = state.getByIds.size(); i--;) { GetByIdDescriptor& getById = state.getByIds[i]; if (verboseCompilationEnabled()) dataLog("Handling GetById stackmap #", getById.stackmapID(), "\n"); auto iter = recordMap.find(getById.stackmapID()); if (iter == recordMap.end()) { // It was optimized out. continue; } CodeOrigin codeOrigin = state.jitCode->common.codeOrigins[getById.callSiteIndex().bits()]; for (unsigned i = 0; i < iter->value.size(); ++i) { StackMaps::Record& record = iter->value[i]; RegisterSet usedRegisters = usedRegistersFor(record); GPRReg result = record.locations[0].directGPR(); GPRReg base = record.locations[1].directGPR(); JITGetByIdGenerator gen( codeBlock, codeOrigin, getById.callSiteIndex(), usedRegisters, JSValueRegs(base), JSValueRegs(result), NeedToSpill); MacroAssembler::Label begin = slowPathJIT.label(); MacroAssembler::Call call = callOperation( state, usedRegisters, slowPathJIT, codeOrigin, &exceptionTarget, operationGetByIdOptimize, result, gen.stubInfo(), base, getById.uid()); gen.reportSlowPathCall(begin, call); getById.m_slowPathDone.append(slowPathJIT.jump()); getById.m_generators.append(gen); } } for (unsigned i = state.putByIds.size(); i--;) { PutByIdDescriptor& putById = state.putByIds[i]; if (verboseCompilationEnabled()) dataLog("Handling PutById stackmap #", putById.stackmapID(), "\n"); auto iter = recordMap.find(putById.stackmapID()); if (iter == recordMap.end()) { // It was optimized out. continue; } CodeOrigin codeOrigin = state.jitCode->common.codeOrigins[putById.callSiteIndex().bits()]; for (unsigned i = 0; i < iter->value.size(); ++i) { StackMaps::Record& record = iter->value[i]; RegisterSet usedRegisters = usedRegistersFor(record); GPRReg base = record.locations[0].directGPR(); GPRReg value = record.locations[1].directGPR(); JITPutByIdGenerator gen( codeBlock, codeOrigin, putById.callSiteIndex(), usedRegisters, JSValueRegs(base), JSValueRegs(value), GPRInfo::patchpointScratchRegister, NeedToSpill, putById.ecmaMode(), putById.putKind()); MacroAssembler::Label begin = slowPathJIT.label(); MacroAssembler::Call call = callOperation( state, usedRegisters, slowPathJIT, codeOrigin, &exceptionTarget, gen.slowPathFunction(), gen.stubInfo(), value, base, putById.uid()); gen.reportSlowPathCall(begin, call); putById.m_slowPathDone.append(slowPathJIT.jump()); putById.m_generators.append(gen); } } for (unsigned i = state.checkIns.size(); i--;) { CheckInDescriptor& checkIn = state.checkIns[i]; if (verboseCompilationEnabled()) dataLog("Handling checkIn stackmap #", checkIn.stackmapID(), "\n"); auto iter = recordMap.find(checkIn.stackmapID()); if (iter == recordMap.end()) { // It was optimized out. continue; } CodeOrigin codeOrigin = state.jitCode->common.codeOrigins[checkIn.callSiteIndex().bits()]; for (unsigned i = 0; i < iter->value.size(); ++i) { StackMaps::Record& record = iter->value[i]; RegisterSet usedRegisters = usedRegistersFor(record); GPRReg result = record.locations[0].directGPR(); GPRReg obj = record.locations[1].directGPR(); StructureStubInfo* stubInfo = codeBlock->addStubInfo(AccessType::In); stubInfo->codeOrigin = codeOrigin; stubInfo->callSiteIndex = checkIn.callSiteIndex(); stubInfo->patch.baseGPR = static_cast<int8_t>(obj); stubInfo->patch.valueGPR = static_cast<int8_t>(result); stubInfo->patch.usedRegisters = usedRegisters; stubInfo->patch.spillMode = NeedToSpill; MacroAssembler::Label begin = slowPathJIT.label(); MacroAssembler::Call slowCall = callOperation( state, usedRegisters, slowPathJIT, codeOrigin, &exceptionTarget, operationInOptimize, result, stubInfo, obj, checkIn.m_uid); checkIn.m_slowPathDone.append(slowPathJIT.jump()); checkIn.m_generators.append(CheckInGenerator(stubInfo, slowCall, begin)); } } exceptionTarget.link(&slowPathJIT); MacroAssembler::Jump exceptionJump = slowPathJIT.jump(); state.finalizer->sideCodeLinkBuffer = std::make_unique<LinkBuffer>(vm, slowPathJIT, codeBlock, JITCompilationCanFail); if (state.finalizer->sideCodeLinkBuffer->didFailToAllocate()) { state.allocationFailed = true; return; } state.finalizer->sideCodeLinkBuffer->link( exceptionJump, state.finalizer->handleExceptionsLinkBuffer->entrypoint()); for (unsigned i = state.getByIds.size(); i--;) { generateICFastPath( state, codeBlock, generatedFunction, recordMap, state.getByIds[i], sizeOfGetById()); } for (unsigned i = state.putByIds.size(); i--;) { generateICFastPath( state, codeBlock, generatedFunction, recordMap, state.putByIds[i], sizeOfPutById()); } for (unsigned i = state.checkIns.size(); i--;) { generateCheckInICFastPath( state, codeBlock, generatedFunction, recordMap, state.checkIns[i], sizeOfIn()); } } adjustCallICsForStackmaps(state.jsCalls, recordMap); for (unsigned i = state.jsCalls.size(); i--;) { JSCall& call = state.jsCalls[i]; CCallHelpers fastPathJIT(&vm, codeBlock); call.emit(fastPathJIT, state.jitCode->stackmaps.stackSizeForLocals()); char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset; generateInlineIfPossibleOutOfLineIfNot(state, vm, codeBlock, fastPathJIT, startOfIC, sizeOfCall(), "JSCall inline cache", [&] (LinkBuffer& linkBuffer, CCallHelpers&, bool) { call.link(vm, linkBuffer); }); } adjustCallICsForStackmaps(state.jsCallVarargses, recordMap); for (unsigned i = state.jsCallVarargses.size(); i--;) { JSCallVarargs& call = state.jsCallVarargses[i]; CCallHelpers fastPathJIT(&vm, codeBlock); call.emit(fastPathJIT, varargsSpillSlotsOffset); char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset; size_t sizeOfIC = sizeOfICFor(call.node()); generateInlineIfPossibleOutOfLineIfNot(state, vm, codeBlock, fastPathJIT, startOfIC, sizeOfIC, "varargs call inline cache", [&] (LinkBuffer& linkBuffer, CCallHelpers&, bool) { call.link(vm, linkBuffer, state.finalizer->handleExceptionsLinkBuffer->entrypoint()); }); } adjustCallICsForStackmaps(state.jsTailCalls, recordMap); for (unsigned i = state.jsTailCalls.size(); i--;) { JSTailCall& call = state.jsTailCalls[i]; CCallHelpers fastPathJIT(&vm, codeBlock); call.emit(*state.jitCode.get(), fastPathJIT); char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset; size_t sizeOfIC = call.estimatedSize(); generateInlineIfPossibleOutOfLineIfNot(state, vm, codeBlock, fastPathJIT, startOfIC, sizeOfIC, "tail call inline cache", [&] (LinkBuffer& linkBuffer, CCallHelpers&, bool) { call.link(vm, linkBuffer); }); } auto iter = recordMap.find(state.handleStackOverflowExceptionStackmapID); // It's sort of remotely possible that we won't have an in-band exception handling // path, for some kinds of functions. if (iter != recordMap.end()) { for (unsigned i = iter->value.size(); i--;) { StackMaps::Record& record = iter->value[i]; CodeLocationLabel source = CodeLocationLabel( bitwise_cast<char*>(generatedFunction) + record.instructionOffset); RELEASE_ASSERT(stackOverflowException.isSet()); MacroAssembler::replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->locationOf(stackOverflowException)); } } iter = recordMap.find(state.handleExceptionStackmapID); // It's sort of remotely possible that we won't have an in-band exception handling // path, for some kinds of functions. if (iter != recordMap.end()) { for (unsigned i = iter->value.size(); i--;) { StackMaps::Record& record = iter->value[i]; CodeLocationLabel source = CodeLocationLabel( bitwise_cast<char*>(generatedFunction) + record.instructionOffset); MacroAssembler::replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->entrypoint()); } } for (unsigned exitIndex = 0; exitIndex < jitCode->osrExit.size(); ++exitIndex) { OSRExitCompilationInfo& info = state.finalizer->osrExit[exitIndex]; OSRExit& exit = jitCode->osrExit[exitIndex]; iter = recordMap.find(exit.m_stackmapID); Vector<const void*> codeAddresses; if (iter != recordMap.end()) { for (unsigned i = iter->value.size(); i--;) { StackMaps::Record& record = iter->value[i]; CodeLocationLabel source = CodeLocationLabel( bitwise_cast<char*>(generatedFunction) + record.instructionOffset); codeAddresses.append(bitwise_cast<char*>(generatedFunction) + record.instructionOffset + MacroAssembler::maxJumpReplacementSize()); if (info.m_isInvalidationPoint) jitCode->common.jumpReplacements.append(JumpReplacement(source, info.m_thunkAddress)); else MacroAssembler::replaceWithJump(source, info.m_thunkAddress); } } if (graph.compilation()) graph.compilation()->addOSRExitSite(codeAddresses); } }
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall) { int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind); JSInterfaceJIT jit(vm); if (entryType == EnterViaCall) jit.emitFunctionPrologue(); #if USE(JSVALUE64) else if (entryType == EnterViaJump) { // We're coming from a specialized thunk that has saved the prior tag registers' contents. // Restore them now. #if CPU(ARM64) jit.popPair(JSInterfaceJIT::tagTypeNumberRegister, JSInterfaceJIT::tagMaskRegister); #else jit.pop(JSInterfaceJIT::tagMaskRegister); jit.pop(JSInterfaceJIT::tagTypeNumberRegister); #endif } #endif jit.emitPutToCallFrameHeader(0, JSStack::CodeBlock); jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame); #if CPU(X86) // Calling convention: f(ecx, edx, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx); jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue. // call the function jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1); jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1); jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); #elif CPU(X86_64) #if !OS(WINDOWS) // Calling convention: f(edi, esi, edx, ecx, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi); jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9); jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction)); #else // Calling convention: f(ecx, edx, r8, r9, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx); // Leave space for the callee parameter home addresses. // At this point the stack is aligned to 16 bytes, but if this changes at some point, we need to emit code to align it. jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx); jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9); jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction)); jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); #endif #elif CPU(ARM64) COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0); COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1); COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1); jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2); jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction)); #elif CPU(ARM) || CPU(SH4) || CPU(MIPS) #if CPU(MIPS) // Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments. jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); #endif // Calling convention is f(argumentGPR0, argumentGPR1, ...). // Host function signature is f(ExecState*). jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1); jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2); jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction)); #if CPU(MIPS) // Restore stack space jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); #endif #else #error "JIT not supported on this platform." UNUSED_PARAM(executableOffsetToFunction); abortWithReason(TGNotSupported); #endif // Check for an exception #if USE(JSVALUE64) jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2); JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2); #else JSInterfaceJIT::Jump exceptionHandler = jit.branch32( JSInterfaceJIT::NotEqual, JSInterfaceJIT::AbsoluteAddress(vm->addressOfException()), JSInterfaceJIT::TrustedImm32(0)); #endif jit.emitFunctionEpilogue(); // Return. jit.ret(); // Handle an exception exceptionHandler.link(&jit); jit.copyCalleeSavesToVMCalleeSavesBuffer(); jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame); #if CPU(X86) && USE(JSVALUE32_64) jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister); jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT0); jit.push(JSInterfaceJIT::regT0); #else #if OS(WINDOWS) // Allocate space on stack for the 4 parameter registers. jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); #endif jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0); #endif jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3); jit.call(JSInterfaceJIT::regT3); #if CPU(X86) && USE(JSVALUE32_64) jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); #elif OS(WINDOWS) jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); #endif jit.jumpToExceptionHandler(); LinkBuffer patchBuffer(*vm, jit, GLOBAL_THUNK_ID); return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJump ? "Tail " : "", toCString(kind).data())); }
// We are creating a bunch of threads that touch the main thread's stack. This will make ASAN unhappy. // The reason this is OK is that we guarantee that the main thread doesn't continue until all threads // that could touch its stack are done executing. SUPPRESS_ASAN void Plan::run() { if (!parseAndValidateModule()) return; auto tryReserveCapacity = [this] (auto& vector, size_t size, const char* what) { if (UNLIKELY(!vector.tryReserveCapacity(size))) { StringBuilder builder; builder.appendLiteral("Failed allocating enough space for "); builder.appendNumber(size); builder.append(what); m_errorMessage = builder.toString(); return false; } return true; }; if (!tryReserveCapacity(m_wasmExitStubs, m_moduleInformation->importFunctionSignatureIndices.size(), " WebAssembly to JavaScript stubs") || !tryReserveCapacity(m_unlinkedWasmToWasmCalls, m_functionLocationInBinary.size(), " unlinked WebAssembly to WebAssembly calls") || !tryReserveCapacity(m_wasmInternalFunctions, m_functionLocationInBinary.size(), " WebAssembly functions") || !tryReserveCapacity(m_compilationContexts, m_functionLocationInBinary.size(), " compilation contexts")) return; m_unlinkedWasmToWasmCalls.resize(m_functionLocationInBinary.size()); m_wasmInternalFunctions.resize(m_functionLocationInBinary.size()); m_compilationContexts.resize(m_functionLocationInBinary.size()); for (unsigned importIndex = 0; importIndex < m_moduleInformation->imports.size(); ++importIndex) { Import* import = &m_moduleInformation->imports[importIndex]; if (import->kind != ExternalKind::Function) continue; unsigned importFunctionIndex = m_wasmExitStubs.size(); if (verbose) dataLogLn("Processing import function number ", importFunctionIndex, ": ", import->module, ": ", import->field); SignatureIndex signatureIndex = m_moduleInformation->importFunctionSignatureIndices.at(import->kindIndex); m_wasmExitStubs.uncheckedAppend(exitStubGenerator(m_vm, m_callLinkInfos, signatureIndex, importFunctionIndex)); } m_currentIndex = 0; auto doWork = [this] { while (true) { uint32_t functionIndex; { auto locker = holdLock(m_lock); if (m_currentIndex >= m_functionLocationInBinary.size()) return; functionIndex = m_currentIndex; ++m_currentIndex; } const uint8_t* functionStart = m_source + m_functionLocationInBinary[functionIndex].start; size_t functionLength = m_functionLocationInBinary[functionIndex].end - m_functionLocationInBinary[functionIndex].start; ASSERT(functionLength <= m_sourceLength); SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex]; const Signature* signature = SignatureInformation::get(m_vm, signatureIndex); unsigned functionIndexSpace = m_wasmExitStubs.size() + functionIndex; ASSERT_UNUSED(functionIndexSpace, m_moduleInformation->signatureIndexFromFunctionIndexSpace(functionIndexSpace) == signatureIndex); ASSERT(validateFunction(m_vm, functionStart, functionLength, signature, *m_moduleInformation, m_moduleSignatureIndicesToUniquedSignatureIndices)); m_unlinkedWasmToWasmCalls[functionIndex] = Vector<UnlinkedWasmToWasmCall>(); auto parseAndCompileResult = parseAndCompile(*m_vm, m_compilationContexts[functionIndex], functionStart, functionLength, signature, m_unlinkedWasmToWasmCalls[functionIndex], *m_moduleInformation, m_moduleSignatureIndicesToUniquedSignatureIndices); if (UNLIKELY(!parseAndCompileResult)) { auto locker = holdLock(m_lock); if (!m_errorMessage) { // Multiple compiles could fail simultaneously. We arbitrarily choose the first. m_errorMessage = makeString(parseAndCompileResult.error(), ", in function at index ", String::number(functionIndex)); // FIXME make this an Expected. } m_currentIndex = m_functionLocationInBinary.size(); // We will terminate on the next execution. continue; } m_wasmInternalFunctions[functionIndex] = WTFMove(*parseAndCompileResult); } }; MonotonicTime startTime; if (verbose || Options::reportCompileTimes()) startTime = MonotonicTime::now(); uint32_t threadCount = Options::useConcurrentJIT() ? WTF::numberOfProcessorCores() : 1; uint32_t numWorkerThreads = threadCount - 1; Vector<ThreadIdentifier> threads; threads.reserveCapacity(numWorkerThreads); for (uint32_t i = 0; i < numWorkerThreads; i++) threads.uncheckedAppend(createThread("jsc.wasm-b3-compilation.thread", doWork)); doWork(); // Let the main thread do some work too. for (uint32_t i = 0; i < numWorkerThreads; i++) waitForThreadCompletion(threads[i]); for (uint32_t functionIndex = 0; functionIndex < m_functionLocationInBinary.size(); functionIndex++) { { CompilationContext& context = m_compilationContexts[functionIndex]; SignatureIndex signatureIndex = m_moduleInformation->internalFunctionSignatureIndices[functionIndex]; String signatureDescription = SignatureInformation::get(m_vm, signatureIndex)->toString(); { LinkBuffer linkBuffer(*m_vm, *context.wasmEntrypointJIT, nullptr); m_wasmInternalFunctions[functionIndex]->wasmEntrypoint.compilation = std::make_unique<B3::Compilation>(FINALIZE_CODE(linkBuffer, ("WebAssembly function[%i] %s", functionIndex, signatureDescription.ascii().data())), WTFMove(context.wasmEntrypointByproducts)); } { LinkBuffer linkBuffer(*m_vm, *context.jsEntrypointJIT, nullptr); linkBuffer.link(context.jsEntrypointToWasmEntrypointCall, FunctionPtr(m_wasmInternalFunctions[functionIndex]->wasmEntrypoint.compilation->code().executableAddress())); m_wasmInternalFunctions[functionIndex]->jsToWasmEntrypoint.compilation = std::make_unique<B3::Compilation>(FINALIZE_CODE(linkBuffer, ("JavaScript->WebAssembly entrypoint[%i] %s", functionIndex, signatureDescription.ascii().data())), WTFMove(context.jsEntrypointByproducts)); } } } if (verbose || Options::reportCompileTimes()) { dataLogLn("Took ", (MonotonicTime::now() - startTime).microseconds(), " us to compile and link the module"); } // Patch the call sites for each WebAssembly function. for (auto& unlinked : m_unlinkedWasmToWasmCalls) { for (auto& call : unlinked) { void* executableAddress; if (m_moduleInformation->isImportedFunctionFromFunctionIndexSpace(call.functionIndex)) { // FIXME imports could have been linked in B3, instead of generating a patchpoint. This condition should be replaced by a RELEASE_ASSERT. https://bugs.webkit.org/show_bug.cgi?id=166462 executableAddress = call.target == UnlinkedWasmToWasmCall::Target::ToJs ? m_wasmExitStubs.at(call.functionIndex).wasmToJs.code().executableAddress() : m_wasmExitStubs.at(call.functionIndex).wasmToWasm.code().executableAddress(); } else { ASSERT(call.target != UnlinkedWasmToWasmCall::Target::ToJs); executableAddress = m_wasmInternalFunctions.at(call.functionIndex - m_wasmExitStubs.size())->wasmEntrypoint.compilation->code().executableAddress(); } MacroAssembler::repatchCall(call.callLocation, CodeLocationLabel(executableAddress)); } } m_failed = false; }
void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. #if DFG_ENABLE(DEBUG_VERBOSE) dataLogF("JIT code for %p start at [%p, %p). Size = %zu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize()); #endif if (!m_graph.m_inlineCallFrames->isEmpty()) { m_graph.m_inlineCallFrames->shrinkToFit(); m_jitCode->common.inlineCallFrames = m_graph.m_inlineCallFrames.release(); } BitVector usedJumpTables; for (unsigned i = m_graph.m_switchData.size(); i--;) { SwitchData& data = m_graph.m_switchData[i]; if (!data.didUseJumpTable) continue; if (data.kind == SwitchString) continue; RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar); usedJumpTables.set(data.switchTableIndex); SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]); table.ctiOffsets.grow(table.branchOffsets.size()); for (unsigned j = table.ctiOffsets.size(); j--;) table.ctiOffsets[j] = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; table.ctiOffsets[myCase.value.switchLookupValue() - table.min] = linkBuffer.locationOf(m_blockHeads[myCase.target->index]); } } for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) { if (usedJumpTables.get(i)) continue; m_codeBlock->switchJumpTable(i).clear(); } // NOTE: we cannot clear string switch tables because (1) we're running concurrently // and we cannot deref StringImpl's and (2) it would be weird to deref those // StringImpl's since we refer to them. for (unsigned i = m_graph.m_switchData.size(); i--;) { SwitchData& data = m_graph.m_switchData[i]; if (!data.didUseJumpTable) continue; if (data.kind != SwitchString) continue; StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]); StringJumpTable::StringOffsetTable::iterator iter; StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end(); for (iter = table.offsetTable.begin(); iter != end; ++iter) iter->value.ctiOffset = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; iter = table.offsetTable.find(myCase.value.stringImpl()); RELEASE_ASSERT(iter != end); iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target->index]); } } // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size() + m_ins.size()); for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) { StructureStubInfo& info = m_codeBlock->structureStubInfo(i); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->call()); info.codeOrigin = m_propertyAccesses[i].m_codeOrigin; info.callReturnLocation = callReturnLocation; info.patch.dfg.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_structureImm), callReturnLocation); info.patch.dfg.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_structureCheck)); #if USE(JSVALUE64) info.patch.dfg.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_loadOrStore)); #else info.patch.dfg.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_tagLoadOrStore)); info.patch.dfg.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_payloadLoadOrStore)); #endif info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->label())); info.patch.dfg.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_done)); info.patch.dfg.deltaCallToStorageLoad = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_propertyStorageLoad)); info.patch.dfg.baseGPR = m_propertyAccesses[i].m_baseGPR; #if USE(JSVALUE64) info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #else info.patch.dfg.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR; info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #endif m_propertyAccesses[i].m_usedRegisters.copyInfo(info.patch.dfg.usedRegisters); info.patch.dfg.registersFlushed = m_propertyAccesses[i].m_registerMode == PropertyAccessRecord::RegistersFlushed; } for (unsigned i = 0; i < m_ins.size(); ++i) { StructureStubInfo& info = m_codeBlock->structureStubInfo(m_propertyAccesses.size() + i); CodeLocationLabel jump = linkBuffer.locationOf(m_ins[i].m_jump); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call()); info.codeOrigin = m_ins[i].m_codeOrigin; info.hotPathBegin = jump; info.callReturnLocation = callReturnLocation; info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label())); info.patch.dfg.baseGPR = m_ins[i].m_baseGPR; info.patch.dfg.valueGPR = m_ins[i].m_resultGPR; m_ins[i].m_usedRegisters.copyInfo(info.patch.dfg.usedRegisters); info.patch.dfg.registersFlushed = false; } m_codeBlock->sortStructureStubInfos(); m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size()); for (unsigned i = 0; i < m_jsCalls.size(); ++i) { CallLinkInfo& info = m_codeBlock->callLinkInfo(i); info.callType = m_jsCalls[i].m_callType; info.isDFG = true; info.codeOrigin = m_jsCalls[i].m_codeOrigin; linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_vm->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress())); info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall); info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck); info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall); info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee); } MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) { OSRExit& exit = m_jitCode->osrExit[i]; linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target); exit.correctJump(linkBuffer); if (exit.m_watchpointIndex != std::numeric_limits<unsigned>::max()) m_jitCode->watchpoints[exit.m_watchpointIndex].correctLabels(linkBuffer); } if (m_graph.compilation()) { ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size()); for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) { Vector<Label>& labels = m_exitSiteLabels[i]; Vector<const void*> addresses; for (unsigned j = 0; j < labels.size(); ++j) addresses.append(linkBuffer.locationOf(labels[j]).executableAddress()); m_graph.compilation()->addOSRExitSite(addresses); } } else ASSERT(!m_exitSiteLabels.size()); m_jitCode->common.compilation = m_graph.compilation(); }
ExceptionHandler uncaughtExceptionHandler() { void* catchRoutine = FunctionPtr(LLInt::getCodePtr(ctiOpThrowNotCaught)).value(); ExceptionHandler exceptionHandler = { 0, catchRoutine}; return exceptionHandler; }
void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. m_jitCode->common.frameRegisterCount = m_graph.frameRegisterCount(); m_jitCode->common.requiredRegisterCountForExit = m_graph.requiredRegisterCountForExit(); if (!m_graph.m_plan.inlineCallFrames->isEmpty()) m_jitCode->common.inlineCallFrames = m_graph.m_plan.inlineCallFrames; #if USE(JSVALUE32_64) m_jitCode->common.doubleConstants = WTFMove(m_graph.m_doubleConstants); #endif m_graph.registerFrozenValues(); BitVector usedJumpTables; for (Bag<SwitchData>::iterator iter = m_graph.m_switchData.begin(); !!iter; ++iter) { SwitchData& data = **iter; if (!data.didUseJumpTable) continue; if (data.kind == SwitchString) continue; RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar); usedJumpTables.set(data.switchTableIndex); SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough.block->index]); table.ctiOffsets.grow(table.branchOffsets.size()); for (unsigned j = table.ctiOffsets.size(); j--;) table.ctiOffsets[j] = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; table.ctiOffsets[myCase.value.switchLookupValue(data.kind) - table.min] = linkBuffer.locationOf(m_blockHeads[myCase.target.block->index]); } } for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) { if (usedJumpTables.get(i)) continue; m_codeBlock->switchJumpTable(i).clear(); } // NOTE: we cannot clear string switch tables because (1) we're running concurrently // and we cannot deref StringImpl's and (2) it would be weird to deref those // StringImpl's since we refer to them. for (Bag<SwitchData>::iterator switchDataIter = m_graph.m_switchData.begin(); !!switchDataIter; ++switchDataIter) { SwitchData& data = **switchDataIter; if (!data.didUseJumpTable) continue; if (data.kind != SwitchString) continue; StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough.block->index]); StringJumpTable::StringOffsetTable::iterator iter; StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end(); for (iter = table.offsetTable.begin(); iter != end; ++iter) iter->value.ctiOffset = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; iter = table.offsetTable.find(myCase.value.stringImpl()); RELEASE_ASSERT(iter != end); iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target.block->index]); } } // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); for (unsigned i = m_getByIds.size(); i--;) m_getByIds[i].finalize(linkBuffer); for (unsigned i = m_putByIds.size(); i--;) m_putByIds[i].finalize(linkBuffer); for (unsigned i = 0; i < m_ins.size(); ++i) { StructureStubInfo& info = *m_ins[i].m_stubInfo; CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call()); info.patch.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_done)); info.patch.deltaCallToJump = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_jump)); info.callReturnLocation = callReturnLocation; info.patch.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label())); } for (unsigned i = 0; i < m_jsCalls.size(); ++i) { JSCallRecord& record = m_jsCalls[i]; CallLinkInfo& info = *record.m_info; linkBuffer.link(record.m_slowCall, FunctionPtr(m_vm->getCTIStub(linkCallThunkGenerator).code().executableAddress())); info.setCallLocations(linkBuffer.locationOfNearCall(record.m_slowCall), linkBuffer.locationOf(record.m_targetToCheck), linkBuffer.locationOfNearCall(record.m_fastCall)); } MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) { OSRExit& exit = m_jitCode->osrExit[i]; OSRExitCompilationInfo& info = m_exitCompilationInfo[i]; linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target); exit.correctJump(linkBuffer); if (info.m_replacementSource.isSet()) { m_jitCode->common.jumpReplacements.append(JumpReplacement( linkBuffer.locationOf(info.m_replacementSource), linkBuffer.locationOf(info.m_replacementDestination))); } } if (m_graph.compilation()) { ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size()); for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) { Vector<Label>& labels = m_exitSiteLabels[i]; Vector<const void*> addresses; for (unsigned j = 0; j < labels.size(); ++j) addresses.append(linkBuffer.locationOf(labels[j]).executableAddress()); m_graph.compilation()->addOSRExitSite(addresses); } } else ASSERT(!m_exitSiteLabels.size()); m_jitCode->common.compilation = m_graph.compilation(); // Link new DFG exception handlers and remove baseline JIT handlers. m_codeBlock->clearExceptionHandlers(); for (unsigned i = 0; i < m_exceptionHandlerOSRExitCallSites.size(); i++) { OSRExitCompilationInfo& info = m_exceptionHandlerOSRExitCallSites[i].exitInfo; if (info.m_replacementDestination.isSet()) { // If this is is *not* set, it means that we already jumped to the OSR exit in pure generated control flow. // i.e, we explicitly emitted an exceptionCheck that we know will be caught in this machine frame. // If this *is set*, it means we will be landing at this code location from genericUnwind from an // exception thrown in a child call frame. CodeLocationLabel catchLabel = linkBuffer.locationOf(info.m_replacementDestination); HandlerInfo newExceptionHandler = m_exceptionHandlerOSRExitCallSites[i].baselineExceptionHandler; CallSiteIndex callSite = m_exceptionHandlerOSRExitCallSites[i].callSiteIndex; newExceptionHandler.start = callSite.bits(); newExceptionHandler.end = callSite.bits() + 1; newExceptionHandler.nativeCode = catchLabel; m_codeBlock->appendExceptionHandler(newExceptionHandler); } } if (m_pcToCodeOriginMapBuilder.didBuildMapping()) m_codeBlock->setPCToCodeOriginMap(std::make_unique<PCToCodeOriginMap>(WTFMove(m_pcToCodeOriginMapBuilder), linkBuffer)); }
void JITCompiler::compileFunction() { setStartOfCode(); compileEntry(); // === Function header code generation === // This is the main entry point, without performing an arity check. // If we needed to perform an arity check we will already have moved the return address, // so enter after this. Label fromArityCheck(this); // Plant a check that sufficient space is available in the JSStack. addPtr(TrustedImm32(virtualRegisterForLocal(m_graph.requiredRegisterCountForExecutionAndExit() - 1).offset() * sizeof(Register)), GPRInfo::callFrameRegister, GPRInfo::regT1); Jump stackOverflow = branchPtr(Above, AbsoluteAddress(m_vm->addressOfStackLimit()), GPRInfo::regT1); // Move the stack pointer down to accommodate locals addPtr(TrustedImm32(m_graph.stackPointerOffset() * sizeof(Register)), GPRInfo::callFrameRegister, stackPointerRegister); checkStackPointerAlignment(); compileSetupRegistersForEntry(); compileEntryExecutionFlag(); // === Function body code generation === m_speculative = std::make_unique<SpeculativeJIT>(*this); compileBody(); setEndOfMainPath(); // === Function footer code generation === // // Generate code to perform the stack overflow handling (if the stack check in // the function header fails), and generate the entry point with arity check. // // Generate the stack overflow handling; if the stack check in the function head fails, // we need to call out to a helper function to throw the StackOverflowError. stackOverflow.link(this); emitStoreCodeOrigin(CodeOrigin(0)); if (maxFrameExtentForSlowPathCall) addPtr(TrustedImm32(-maxFrameExtentForSlowPathCall), stackPointerRegister); m_speculative->callOperationWithCallFrameRollbackOnException(operationThrowStackOverflowError, m_codeBlock); // The fast entry point into a function does not check the correct number of arguments // have been passed to the call (we only use the fast entry point where we can statically // determine the correct number of arguments have been passed, or have already checked). // In cases where an arity check is necessary, we enter here. // FIXME: change this from a cti call to a DFG style operation (normal C calling conventions). m_arityCheck = label(); compileEntry(); load32(AssemblyHelpers::payloadFor((VirtualRegister)JSStack::ArgumentCount), GPRInfo::regT1); branch32(AboveOrEqual, GPRInfo::regT1, TrustedImm32(m_codeBlock->numParameters())).linkTo(fromArityCheck, this); emitStoreCodeOrigin(CodeOrigin(0)); if (maxFrameExtentForSlowPathCall) addPtr(TrustedImm32(-maxFrameExtentForSlowPathCall), stackPointerRegister); m_speculative->callOperationWithCallFrameRollbackOnException(m_codeBlock->m_isConstructor ? operationConstructArityCheck : operationCallArityCheck, GPRInfo::regT0); if (maxFrameExtentForSlowPathCall) addPtr(TrustedImm32(maxFrameExtentForSlowPathCall), stackPointerRegister); branchTest32(Zero, GPRInfo::returnValueGPR).linkTo(fromArityCheck, this); emitStoreCodeOrigin(CodeOrigin(0)); move(GPRInfo::returnValueGPR, GPRInfo::argumentGPR0); m_callArityFixup = call(); jump(fromArityCheck); // Generate slow path code. m_speculative->runSlowPathGenerators(m_pcToCodeOriginMapBuilder); m_pcToCodeOriginMapBuilder.appendItem(label(), PCToCodeOriginMapBuilder::defaultCodeOrigin()); compileExceptionHandlers(); linkOSRExits(); // Create OSR entry trampolines if necessary. m_speculative->createOSREntries(); setEndOfCode(); // === Link === auto linkBuffer = std::make_unique<LinkBuffer>(*m_vm, *this, m_codeBlock, JITCompilationCanFail); if (linkBuffer->didFailToAllocate()) { m_graph.m_plan.finalizer = std::make_unique<FailedFinalizer>(m_graph.m_plan); return; } link(*linkBuffer); m_speculative->linkOSREntries(*linkBuffer); m_jitCode->shrinkToFit(); codeBlock()->shrinkToFit(CodeBlock::LateShrink); linkBuffer->link(m_callArityFixup, FunctionPtr((m_vm->getCTIStub(arityFixupGenerator)).code().executableAddress())); disassemble(*linkBuffer); MacroAssemblerCodePtr withArityCheck = linkBuffer->locationOf(m_arityCheck); m_graph.m_plan.finalizer = std::make_unique<JITFinalizer>( m_graph.m_plan, m_jitCode.release(), WTFMove(linkBuffer), withArityCheck); }
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind) { int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind); JSInterfaceJIT jit; jit.emitPutImmediateToCallFrameHeader(0, JSStack::CodeBlock); jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame); #if CPU(X86) // Load caller frame's scope chain into this callframe so that whatever we call can // get to its global data. jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT0); jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0); jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); jit.peek(JSInterfaceJIT::regT1); jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ReturnPC); // Calling convention: f(ecx, edx, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx); jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister); // Align stack after call. // call the function jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1); jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1); jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction)); jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(void*)), JSInterfaceJIT::stackPointerRegister); #elif CPU(X86_64) // Load caller frame's scope chain into this callframe so that whatever we call can // get to its global data. jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT0); jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0); jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); jit.peek(JSInterfaceJIT::regT1); jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ReturnPC); #if !OS(WINDOWS) // Calling convention: f(edi, esi, edx, ecx, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi); jit.subPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); // Align stack after call. jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi); jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9); jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction)); jit.addPtr(JSInterfaceJIT::TrustedImm32(16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); #else // Calling convention: f(ecx, edx, r8, r9, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx); // Leave space for the callee parameter home addresses and align the stack. jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx); jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9); jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction)); jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); #endif #elif CPU(ARM) // Load caller frame's scope chain into this callframe so that whatever we call can // get to its global data. jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT2); jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2); jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC); // Calling convention: f(r0 == regT0, r1 == regT1, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, ARMRegisters::r0); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARMRegisters::r1); jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.loadPtr(JSInterfaceJIT::Address(ARMRegisters::r1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2); jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction)); jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3); #elif CPU(SH4) // Load caller frame's scope chain into this callframe so that whatever we call can // get to its global data. jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT2); jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2); jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC); // Calling convention: f(r0 == regT4, r1 == regT5, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::regT4); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT5); jit.move(JSInterfaceJIT::regT2, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT5, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2); jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction), JSInterfaceJIT::regT0); jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3); #elif CPU(MIPS) // Load caller frame's scope chain into this callframe so that whatever we call can // get to its global data. jit.emitGetFromCallFrameHeaderPtr(JSStack::CallerFrame, JSInterfaceJIT::regT0); jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0); jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT3); // Callee preserved jit.emitPutToCallFrameHeader(JSInterfaceJIT::regT3, JSStack::ReturnPC); // Calling convention: f(a0, a1, a2, a3); // Host function signature: f(ExecState*); // Allocate stack space for 16 bytes (8-byte aligned) // 16 bytes (unused) for 4 arguments jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); // Setup arg0 jit.move(JSInterfaceJIT::callFrameRegister, MIPSRegisters::a0); // Call jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, MIPSRegisters::a2); jit.loadPtr(JSInterfaceJIT::Address(MIPSRegisters::a2, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2); jit.move(JSInterfaceJIT::regT0, JSInterfaceJIT::callFrameRegister); // Eagerly restore caller frame register to avoid loading from stack. jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction)); // Restore stack space jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); jit.restoreReturnAddressBeforeReturn(JSInterfaceJIT::regT3); #else #error "JIT not supported on this platform." UNUSED_PARAM(executableOffsetToFunction); breakpoint(); #endif // Check for an exception #if USE(JSVALUE64) jit.load64(&(vm->exception), JSInterfaceJIT::regT2); JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2); #else JSInterfaceJIT::Jump exceptionHandler = jit.branch32( JSInterfaceJIT::NotEqual, JSInterfaceJIT::AbsoluteAddress(reinterpret_cast<char*>(&vm->exception) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), JSInterfaceJIT::TrustedImm32(JSValue::EmptyValueTag)); #endif // Return. jit.ret(); // Handle an exception exceptionHandler.link(&jit); // Grab the return address. jit.preserveReturnAddressAfterCall(JSInterfaceJIT::regT1); jit.move(JSInterfaceJIT::TrustedImmPtr(&vm->exceptionLocation), JSInterfaceJIT::regT2); jit.storePtr(JSInterfaceJIT::regT1, JSInterfaceJIT::regT2); jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame); jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(ctiVMThrowTrampolineSlowpath).value()), JSInterfaceJIT::regT1); jit.jump(JSInterfaceJIT::regT1); LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID); return FINALIZE_CODE(patchBuffer, ("native %s trampoline", toCString(kind).data())); }
MacroAssemblerCodeRef linkCallGenerator(VM* vm) { return linkForGenerator(vm, FunctionPtr(cti_vm_lazyLinkCall), FunctionPtr(cti_op_call_NotJSFunction), "call"); }
static void fixFunctionBasedOnStackMaps( State& state, CodeBlock* codeBlock, JITCode* jitCode, GeneratedFunction generatedFunction, StackMaps::RecordMap& recordMap, bool didSeeUnwindInfo) { Graph& graph = state.graph; VM& vm = graph.m_vm; StackMaps stackmaps = jitCode->stackmaps; int localsOffset = offsetOfStackRegion(recordMap, state.capturedStackmapID) + graph.m_nextMachineLocal; int varargsSpillSlotsOffset; if (state.varargsSpillSlotsStackmapID != UINT_MAX) varargsSpillSlotsOffset = offsetOfStackRegion(recordMap, state.varargsSpillSlotsStackmapID); else varargsSpillSlotsOffset = 0; for (unsigned i = graph.m_inlineVariableData.size(); i--;) { InlineCallFrame* inlineCallFrame = graph.m_inlineVariableData[i].inlineCallFrame; if (inlineCallFrame->argumentsRegister.isValid()) inlineCallFrame->argumentsRegister += localsOffset; if (inlineCallFrame->argumentCountRegister.isValid()) inlineCallFrame->argumentCountRegister += localsOffset; for (unsigned argument = inlineCallFrame->arguments.size(); argument-- > 1;) { inlineCallFrame->arguments[argument] = inlineCallFrame->arguments[argument].withLocalsOffset(localsOffset); } if (inlineCallFrame->isClosureCall) { inlineCallFrame->calleeRecovery = inlineCallFrame->calleeRecovery.withLocalsOffset(localsOffset); } } if (codeBlock->usesArguments()) { codeBlock->setArgumentsRegister( VirtualRegister(codeBlock->argumentsRegister().offset() + localsOffset)); } MacroAssembler::Label stackOverflowException; { CCallHelpers checkJIT(&vm, codeBlock); // At this point it's perfectly fair to just blow away all state and restore the // JS JIT view of the universe. checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0); checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); MacroAssembler::Call callLookupExceptionHandler = checkJIT.call(); checkJIT.jumpToExceptionHandler(); stackOverflowException = checkJIT.label(); checkJIT.move(MacroAssembler::TrustedImmPtr(&vm), GPRInfo::argumentGPR0); checkJIT.move(GPRInfo::callFrameRegister, GPRInfo::argumentGPR1); MacroAssembler::Call callLookupExceptionHandlerFromCallerFrame = checkJIT.call(); checkJIT.jumpToExceptionHandler(); auto linkBuffer = std::make_unique<LinkBuffer>( vm, checkJIT, codeBlock, JITCompilationMustSucceed); linkBuffer->link(callLookupExceptionHandler, FunctionPtr(lookupExceptionHandler)); linkBuffer->link(callLookupExceptionHandlerFromCallerFrame, FunctionPtr(lookupExceptionHandlerFromCallerFrame)); state.finalizer->handleExceptionsLinkBuffer = WTF::move(linkBuffer); } ExitThunkGenerator exitThunkGenerator(state); exitThunkGenerator.emitThunks(); if (exitThunkGenerator.didThings()) { RELEASE_ASSERT(state.finalizer->osrExit.size()); RELEASE_ASSERT(didSeeUnwindInfo); auto linkBuffer = std::make_unique<LinkBuffer>( vm, exitThunkGenerator, codeBlock, JITCompilationMustSucceed); RELEASE_ASSERT(state.finalizer->osrExit.size() == state.jitCode->osrExit.size()); for (unsigned i = 0; i < state.jitCode->osrExit.size(); ++i) { OSRExitCompilationInfo& info = state.finalizer->osrExit[i]; OSRExit& exit = jitCode->osrExit[i]; if (verboseCompilationEnabled()) dataLog("Handling OSR stackmap #", exit.m_stackmapID, " for ", exit.m_codeOrigin, "\n"); auto iter = recordMap.find(exit.m_stackmapID); if (iter == recordMap.end()) { // It was optimized out. continue; } info.m_thunkAddress = linkBuffer->locationOf(info.m_thunkLabel); exit.m_patchableCodeOffset = linkBuffer->offsetOf(info.m_thunkJump); for (unsigned j = exit.m_values.size(); j--;) { ExitValue value = exit.m_values[j]; if (!value.isInJSStackSomehow()) continue; if (!value.virtualRegister().isLocal()) continue; exit.m_values[j] = value.withVirtualRegister( VirtualRegister(value.virtualRegister().offset() + localsOffset)); } if (verboseCompilationEnabled()) { DumpContext context; dataLog(" Exit values: ", inContext(exit.m_values, &context), "\n"); } } state.finalizer->exitThunksLinkBuffer = WTF::move(linkBuffer); } if (!state.getByIds.isEmpty() || !state.putByIds.isEmpty() || !state.checkIns.isEmpty()) { CCallHelpers slowPathJIT(&vm, codeBlock); CCallHelpers::JumpList exceptionTarget; for (unsigned i = state.getByIds.size(); i--;) { GetByIdDescriptor& getById = state.getByIds[i]; if (verboseCompilationEnabled()) dataLog("Handling GetById stackmap #", getById.stackmapID(), "\n"); auto iter = recordMap.find(getById.stackmapID()); if (iter == recordMap.end()) { // It was optimized out. continue; } for (unsigned i = 0; i < iter->value.size(); ++i) { StackMaps::Record& record = iter->value[i]; RegisterSet usedRegisters = usedRegistersFor(record); GPRReg result = record.locations[0].directGPR(); GPRReg base = record.locations[1].directGPR(); JITGetByIdGenerator gen( codeBlock, getById.codeOrigin(), usedRegisters, JSValueRegs(base), JSValueRegs(result), NeedToSpill); MacroAssembler::Label begin = slowPathJIT.label(); MacroAssembler::Call call = callOperation( state, usedRegisters, slowPathJIT, getById.codeOrigin(), &exceptionTarget, operationGetByIdOptimize, result, gen.stubInfo(), base, getById.uid()); gen.reportSlowPathCall(begin, call); getById.m_slowPathDone.append(slowPathJIT.jump()); getById.m_generators.append(gen); } } for (unsigned i = state.putByIds.size(); i--;) { PutByIdDescriptor& putById = state.putByIds[i]; if (verboseCompilationEnabled()) dataLog("Handling PutById stackmap #", putById.stackmapID(), "\n"); auto iter = recordMap.find(putById.stackmapID()); if (iter == recordMap.end()) { // It was optimized out. continue; } for (unsigned i = 0; i < iter->value.size(); ++i) { StackMaps::Record& record = iter->value[i]; RegisterSet usedRegisters = usedRegistersFor(record); GPRReg base = record.locations[0].directGPR(); GPRReg value = record.locations[1].directGPR(); JITPutByIdGenerator gen( codeBlock, putById.codeOrigin(), usedRegisters, JSValueRegs(base), JSValueRegs(value), GPRInfo::patchpointScratchRegister, NeedToSpill, putById.ecmaMode(), putById.putKind()); MacroAssembler::Label begin = slowPathJIT.label(); MacroAssembler::Call call = callOperation( state, usedRegisters, slowPathJIT, putById.codeOrigin(), &exceptionTarget, gen.slowPathFunction(), gen.stubInfo(), value, base, putById.uid()); gen.reportSlowPathCall(begin, call); putById.m_slowPathDone.append(slowPathJIT.jump()); putById.m_generators.append(gen); } } for (unsigned i = state.checkIns.size(); i--;) { CheckInDescriptor& checkIn = state.checkIns[i]; if (verboseCompilationEnabled()) dataLog("Handling checkIn stackmap #", checkIn.stackmapID(), "\n"); auto iter = recordMap.find(checkIn.stackmapID()); if (iter == recordMap.end()) { // It was optimized out. continue; } for (unsigned i = 0; i < iter->value.size(); ++i) { StackMaps::Record& record = iter->value[i]; RegisterSet usedRegisters = usedRegistersFor(record); GPRReg result = record.locations[0].directGPR(); GPRReg obj = record.locations[1].directGPR(); StructureStubInfo* stubInfo = codeBlock->addStubInfo(); stubInfo->codeOrigin = checkIn.codeOrigin(); stubInfo->patch.baseGPR = static_cast<int8_t>(obj); stubInfo->patch.valueGPR = static_cast<int8_t>(result); stubInfo->patch.usedRegisters = usedRegisters; stubInfo->patch.spillMode = NeedToSpill; MacroAssembler::Label begin = slowPathJIT.label(); MacroAssembler::Call slowCall = callOperation( state, usedRegisters, slowPathJIT, checkIn.codeOrigin(), &exceptionTarget, operationInOptimize, result, stubInfo, obj, checkIn.m_id); checkIn.m_slowPathDone.append(slowPathJIT.jump()); checkIn.m_generators.append(CheckInGenerator(stubInfo, slowCall, begin)); } } exceptionTarget.link(&slowPathJIT); MacroAssembler::Jump exceptionJump = slowPathJIT.jump(); state.finalizer->sideCodeLinkBuffer = std::make_unique<LinkBuffer>(vm, slowPathJIT, codeBlock, JITCompilationMustSucceed); state.finalizer->sideCodeLinkBuffer->link( exceptionJump, state.finalizer->handleExceptionsLinkBuffer->entrypoint()); for (unsigned i = state.getByIds.size(); i--;) { generateICFastPath( state, codeBlock, generatedFunction, recordMap, state.getByIds[i], sizeOfGetById()); } for (unsigned i = state.putByIds.size(); i--;) { generateICFastPath( state, codeBlock, generatedFunction, recordMap, state.putByIds[i], sizeOfPutById()); } for (unsigned i = state.checkIns.size(); i--;) { generateCheckInICFastPath( state, codeBlock, generatedFunction, recordMap, state.checkIns[i], sizeOfIn()); } } adjustCallICsForStackmaps(state.jsCalls, recordMap); for (unsigned i = state.jsCalls.size(); i--;) { JSCall& call = state.jsCalls[i]; CCallHelpers fastPathJIT(&vm, codeBlock); call.emit(fastPathJIT); char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset; LinkBuffer linkBuffer(vm, fastPathJIT, startOfIC, sizeOfCall()); if (!linkBuffer.isValid()) { dataLog("Failed to insert inline cache for call because we thought the size would be ", sizeOfCall(), " but it ended up being ", fastPathJIT.m_assembler.codeSize(), " prior to compaction.\n"); RELEASE_ASSERT_NOT_REACHED(); } MacroAssembler::AssemblerType_T::fillNops( startOfIC + linkBuffer.size(), sizeOfCall() - linkBuffer.size()); call.link(vm, linkBuffer); } adjustCallICsForStackmaps(state.jsCallVarargses, recordMap); for (unsigned i = state.jsCallVarargses.size(); i--;) { JSCallVarargs& call = state.jsCallVarargses[i]; CCallHelpers fastPathJIT(&vm, codeBlock); call.emit(fastPathJIT, graph, varargsSpillSlotsOffset); char* startOfIC = bitwise_cast<char*>(generatedFunction) + call.m_instructionOffset; size_t sizeOfIC = sizeOfICFor(call.node()); LinkBuffer linkBuffer(vm, fastPathJIT, startOfIC, sizeOfIC); if (!linkBuffer.isValid()) { dataLog("Failed to insert inline cache for varargs call (specifically, ", Graph::opName(call.node()->op()), ") because we thought the size would be ", sizeOfIC, " but it ended up being ", fastPathJIT.m_assembler.codeSize(), " prior to compaction.\n"); RELEASE_ASSERT_NOT_REACHED(); } MacroAssembler::AssemblerType_T::fillNops( startOfIC + linkBuffer.size(), sizeOfIC - linkBuffer.size()); call.link(vm, linkBuffer, state.finalizer->handleExceptionsLinkBuffer->entrypoint()); } RepatchBuffer repatchBuffer(codeBlock); auto iter = recordMap.find(state.handleStackOverflowExceptionStackmapID); // It's sort of remotely possible that we won't have an in-band exception handling // path, for some kinds of functions. if (iter != recordMap.end()) { for (unsigned i = iter->value.size(); i--;) { StackMaps::Record& record = iter->value[i]; CodeLocationLabel source = CodeLocationLabel( bitwise_cast<char*>(generatedFunction) + record.instructionOffset); RELEASE_ASSERT(stackOverflowException.isSet()); repatchBuffer.replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->locationOf(stackOverflowException)); } } iter = recordMap.find(state.handleExceptionStackmapID); // It's sort of remotely possible that we won't have an in-band exception handling // path, for some kinds of functions. if (iter != recordMap.end()) { for (unsigned i = iter->value.size(); i--;) { StackMaps::Record& record = iter->value[i]; CodeLocationLabel source = CodeLocationLabel( bitwise_cast<char*>(generatedFunction) + record.instructionOffset); repatchBuffer.replaceWithJump(source, state.finalizer->handleExceptionsLinkBuffer->entrypoint()); } } for (unsigned exitIndex = 0; exitIndex < jitCode->osrExit.size(); ++exitIndex) { OSRExitCompilationInfo& info = state.finalizer->osrExit[exitIndex]; OSRExit& exit = jitCode->osrExit[exitIndex]; iter = recordMap.find(exit.m_stackmapID); Vector<const void*> codeAddresses; if (iter != recordMap.end()) { for (unsigned i = iter->value.size(); i--;) { StackMaps::Record& record = iter->value[i]; CodeLocationLabel source = CodeLocationLabel( bitwise_cast<char*>(generatedFunction) + record.instructionOffset); codeAddresses.append(bitwise_cast<char*>(generatedFunction) + record.instructionOffset + MacroAssembler::maxJumpReplacementSize()); if (info.m_isInvalidationPoint) jitCode->common.jumpReplacements.append(JumpReplacement(source, info.m_thunkAddress)); else repatchBuffer.replaceWithJump(source, info.m_thunkAddress); } } if (graph.compilation()) graph.compilation()->addOSRExitSite(codeAddresses); } }
static MacroAssemblerCodeRef nativeForGenerator(VM* vm, CodeSpecializationKind kind, ThunkEntryType entryType = EnterViaCall) { int executableOffsetToFunction = NativeExecutable::offsetOfNativeFunctionFor(kind); JSInterfaceJIT jit(vm); if (entryType == EnterViaCall) jit.emitFunctionPrologue(); jit.emitPutImmediateToCallFrameHeader(0, JSStack::CodeBlock); jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame); #if CPU(X86) // Load caller frame's scope chain into this callframe so that whatever we call can // get to its global data. jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0); jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0); jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); // Calling convention: f(ecx, edx, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx); jit.subPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); // Align stack after prologue. // call the function jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::regT1); jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT1); jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT1, executableOffsetToFunction)); jit.addPtr(JSInterfaceJIT::TrustedImm32(8), JSInterfaceJIT::stackPointerRegister); #elif CPU(X86_64) // Load caller frame's scope chain into this callframe so that whatever we call can // get to its global data. jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT0); jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT0); jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); #if !OS(WINDOWS) // Calling convention: f(edi, esi, edx, ecx, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::edi); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::esi); jit.loadPtr(JSInterfaceJIT::Address(X86Registers::esi, JSFunction::offsetOfExecutable()), X86Registers::r9); jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction)); #else // Calling convention: f(ecx, edx, r8, r9, ...); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, X86Registers::ecx); // Leave space for the callee parameter home addresses and align the stack. jit.subPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, X86Registers::edx); jit.loadPtr(JSInterfaceJIT::Address(X86Registers::edx, JSFunction::offsetOfExecutable()), X86Registers::r9); jit.call(JSInterfaceJIT::Address(X86Registers::r9, executableOffsetToFunction)); jit.addPtr(JSInterfaceJIT::TrustedImm32(4 * sizeof(int64_t) + 16 - sizeof(int64_t)), JSInterfaceJIT::stackPointerRegister); #endif #elif CPU(ARM64) COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT1, prev_callframe_not_trampled_by_T1); COMPILE_ASSERT(ARM64Registers::x3 != JSInterfaceJIT::regT3, prev_callframe_not_trampled_by_T3); COMPILE_ASSERT(ARM64Registers::x0 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_0); COMPILE_ASSERT(ARM64Registers::x1 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_1); COMPILE_ASSERT(ARM64Registers::x2 != JSInterfaceJIT::regT3, T3_not_trampled_by_arg_2); // Load caller frame's scope chain into this callframe so that whatever we call can // get to its global data. jit.emitGetCallerFrameFromCallFrameHeaderPtr(ARM64Registers::x3); jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, ARM64Registers::x3); jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); // Host function signature: f(ExecState*); jit.move(JSInterfaceJIT::callFrameRegister, ARM64Registers::x0); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, ARM64Registers::x1); jit.loadPtr(JSInterfaceJIT::Address(ARM64Registers::x1, JSFunction::offsetOfExecutable()), ARM64Registers::x2); jit.call(JSInterfaceJIT::Address(ARM64Registers::x2, executableOffsetToFunction)); #elif CPU(ARM) || CPU(SH4) || CPU(MIPS) // Load caller frame's scope chain into this callframe so that whatever we call can get to its global data. jit.emitGetCallerFrameFromCallFrameHeaderPtr(JSInterfaceJIT::regT2); jit.emitGetFromCallFrameHeaderPtr(JSStack::ScopeChain, JSInterfaceJIT::regT1, JSInterfaceJIT::regT2); jit.emitPutCellToCallFrameHeader(JSInterfaceJIT::regT1, JSStack::ScopeChain); #if CPU(MIPS) // Allocate stack space for (unused) 16 bytes (8-byte aligned) for 4 arguments. jit.subPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); #endif // Calling convention is f(argumentGPR0, argumentGPR1, ...). // Host function signature is f(ExecState*). jit.move(JSInterfaceJIT::callFrameRegister, JSInterfaceJIT::argumentGPR0); jit.emitGetFromCallFrameHeaderPtr(JSStack::Callee, JSInterfaceJIT::argumentGPR1); jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::argumentGPR1, JSFunction::offsetOfExecutable()), JSInterfaceJIT::regT2); jit.call(JSInterfaceJIT::Address(JSInterfaceJIT::regT2, executableOffsetToFunction)); #if CPU(MIPS) // Restore stack space jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); #endif #else #error "JIT not supported on this platform." UNUSED_PARAM(executableOffsetToFunction); breakpoint(); #endif // Check for an exception #if USE(JSVALUE64) jit.load64(vm->addressOfException(), JSInterfaceJIT::regT2); JSInterfaceJIT::Jump exceptionHandler = jit.branchTest64(JSInterfaceJIT::NonZero, JSInterfaceJIT::regT2); #else JSInterfaceJIT::Jump exceptionHandler = jit.branch32( JSInterfaceJIT::NotEqual, JSInterfaceJIT::AbsoluteAddress(reinterpret_cast<char*>(vm->addressOfException()) + OBJECT_OFFSETOF(EncodedValueDescriptor, asBits.tag)), JSInterfaceJIT::TrustedImm32(JSValue::EmptyValueTag)); #endif jit.emitFunctionEpilogue(); // Return. jit.ret(); // Handle an exception exceptionHandler.link(&jit); jit.storePtr(JSInterfaceJIT::callFrameRegister, &vm->topCallFrame); #if CPU(X86) && USE(JSVALUE32_64) jit.addPtr(JSInterfaceJIT::TrustedImm32(-12), JSInterfaceJIT::stackPointerRegister); jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister), JSInterfaceJIT::regT0); jit.push(JSInterfaceJIT::regT0); #else jit.loadPtr(JSInterfaceJIT::Address(JSInterfaceJIT::callFrameRegister), JSInterfaceJIT::argumentGPR0); #endif jit.move(JSInterfaceJIT::TrustedImmPtr(FunctionPtr(operationVMHandleException).value()), JSInterfaceJIT::regT3); jit.call(JSInterfaceJIT::regT3); #if CPU(X86) && USE(JSVALUE32_64) jit.addPtr(JSInterfaceJIT::TrustedImm32(16), JSInterfaceJIT::stackPointerRegister); #endif jit.jumpToExceptionHandler(); LinkBuffer patchBuffer(*vm, &jit, GLOBAL_THUNK_ID); return FINALIZE_CODE(patchBuffer, ("native %s%s trampoline", entryType == EnterViaJump ? "Tail " : "", toCString(kind).data())); }
void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. #if DFG_ENABLE(DEBUG_VERBOSE) dataLogF("JIT code for %p start at [%p, %p). Size = %zu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize()); #endif // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); m_codeBlock->callReturnIndexVector().reserveCapacity(m_exceptionChecks.size()); for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) { unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call); CodeOrigin codeOrigin = m_exceptionChecks[i].m_codeOrigin; while (codeOrigin.inlineCallFrame) codeOrigin = codeOrigin.inlineCallFrame->caller; unsigned exceptionInfo = codeOrigin.bytecodeIndex; m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo)); } Vector<CodeOriginAtCallReturnOffset, 0, UnsafeVectorOverflow>& codeOrigins = m_codeBlock->codeOrigins(); codeOrigins.resize(m_exceptionChecks.size()); for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) { CallExceptionRecord& record = m_exceptionChecks[i]; unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call); codeOrigins[i].codeOrigin = record.m_codeOrigin; codeOrigins[i].callReturnOffset = returnAddressOffset; } m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size()); for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) { StructureStubInfo& info = m_codeBlock->structureStubInfo(i); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->call()); info.codeOrigin = m_propertyAccesses[i].m_codeOrigin; info.callReturnLocation = callReturnLocation; info.patch.dfg.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_structureImm), callReturnLocation); info.patch.dfg.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_structureCheck)); #if USE(JSVALUE64) info.patch.dfg.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_loadOrStore)); #else info.patch.dfg.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_tagLoadOrStore)); info.patch.dfg.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_payloadLoadOrStore)); #endif info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->label())); info.patch.dfg.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_done)); info.patch.dfg.deltaCallToStorageLoad = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_propertyStorageLoad)); info.patch.dfg.baseGPR = m_propertyAccesses[i].m_baseGPR; #if USE(JSVALUE64) info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #else info.patch.dfg.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR; info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #endif m_propertyAccesses[i].m_usedRegisters.copyInfo(info.patch.dfg.usedRegisters); info.patch.dfg.registersFlushed = m_propertyAccesses[i].m_registerMode == PropertyAccessRecord::RegistersFlushed; } m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size()); for (unsigned i = 0; i < m_jsCalls.size(); ++i) { CallLinkInfo& info = m_codeBlock->callLinkInfo(i); info.callType = m_jsCalls[i].m_callType; info.isDFG = true; info.codeOrigin = m_jsCalls[i].m_codeOrigin; linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_vm->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress())); info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall); info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck); info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall); info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee); } MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < codeBlock()->numberOfOSRExits(); ++i) { OSRExit& exit = codeBlock()->osrExit(i); linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target); exit.correctJump(linkBuffer); if (exit.m_watchpointIndex != std::numeric_limits<unsigned>::max()) codeBlock()->watchpoint(exit.m_watchpointIndex).correctLabels(linkBuffer); } if (m_graph.m_compilation) { ASSERT(m_exitSiteLabels.size() == codeBlock()->numberOfOSRExits()); for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) { Vector<Label>& labels = m_exitSiteLabels[i]; Vector<const void*> addresses; for (unsigned j = 0; j < labels.size(); ++j) addresses.append(linkBuffer.locationOf(labels[j]).executableAddress()); m_graph.m_compilation->addOSRExitSite(addresses); } } else ASSERT(!m_exitSiteLabels.size()); codeBlock()->saveCompilation(m_graph.m_compilation); codeBlock()->shrinkToFit(CodeBlock::LateShrink); }