void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. #if DFG_ENABLE(DEBUG_VERBOSE) dataLogF("JIT code for %p start at [%p, %p). Size = %zu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize()); #endif // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); m_codeBlock->callReturnIndexVector().reserveCapacity(m_exceptionChecks.size()); for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) { unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call); CodeOrigin codeOrigin = m_exceptionChecks[i].m_codeOrigin; while (codeOrigin.inlineCallFrame) codeOrigin = codeOrigin.inlineCallFrame->caller; unsigned exceptionInfo = codeOrigin.bytecodeIndex; m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo)); } Vector<CodeOriginAtCallReturnOffset, 0, UnsafeVectorOverflow>& codeOrigins = m_codeBlock->codeOrigins(); codeOrigins.resize(m_exceptionChecks.size()); for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) { CallExceptionRecord& record = m_exceptionChecks[i]; unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call); codeOrigins[i].codeOrigin = record.m_codeOrigin; codeOrigins[i].callReturnOffset = returnAddressOffset; } m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size()); for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) { StructureStubInfo& info = m_codeBlock->structureStubInfo(i); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->call()); info.codeOrigin = m_propertyAccesses[i].m_codeOrigin; info.callReturnLocation = callReturnLocation; info.patch.dfg.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_structureImm), callReturnLocation); info.patch.dfg.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_structureCheck)); #if USE(JSVALUE64) info.patch.dfg.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_loadOrStore)); #else info.patch.dfg.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_tagLoadOrStore)); info.patch.dfg.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_payloadLoadOrStore)); #endif info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->label())); info.patch.dfg.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_done)); info.patch.dfg.deltaCallToStorageLoad = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_propertyStorageLoad)); info.patch.dfg.baseGPR = m_propertyAccesses[i].m_baseGPR; #if USE(JSVALUE64) info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #else info.patch.dfg.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR; info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #endif m_propertyAccesses[i].m_usedRegisters.copyInfo(info.patch.dfg.usedRegisters); info.patch.dfg.registersFlushed = m_propertyAccesses[i].m_registerMode == PropertyAccessRecord::RegistersFlushed; } m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size()); for (unsigned i = 0; i < m_jsCalls.size(); ++i) { CallLinkInfo& info = m_codeBlock->callLinkInfo(i); info.callType = m_jsCalls[i].m_callType; info.isDFG = true; info.codeOrigin = m_jsCalls[i].m_codeOrigin; linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_vm->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress())); info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall); info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck); info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall); info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee); } MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < codeBlock()->numberOfOSRExits(); ++i) { OSRExit& exit = codeBlock()->osrExit(i); linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target); exit.correctJump(linkBuffer); if (exit.m_watchpointIndex != std::numeric_limits<unsigned>::max()) codeBlock()->watchpoint(exit.m_watchpointIndex).correctLabels(linkBuffer); } if (m_graph.m_compilation) { ASSERT(m_exitSiteLabels.size() == codeBlock()->numberOfOSRExits()); for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) { Vector<Label>& labels = m_exitSiteLabels[i]; Vector<const void*> addresses; for (unsigned j = 0; j < labels.size(); ++j) addresses.append(linkBuffer.locationOf(labels[j]).executableAddress()); m_graph.m_compilation->addOSRExitSite(addresses); } } else ASSERT(!m_exitSiteLabels.size()); codeBlock()->saveCompilation(m_graph.m_compilation); codeBlock()->shrinkToFit(CodeBlock::LateShrink); }
void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. m_jitCode->common.frameRegisterCount = m_graph.frameRegisterCount(); m_jitCode->common.requiredRegisterCountForExit = m_graph.requiredRegisterCountForExit(); if (!m_graph.m_plan.inlineCallFrames->isEmpty()) m_jitCode->common.inlineCallFrames = m_graph.m_plan.inlineCallFrames; #if USE(JSVALUE32_64) m_jitCode->common.doubleConstants = WTFMove(m_graph.m_doubleConstants); #endif m_graph.registerFrozenValues(); BitVector usedJumpTables; for (Bag<SwitchData>::iterator iter = m_graph.m_switchData.begin(); !!iter; ++iter) { SwitchData& data = **iter; if (!data.didUseJumpTable) continue; if (data.kind == SwitchString) continue; RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar); usedJumpTables.set(data.switchTableIndex); SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough.block->index]); table.ctiOffsets.grow(table.branchOffsets.size()); for (unsigned j = table.ctiOffsets.size(); j--;) table.ctiOffsets[j] = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; table.ctiOffsets[myCase.value.switchLookupValue(data.kind) - table.min] = linkBuffer.locationOf(m_blockHeads[myCase.target.block->index]); } } for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) { if (usedJumpTables.get(i)) continue; m_codeBlock->switchJumpTable(i).clear(); } // NOTE: we cannot clear string switch tables because (1) we're running concurrently // and we cannot deref StringImpl's and (2) it would be weird to deref those // StringImpl's since we refer to them. for (Bag<SwitchData>::iterator switchDataIter = m_graph.m_switchData.begin(); !!switchDataIter; ++switchDataIter) { SwitchData& data = **switchDataIter; if (!data.didUseJumpTable) continue; if (data.kind != SwitchString) continue; StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough.block->index]); StringJumpTable::StringOffsetTable::iterator iter; StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end(); for (iter = table.offsetTable.begin(); iter != end; ++iter) iter->value.ctiOffset = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; iter = table.offsetTable.find(myCase.value.stringImpl()); RELEASE_ASSERT(iter != end); iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target.block->index]); } } // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); for (unsigned i = m_getByIds.size(); i--;) m_getByIds[i].finalize(linkBuffer); for (unsigned i = m_putByIds.size(); i--;) m_putByIds[i].finalize(linkBuffer); for (unsigned i = 0; i < m_ins.size(); ++i) { StructureStubInfo& info = *m_ins[i].m_stubInfo; CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call()); info.patch.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_done)); info.patch.deltaCallToJump = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_jump)); info.callReturnLocation = callReturnLocation; info.patch.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label())); } for (unsigned i = 0; i < m_jsCalls.size(); ++i) { JSCallRecord& record = m_jsCalls[i]; CallLinkInfo& info = *record.m_info; linkBuffer.link(record.m_slowCall, FunctionPtr(m_vm->getCTIStub(linkCallThunkGenerator).code().executableAddress())); info.setCallLocations(linkBuffer.locationOfNearCall(record.m_slowCall), linkBuffer.locationOf(record.m_targetToCheck), linkBuffer.locationOfNearCall(record.m_fastCall)); } MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) { OSRExit& exit = m_jitCode->osrExit[i]; OSRExitCompilationInfo& info = m_exitCompilationInfo[i]; linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target); exit.correctJump(linkBuffer); if (info.m_replacementSource.isSet()) { m_jitCode->common.jumpReplacements.append(JumpReplacement( linkBuffer.locationOf(info.m_replacementSource), linkBuffer.locationOf(info.m_replacementDestination))); } } if (m_graph.compilation()) { ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size()); for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) { Vector<Label>& labels = m_exitSiteLabels[i]; Vector<const void*> addresses; for (unsigned j = 0; j < labels.size(); ++j) addresses.append(linkBuffer.locationOf(labels[j]).executableAddress()); m_graph.compilation()->addOSRExitSite(addresses); } } else ASSERT(!m_exitSiteLabels.size()); m_jitCode->common.compilation = m_graph.compilation(); // Link new DFG exception handlers and remove baseline JIT handlers. m_codeBlock->clearExceptionHandlers(); for (unsigned i = 0; i < m_exceptionHandlerOSRExitCallSites.size(); i++) { OSRExitCompilationInfo& info = m_exceptionHandlerOSRExitCallSites[i].exitInfo; if (info.m_replacementDestination.isSet()) { // If this is is *not* set, it means that we already jumped to the OSR exit in pure generated control flow. // i.e, we explicitly emitted an exceptionCheck that we know will be caught in this machine frame. // If this *is set*, it means we will be landing at this code location from genericUnwind from an // exception thrown in a child call frame. CodeLocationLabel catchLabel = linkBuffer.locationOf(info.m_replacementDestination); HandlerInfo newExceptionHandler = m_exceptionHandlerOSRExitCallSites[i].baselineExceptionHandler; CallSiteIndex callSite = m_exceptionHandlerOSRExitCallSites[i].callSiteIndex; newExceptionHandler.start = callSite.bits(); newExceptionHandler.end = callSite.bits() + 1; newExceptionHandler.nativeCode = catchLabel; m_codeBlock->appendExceptionHandler(newExceptionHandler); } } if (m_pcToCodeOriginMapBuilder.didBuildMapping()) m_codeBlock->setPCToCodeOriginMap(std::make_unique<PCToCodeOriginMap>(WTFMove(m_pcToCodeOriginMapBuilder), linkBuffer)); }
void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. #if DFG_ENABLE(DEBUG_VERBOSE) dataLogF("JIT code for %p start at [%p, %p). Size = %zu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize()); #endif if (!m_graph.m_inlineCallFrames->isEmpty()) { m_graph.m_inlineCallFrames->shrinkToFit(); m_jitCode->common.inlineCallFrames = m_graph.m_inlineCallFrames.release(); } BitVector usedJumpTables; for (unsigned i = m_graph.m_switchData.size(); i--;) { SwitchData& data = m_graph.m_switchData[i]; if (!data.didUseJumpTable) continue; if (data.kind == SwitchString) continue; RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar); usedJumpTables.set(data.switchTableIndex); SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]); table.ctiOffsets.grow(table.branchOffsets.size()); for (unsigned j = table.ctiOffsets.size(); j--;) table.ctiOffsets[j] = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; table.ctiOffsets[myCase.value.switchLookupValue() - table.min] = linkBuffer.locationOf(m_blockHeads[myCase.target->index]); } } for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) { if (usedJumpTables.get(i)) continue; m_codeBlock->switchJumpTable(i).clear(); } // NOTE: we cannot clear string switch tables because (1) we're running concurrently // and we cannot deref StringImpl's and (2) it would be weird to deref those // StringImpl's since we refer to them. for (unsigned i = m_graph.m_switchData.size(); i--;) { SwitchData& data = m_graph.m_switchData[i]; if (!data.didUseJumpTable) continue; if (data.kind != SwitchString) continue; StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]); StringJumpTable::StringOffsetTable::iterator iter; StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end(); for (iter = table.offsetTable.begin(); iter != end; ++iter) iter->value.ctiOffset = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; iter = table.offsetTable.find(myCase.value.stringImpl()); RELEASE_ASSERT(iter != end); iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target->index]); } } // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size() + m_ins.size()); for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) { StructureStubInfo& info = m_codeBlock->structureStubInfo(i); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->call()); info.codeOrigin = m_propertyAccesses[i].m_codeOrigin; info.callReturnLocation = callReturnLocation; info.patch.dfg.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_structureImm), callReturnLocation); info.patch.dfg.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_structureCheck)); #if USE(JSVALUE64) info.patch.dfg.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_loadOrStore)); #else info.patch.dfg.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_tagLoadOrStore)); info.patch.dfg.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_payloadLoadOrStore)); #endif info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->label())); info.patch.dfg.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_done)); info.patch.dfg.deltaCallToStorageLoad = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_propertyStorageLoad)); info.patch.dfg.baseGPR = m_propertyAccesses[i].m_baseGPR; #if USE(JSVALUE64) info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #else info.patch.dfg.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR; info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #endif m_propertyAccesses[i].m_usedRegisters.copyInfo(info.patch.dfg.usedRegisters); info.patch.dfg.registersFlushed = m_propertyAccesses[i].m_registerMode == PropertyAccessRecord::RegistersFlushed; } for (unsigned i = 0; i < m_ins.size(); ++i) { StructureStubInfo& info = m_codeBlock->structureStubInfo(m_propertyAccesses.size() + i); CodeLocationLabel jump = linkBuffer.locationOf(m_ins[i].m_jump); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call()); info.codeOrigin = m_ins[i].m_codeOrigin; info.hotPathBegin = jump; info.callReturnLocation = callReturnLocation; info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label())); info.patch.dfg.baseGPR = m_ins[i].m_baseGPR; info.patch.dfg.valueGPR = m_ins[i].m_resultGPR; m_ins[i].m_usedRegisters.copyInfo(info.patch.dfg.usedRegisters); info.patch.dfg.registersFlushed = false; } m_codeBlock->sortStructureStubInfos(); m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size()); for (unsigned i = 0; i < m_jsCalls.size(); ++i) { CallLinkInfo& info = m_codeBlock->callLinkInfo(i); info.callType = m_jsCalls[i].m_callType; info.isDFG = true; info.codeOrigin = m_jsCalls[i].m_codeOrigin; linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_vm->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress())); info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall); info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck); info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall); info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee); } MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) { OSRExit& exit = m_jitCode->osrExit[i]; linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target); exit.correctJump(linkBuffer); if (exit.m_watchpointIndex != std::numeric_limits<unsigned>::max()) m_jitCode->watchpoints[exit.m_watchpointIndex].correctLabels(linkBuffer); } if (m_graph.compilation()) { ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size()); for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) { Vector<Label>& labels = m_exitSiteLabels[i]; Vector<const void*> addresses; for (unsigned j = 0; j < labels.size(); ++j) addresses.append(linkBuffer.locationOf(labels[j]).executableAddress()); m_graph.compilation()->addOSRExitSite(addresses); } } else ASSERT(!m_exitSiteLabels.size()); m_jitCode->common.compilation = m_graph.compilation(); }
void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. #if DFG_ENABLE(DEBUG_VERBOSE) dataLog("JIT code for %p start at [%p, %p). Size = %zu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize()); #endif // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); m_codeBlock->callReturnIndexVector().reserveCapacity(m_exceptionChecks.size()); for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) { unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call); CodeOrigin codeOrigin = m_exceptionChecks[i].m_codeOrigin; while (codeOrigin.inlineCallFrame) codeOrigin = codeOrigin.inlineCallFrame->caller; unsigned exceptionInfo = codeOrigin.bytecodeIndex; m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo)); } Vector<CodeOriginAtCallReturnOffset>& codeOrigins = m_codeBlock->codeOrigins(); codeOrigins.resize(m_exceptionChecks.size()); for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) { CallExceptionRecord& record = m_exceptionChecks[i]; unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call); codeOrigins[i].codeOrigin = record.m_codeOrigin; codeOrigins[i].callReturnOffset = returnAddressOffset; record.m_token.assertCodeOriginIndex(i); } m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size()); for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) { StructureStubInfo& info = m_codeBlock->structureStubInfo(i); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->call()); info.codeOrigin = m_propertyAccesses[i].m_codeOrigin; info.callReturnLocation = callReturnLocation; info.patch.dfg.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_structureImm), callReturnLocation); info.patch.dfg.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_structureCheck)); #if USE(JSVALUE64) info.patch.dfg.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_loadOrStore)); #else info.patch.dfg.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_tagLoadOrStore)); info.patch.dfg.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_payloadLoadOrStore)); #endif info.patch.dfg.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_slowPathGenerator->label())); info.patch.dfg.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_done)); info.patch.dfg.baseGPR = m_propertyAccesses[i].m_baseGPR; #if USE(JSVALUE64) info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #else info.patch.dfg.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR; info.patch.dfg.valueGPR = m_propertyAccesses[i].m_valueGPR; #endif info.patch.dfg.scratchGPR = m_propertyAccesses[i].m_scratchGPR; info.patch.dfg.registersFlushed = m_propertyAccesses[i].m_registerMode == PropertyAccessRecord::RegistersFlushed; } m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size()); for (unsigned i = 0; i < m_jsCalls.size(); ++i) { CallLinkInfo& info = m_codeBlock->callLinkInfo(i); info.callType = m_jsCalls[i].m_callType; info.isDFG = true; info.callReturnLocation = CodeLocationLabel(linkBuffer.locationOf(m_jsCalls[i].m_slowCall)); info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck); info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall); } MacroAssemblerCodeRef osrExitThunk = globalData()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < codeBlock()->numberOfOSRExits(); ++i) { OSRExit& exit = codeBlock()->osrExit(i); linkBuffer.link(exit.m_check.lateJump(), target); exit.m_check.correctLateJump(linkBuffer); if (exit.m_watchpointIndex != std::numeric_limits<unsigned>::max()) codeBlock()->watchpoint(exit.m_watchpointIndex).correctLabels(linkBuffer); } codeBlock()->shrinkToFit(CodeBlock::LateShrink); }
void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. m_jitCode->common.frameRegisterCount = m_graph.frameRegisterCount(); m_jitCode->common.requiredRegisterCountForExit = m_graph.requiredRegisterCountForExit(); if (!m_graph.m_inlineCallFrames->isEmpty()) m_jitCode->common.inlineCallFrames = m_graph.m_inlineCallFrames.release(); m_jitCode->common.machineCaptureStart = m_graph.m_machineCaptureStart; m_jitCode->common.slowArguments = std::move(m_graph.m_slowArguments); BitVector usedJumpTables; for (unsigned i = m_graph.m_switchData.size(); i--;) { SwitchData& data = m_graph.m_switchData[i]; if (!data.didUseJumpTable) continue; if (data.kind == SwitchString) continue; RELEASE_ASSERT(data.kind == SwitchImm || data.kind == SwitchChar); usedJumpTables.set(data.switchTableIndex); SimpleJumpTable& table = m_codeBlock->switchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]); table.ctiOffsets.grow(table.branchOffsets.size()); for (unsigned j = table.ctiOffsets.size(); j--;) table.ctiOffsets[j] = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; table.ctiOffsets[myCase.value.switchLookupValue() - table.min] = linkBuffer.locationOf(m_blockHeads[myCase.target->index]); } } for (unsigned i = m_codeBlock->numberOfSwitchJumpTables(); i--;) { if (usedJumpTables.get(i)) continue; m_codeBlock->switchJumpTable(i).clear(); } // NOTE: we cannot clear string switch tables because (1) we're running concurrently // and we cannot deref StringImpl's and (2) it would be weird to deref those // StringImpl's since we refer to them. for (unsigned i = m_graph.m_switchData.size(); i--;) { SwitchData& data = m_graph.m_switchData[i]; if (!data.didUseJumpTable) continue; if (data.kind != SwitchString) continue; StringJumpTable& table = m_codeBlock->stringSwitchJumpTable(data.switchTableIndex); table.ctiDefault = linkBuffer.locationOf(m_blockHeads[data.fallThrough->index]); StringJumpTable::StringOffsetTable::iterator iter; StringJumpTable::StringOffsetTable::iterator end = table.offsetTable.end(); for (iter = table.offsetTable.begin(); iter != end; ++iter) iter->value.ctiOffset = table.ctiDefault; for (unsigned j = data.cases.size(); j--;) { SwitchCase& myCase = data.cases[j]; iter = table.offsetTable.find(myCase.value.stringImpl()); RELEASE_ASSERT(iter != end); iter->value.ctiOffset = linkBuffer.locationOf(m_blockHeads[myCase.target->index]); } } // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); for (unsigned i = m_getByIds.size(); i--;) m_getByIds[i].finalize(linkBuffer); for (unsigned i = m_putByIds.size(); i--;) m_putByIds[i].finalize(linkBuffer); for (unsigned i = 0; i < m_ins.size(); ++i) { StructureStubInfo& info = *m_ins[i].m_stubInfo; CodeLocationLabel jump = linkBuffer.locationOf(m_ins[i].m_jump); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->call()); info.hotPathBegin = jump; info.callReturnLocation = callReturnLocation; info.patch.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_ins[i].m_slowPathGenerator->label())); } m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size()); for (unsigned i = 0; i < m_jsCalls.size(); ++i) { CallLinkInfo& info = m_codeBlock->callLinkInfo(i); info.callType = m_jsCalls[i].m_callType; info.isDFG = true; info.codeOrigin = m_jsCalls[i].m_codeOrigin; linkBuffer.link(m_jsCalls[i].m_slowCall, FunctionPtr((m_vm->getCTIStub(info.callType == CallLinkInfo::Construct ? linkConstructThunkGenerator : linkCallThunkGenerator)).code().executableAddress())); info.callReturnLocation = linkBuffer.locationOfNearCall(m_jsCalls[i].m_slowCall); info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck); info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall); info.calleeGPR = static_cast<unsigned>(m_jsCalls[i].m_callee); } MacroAssemblerCodeRef osrExitThunk = vm()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < m_jitCode->osrExit.size(); ++i) { OSRExit& exit = m_jitCode->osrExit[i]; OSRExitCompilationInfo& info = m_exitCompilationInfo[i]; linkBuffer.link(exit.getPatchableCodeOffsetAsJump(), target); exit.correctJump(linkBuffer); if (info.m_replacementSource.isSet()) { m_jitCode->common.jumpReplacements.append(JumpReplacement( linkBuffer.locationOf(info.m_replacementSource), linkBuffer.locationOf(info.m_replacementDestination))); } } if (m_graph.compilation()) { ASSERT(m_exitSiteLabels.size() == m_jitCode->osrExit.size()); for (unsigned i = 0; i < m_exitSiteLabels.size(); ++i) { Vector<Label>& labels = m_exitSiteLabels[i]; Vector<const void*> addresses; for (unsigned j = 0; j < labels.size(); ++j) addresses.append(linkBuffer.locationOf(labels[j]).executableAddress()); m_graph.compilation()->addOSRExitSite(addresses); } } else ASSERT(!m_exitSiteLabels.size()); m_jitCode->common.compilation = m_graph.compilation(); }
void JITCompiler::link(LinkBuffer& linkBuffer) { // Link the code, populate data in CodeBlock data structures. #if DFG_ENABLE(DEBUG_VERBOSE) fprintf(stderr, "JIT code for %p start at [%p, %p). Size = %lu.\n", m_codeBlock, linkBuffer.debugAddress(), static_cast<char*>(linkBuffer.debugAddress()) + linkBuffer.debugSize(), linkBuffer.debugSize()); #endif // Link all calls out from the JIT code to their respective functions. for (unsigned i = 0; i < m_calls.size(); ++i) linkBuffer.link(m_calls[i].m_call, m_calls[i].m_function); if (m_codeBlock->needsCallReturnIndices()) { m_codeBlock->callReturnIndexVector().reserveCapacity(m_exceptionChecks.size()); for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) { unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call); CodeOrigin codeOrigin = m_exceptionChecks[i].m_codeOrigin; while (codeOrigin.inlineCallFrame) codeOrigin = codeOrigin.inlineCallFrame->caller; unsigned exceptionInfo = codeOrigin.bytecodeIndex; m_codeBlock->callReturnIndexVector().append(CallReturnOffsetToBytecodeOffset(returnAddressOffset, exceptionInfo)); } } unsigned numCallsFromInlineCode = 0; for (unsigned i = 0; i < m_exceptionChecks.size(); ++i) { if (m_exceptionChecks[i].m_codeOrigin.inlineCallFrame) numCallsFromInlineCode++; } if (numCallsFromInlineCode) { Vector<CodeOriginAtCallReturnOffset>& codeOrigins = m_codeBlock->codeOrigins(); codeOrigins.resize(numCallsFromInlineCode); for (unsigned i = 0, j = 0; i < m_exceptionChecks.size(); ++i) { CallExceptionRecord& record = m_exceptionChecks[i]; if (record.m_codeOrigin.inlineCallFrame) { unsigned returnAddressOffset = linkBuffer.returnAddressOffset(m_exceptionChecks[i].m_call); codeOrigins[j].codeOrigin = record.m_codeOrigin; codeOrigins[j].callReturnOffset = returnAddressOffset; j++; } } } m_codeBlock->setNumberOfStructureStubInfos(m_propertyAccesses.size()); for (unsigned i = 0; i < m_propertyAccesses.size(); ++i) { StructureStubInfo& info = m_codeBlock->structureStubInfo(i); CodeLocationCall callReturnLocation = linkBuffer.locationOf(m_propertyAccesses[i].m_functionCall); info.callReturnLocation = callReturnLocation; info.deltaCheckImmToCall = differenceBetweenCodePtr(linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCheckImmToCall), callReturnLocation); info.deltaCallToStructCheck = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToStructCheck)); #if USE(JSVALUE64) info.deltaCallToLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToLoadOrStore)); #else info.deltaCallToTagLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToTagLoadOrStore)); info.deltaCallToPayloadLoadOrStore = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToPayloadLoadOrStore)); #endif info.deltaCallToSlowCase = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToSlowCase)); info.deltaCallToDone = differenceBetweenCodePtr(callReturnLocation, linkBuffer.locationOf(m_propertyAccesses[i].m_deltaCallToDone)); info.baseGPR = m_propertyAccesses[i].m_baseGPR; #if USE(JSVALUE64) info.valueGPR = m_propertyAccesses[i].m_valueGPR; #else info.valueTagGPR = m_propertyAccesses[i].m_valueTagGPR; info.valueGPR = m_propertyAccesses[i].m_valueGPR; #endif info.scratchGPR = m_propertyAccesses[i].m_scratchGPR; } m_codeBlock->setNumberOfCallLinkInfos(m_jsCalls.size()); for (unsigned i = 0; i < m_jsCalls.size(); ++i) { CallLinkInfo& info = m_codeBlock->callLinkInfo(i); info.callType = m_jsCalls[i].m_callType; info.isDFG = true; info.callReturnLocation = CodeLocationLabel(linkBuffer.locationOf(m_jsCalls[i].m_slowCall)); info.hotPathBegin = linkBuffer.locationOf(m_jsCalls[i].m_targetToCheck); info.hotPathOther = linkBuffer.locationOfNearCall(m_jsCalls[i].m_fastCall); } MacroAssemblerCodeRef osrExitThunk = globalData()->getCTIStub(osrExitGenerationThunkGenerator); CodeLocationLabel target = CodeLocationLabel(osrExitThunk.code()); for (unsigned i = 0; i < codeBlock()->numberOfOSRExits(); ++i) { OSRExit& exit = codeBlock()->osrExit(i); linkBuffer.link(exit.m_check.lateJump(), target); exit.m_check.correctLateJump(linkBuffer); } codeBlock()->shrinkWeakReferencesToFit(); codeBlock()->shrinkWeakReferenceTransitionsToFit(); }