void link(CompilerState& state, const LinkDesc& desc)
{
StackMaps sm;
DataView dv(state.m_stackMapsSection->data());
sm.parse(&dv);
auto rm = sm.computeRecordMap();
EMASSERT(state.m_codeSectionList.size() == 1);
uint8_t* prologue = const_cast<uint8_t*>(reinterpret_cast<const uint8_t*>(state.m_codeSectionList.front().data()));
uint8_t* body = static_cast<uint8_t*>(state.m_entryPoint);
desc.m_patchPrologue(desc.m_opaque, prologue);
for (auto& record : rm) {
EMASSERT(record.second.size() == 1);
auto found = state.m_patchMap.find(record.first);
if (found == state.m_patchMap.end()) {
// should be the tcg helpers.
continue;
}
PatchDesc& patchDesc = found->second;
switch (patchDesc.m_type) {
case PatchType::Assist: {
desc.m_patchAssist(desc.m_opaque, body + record.second[0].instructionOffset, desc.m_dispAssist);
} break;
case PatchType::TcgDirect: {
auto& recordUnit = record.second[0];
desc.m_patchTcgDirect(desc.m_opaque, body + recordUnit.instructionOffset, desc.m_dispTcgDirect);
} break;
case PatchType::TcgIndirect: {
auto& recordUnit = record.second[0];
desc.m_patchTcgIndirect(desc.m_opaque, body + recordUnit.instructionOffset, desc.m_dispTcgIndirect);
} break;
default:
EMUNREACHABLE();
}
}
}
void AArch64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
const MachineInstr &MI) {
unsigned NumNOPBytes = MI.getOperand(1).getImm();
SM.recordStackMap(MI);
assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
// Scan ahead to trim the shadow.
const MachineBasicBlock &MBB = *MI.getParent();
MachineBasicBlock::const_iterator MII(MI);
++MII;
while (NumNOPBytes > 0) {
if (MII == MBB.end() || MII->isCall() ||
MII->getOpcode() == AArch64::DBG_VALUE ||
MII->getOpcode() == TargetOpcode::PATCHPOINT ||
MII->getOpcode() == TargetOpcode::STACKMAP)
break;
++MII;
NumNOPBytes -= 4;
}
// Emit nops.
for (unsigned i = 0; i < NumNOPBytes; i += 4)
EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
}
// Lower a patchpoint of the form:
// [<def>], <id>, <numBytes>, <target>, <numArgs>, <cc>, ...
static void LowerPATCHPOINT(MCStreamer &OS, StackMaps &SM,
const MachineInstr &MI, bool Is64Bit, const MCSubtargetInfo& STI) {
assert(Is64Bit && "Patchpoint currently only supports X86-64");
SM.recordPatchPoint(MI);
PatchPointOpers opers(&MI);
unsigned ScratchIdx = opers.getNextScratchIdx();
unsigned EncodedBytes = 0;
int64_t CallTarget = opers.getMetaOper(PatchPointOpers::TargetPos).getImm();
if (CallTarget) {
// Emit MOV to materialize the target address and the CALL to target.
// This is encoded with 12-13 bytes, depending on which register is used.
unsigned ScratchReg = MI.getOperand(ScratchIdx).getReg();
if (X86II::isX86_64ExtendedReg(ScratchReg))
EncodedBytes = 13;
else
EncodedBytes = 12;
OS.EmitInstruction(MCInstBuilder(X86::MOV64ri).addReg(ScratchReg)
.addImm(CallTarget), STI);
OS.EmitInstruction(MCInstBuilder(X86::CALL64r).addReg(ScratchReg), STI);
}
// Emit padding.
unsigned NumBytes = opers.getMetaOper(PatchPointOpers::NBytesPos).getImm();
assert(NumBytes >= EncodedBytes &&
"Patchpoint can't request size less than the length of a call.");
EmitNops(OS, NumBytes - EncodedBytes, Is64Bit, STI);
}
// Lower a stackmap of the form:
// <id>, <shadowBytes>, ...
static void LowerSTACKMAP(MCStreamer &OS, StackMaps &SM,
const MachineInstr &MI, bool Is64Bit, const MCSubtargetInfo& STI) {
unsigned NumBytes = MI.getOperand(1).getImm();
SM.recordStackMap(MI);
// Emit padding.
// FIXME: These nops ensure that the stackmap's shadow is covered by
// instructions from the same basic block, but the nops should not be
// necessary if instructions from the same block follow the stackmap.
EmitNops(OS, NumBytes, Is64Bit, STI);
}
void ARM64AsmPrinter::LowerSTACKMAP(MCStreamer &OutStreamer, StackMaps &SM,
const MachineInstr &MI) {
unsigned NumNOPBytes = MI.getOperand(1).getImm();
SM.recordStackMap(MI);
// Emit padding.
assert(NumNOPBytes % 4 == 0 && "Invalid number of NOP bytes requested!");
for (unsigned i = 0; i < NumNOPBytes; i += 4)
EmitToStreamer(OutStreamer, MCInstBuilder(ARM64::HINT).addImm(0));
}
void LLVMDisasContext::link()
{
StackMaps sm;
const LinkDesc desc = {
nullptr,
m_dispDirect,
m_dispIndirect,
patchProloge,
patchDirect,
patchIndirect,
};
DataView dv(state()->m_stackMapsSection->data());
sm.parse(&dv);
auto rm = sm.computeRecordMap();
EMASSERT(state()->m_codeSectionList.size() == 1);
uint8_t* prologue = state()->m_codeSectionList.front();
uint8_t* body = static_cast<uint8_t*>(state()->m_entryPoint);
desc.m_patchPrologue(desc.m_opaque, prologue);
for (auto& record : rm) {
auto found = state()->m_patchMap.find(record.first);
if (found == state()->m_patchMap.end()) {
// should be the tcg helpers.
continue;
}
PatchDesc& patchDesc = found->second;
switch (patchDesc.m_type) {
case PatchType::TcgDirect: {
for (auto& recordUnit : record.second) {
desc.m_patchTcgDirect(desc.m_opaque, body + recordUnit.instructionOffset, desc.m_dispTcgDirect);
}
} break;
case PatchType::TcgIndirect: {
for (auto& recordUnit : record.second) {
desc.m_patchTcgIndirect(desc.m_opaque, body + recordUnit.instructionOffset, desc.m_dispTcgIndirect);
}
} break;
default:
EMUNREACHABLE();
}
}
}
static void LowerSTATEPOINT(MCStreamer &OS, StackMaps &SM,
const MachineInstr &MI, bool Is64Bit,
const TargetMachine& TM,
const MCSubtargetInfo& STI,
X86MCInstLower &MCInstLowering) {
assert(Is64Bit && "Statepoint currently only supports X86-64");
// Lower call target and choose correct opcode
const MachineOperand &call_target = StatepointOpers(&MI).getCallTarget();
MCOperand call_target_mcop;
unsigned call_opcode;
switch (call_target.getType()) {
case MachineOperand::MO_GlobalAddress:
case MachineOperand::MO_ExternalSymbol:
call_target_mcop = MCInstLowering.LowerSymbolOperand(
call_target,
MCInstLowering.GetSymbolFromOperand(call_target));
call_opcode = X86::CALL64pcrel32;
// Currently, we only support relative addressing with statepoints.
// Otherwise, we'll need a scratch register to hold the target
// address. You'll fail asserts during load & relocation if this
// symbol is to far away. (TODO: support non-relative addressing)
break;
case MachineOperand::MO_Immediate:
call_target_mcop = MCOperand::CreateImm(call_target.getImm());
call_opcode = X86::CALL64pcrel32;
// Currently, we only support relative addressing with statepoints.
// Otherwise, we'll need a scratch register to hold the target
// immediate. You'll fail asserts during load & relocation if this
// address is to far away. (TODO: support non-relative addressing)
break;
case MachineOperand::MO_Register:
call_target_mcop = MCOperand::CreateReg(call_target.getReg());
call_opcode = X86::CALL64r;
break;
default:
llvm_unreachable("Unsupported operand type in statepoint call target");
break;
}
// Emit call
MCInst call_inst;
call_inst.setOpcode(call_opcode);
call_inst.addOperand(call_target_mcop);
OS.EmitInstruction(call_inst, STI);
// Record our statepoint node in the same section used by STACKMAP
// and PATCHPOINT
SM.recordStatepoint(MI);
}
// Lower a patchpoint of the form:
// [<def>], <id>, <numBytes>, <target>, <numArgs>
void AArch64AsmPrinter::LowerPATCHPOINT(MCStreamer &OutStreamer, StackMaps &SM,
const MachineInstr &MI) {
SM.recordPatchPoint(MI);
PatchPointOpers Opers(&MI);
int64_t CallTarget = Opers.getMetaOper(PatchPointOpers::TargetPos).getImm();
unsigned EncodedBytes = 0;
if (CallTarget) {
assert((CallTarget & 0xFFFFFFFFFFFF) == CallTarget &&
"High 16 bits of call target should be zero.");
unsigned ScratchReg = MI.getOperand(Opers.getNextScratchIdx()).getReg();
EncodedBytes = 16;
// Materialize the jump address:
EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVZXi)
.addReg(ScratchReg)
.addImm((CallTarget >> 32) & 0xFFFF)
.addImm(32));
EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
.addReg(ScratchReg)
.addReg(ScratchReg)
.addImm((CallTarget >> 16) & 0xFFFF)
.addImm(16));
EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::MOVKXi)
.addReg(ScratchReg)
.addReg(ScratchReg)
.addImm(CallTarget & 0xFFFF)
.addImm(0));
EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::BLR).addReg(ScratchReg));
}
// Emit padding.
unsigned NumBytes = Opers.getMetaOper(PatchPointOpers::NBytesPos).getImm();
assert(NumBytes >= EncodedBytes &&
"Patchpoint can't request size less than the length of a call.");
assert((NumBytes - EncodedBytes) % 4 == 0 &&
"Invalid number of NOP bytes requested!");
for (unsigned i = EncodedBytes; i < NumBytes; i += 4)
EmitToStreamer(OutStreamer, MCInstBuilder(AArch64::HINT).addImm(0));
}