/*
* emit mov <ofs>(%<sreg>), %<dreg>
* note that for non 64-bit ops, higher bits have to be cleared.
*/
static void
emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
int32_t ofs)
{
uint32_t mods, opsz;
const uint8_t op32 = 0x8B;
const uint8_t op16[] = {0x0F, 0xB7};
const uint8_t op8[] = {0x0F, 0xB6};
emit_rex(st, op, dreg, sreg);
opsz = BPF_SIZE(op);
if (opsz == BPF_B)
emit_bytes(st, op8, sizeof(op8));
else if (opsz == BPF_H)
emit_bytes(st, op16, sizeof(op16));
else
emit_bytes(st, &op32, sizeof(op32));
mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32;
emit_modregrm(st, mods, dreg, sreg);
if (sreg == RSP || sreg == R12)
emit_sib(st, SIB_SCALE_1, sreg, sreg);
emit_imm(st, ofs, imm_size(ofs));
}
/*
* emit jmp <ofs>
* where 'ofs' is the target offset for the native code.
*/
static void
emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs)
{
int32_t joff;
uint32_t imsz;
const uint8_t op8 = 0xEB;
const uint8_t op32 = 0xE9;
const int32_t sz8 = sizeof(op8) + sizeof(uint8_t);
const int32_t sz32 = sizeof(op32) + sizeof(uint32_t);
/* max possible jmp instruction size */
const int32_t iszm = RTE_MAX(sz8, sz32);
joff = ofs - st->sz;
imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
if (imsz == 1) {
emit_bytes(st, &op8, sizeof(op8));
joff -= sz8;
} else {
emit_bytes(st, &op32, sizeof(op32));
joff -= sz32;
}
emit_imm(st, joff, imsz);
}
/*
* emit ror <imm8>, %<dreg>
*/
static void
emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
{
const uint8_t prfx = 0x66;
const uint8_t ops = 0xC1;
const uint8_t mods = 1;
emit_bytes(st, &prfx, sizeof(prfx));
emit_rex(st, BPF_ALU, 0, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, mods, dreg);
emit_imm(st, imm, imm_size(imm));
}
/*
* emit REX byte
*/
static void
emit_rex(struct bpf_jit_state *st, uint32_t op, uint32_t reg, uint32_t rm)
{
uint8_t rex;
/* mark operand registers as used*/
USED(st->reguse, reg);
USED(st->reguse, rm);
rex = 0;
if (BPF_CLASS(op) == EBPF_ALU64 ||
op == (BPF_ST | BPF_MEM | EBPF_DW) ||
op == (BPF_STX | BPF_MEM | EBPF_DW) ||
op == (BPF_STX | EBPF_XADD | EBPF_DW) ||
op == (BPF_LD | BPF_IMM | EBPF_DW) ||
(BPF_CLASS(op) == BPF_LDX &&
BPF_MODE(op) == BPF_MEM &&
BPF_SIZE(op) != BPF_W))
rex |= REX_W;
if (IS_EXT_REG(reg))
rex |= REX_R;
if (IS_EXT_REG(rm))
rex |= REX_B;
/* store using SIL, DIL */
if (op == (BPF_STX | BPF_MEM | BPF_B) && (reg == RDI || reg == RSI))
rex |= REX_PREFIX;
if (rex != 0) {
rex |= REX_PREFIX;
emit_bytes(st, &rex, sizeof(rex));
}
}
/*
* emit one of:
* add <imm>, %<dreg>
* and <imm>, %<dreg>
* or <imm>, %<dreg>
* sub <imm>, %<dreg>
* xor <imm>, %<dreg>
*/
static void
emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
{
uint8_t mod, opcode;
uint32_t bop, imsz;
const uint8_t op8 = 0x83;
const uint8_t op32 = 0x81;
static const uint8_t mods[] = {
[GET_BPF_OP(BPF_ADD)] = 0,
[GET_BPF_OP(BPF_AND)] = 4,
[GET_BPF_OP(BPF_OR)] = 1,
[GET_BPF_OP(BPF_SUB)] = 5,
[GET_BPF_OP(BPF_XOR)] = 6,
};
bop = GET_BPF_OP(op);
mod = mods[bop];
imsz = imm_size(imm);
opcode = (imsz == 1) ? op8 : op32;
emit_rex(st, op, 0, dreg);
emit_bytes(st, &opcode, sizeof(opcode));
emit_modregrm(st, MOD_DIRECT, mod, dreg);
emit_imm(st, imm, imsz);
}
/*
* emit SIB byte
*/
static void
emit_sib(struct bpf_jit_state *st, uint32_t scale, uint32_t idx, uint32_t base)
{
uint8_t v;
v = scale << 6 | (idx & 7) << 3 | (base & 7);
emit_bytes(st, &v, sizeof(v));
}
/*
* emit MODRegRM byte
*/
static void
emit_modregrm(struct bpf_jit_state *st, uint32_t mod, uint32_t reg, uint32_t rm)
{
uint8_t v;
v = mod << 6 | (reg & 7) << 3 | (rm & 7);
emit_bytes(st, &v, sizeof(v));
}
static void
emit_imm(struct bpf_jit_state *st, const uint32_t imm, uint32_t sz)
{
union bpf_jit_imm v;
v.u32 = imm;
emit_bytes(st, v.u8, sz);
}
/*
* emit lock add %<sreg>, <ofs>(%<dreg>)
*/
static void
emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
uint32_t dreg, int32_t ofs)
{
uint32_t imsz, mods;
const uint8_t lck = 0xF0; /* lock prefix */
const uint8_t ops = 0x01; /* add opcode */
imsz = imm_size(ofs);
mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
emit_bytes(st, &lck, sizeof(lck));
emit_rex(st, op, sreg, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, mods, sreg, dreg);
emit_imm(st, ofs, imsz);
}
/*
* emit movzwl %<sreg>, %<dreg>
*/
static void
emit_movzwl(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
{
static const uint8_t ops[] = {0x0F, 0xB7};
emit_rex(st, BPF_ALU, sreg, dreg);
emit_bytes(st, ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, sreg, dreg);
}
/*
* emit xchg %<sreg>, %<dreg>
*/
static void
emit_xchg_reg(struct bpf_jit_state *st, uint32_t sreg, uint32_t dreg)
{
const uint8_t ops = 0x87;
emit_rex(st, EBPF_ALU64, sreg, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, sreg, dreg);
}
/*
* emit neg %<dreg>
*/
static void
emit_neg(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
{
const uint8_t ops = 0xF7;
const uint8_t mods = 3;
emit_rex(st, op, 0, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, mods, dreg);
}
/*
* emit:
* mov <imm64>, (%rax)
* call *%rax
*/
static void
emit_call(struct bpf_jit_state *st, uintptr_t trg)
{
const uint8_t ops = 0xFF;
const uint8_t mods = 2;
emit_ld_imm64(st, RAX, trg, trg >> 32);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, mods, RAX);
}
/*
* emit one of:
* mov %<sreg>, <ofs>(%<dreg>)
* mov <imm>, <ofs>(%<dreg>)
*/
static void
emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
uint32_t dreg, uint32_t imm, int32_t ofs)
{
uint32_t mods, imsz, opsz, opx;
const uint8_t prfx16 = 0x66;
/* 8 bit instruction opcodes */
static const uint8_t op8[] = {0xC6, 0x88};
/* 16/32/64 bit instruction opcodes */
static const uint8_t ops[] = {0xC7, 0x89};
/* is the instruction has immediate value or src reg? */
opx = (BPF_CLASS(op) == BPF_STX);
opsz = BPF_SIZE(op);
if (opsz == BPF_H)
emit_bytes(st, &prfx16, sizeof(prfx16));
emit_rex(st, op, sreg, dreg);
if (opsz == BPF_B)
emit_bytes(st, &op8[opx], sizeof(op8[opx]));
else
emit_bytes(st, &ops[opx], sizeof(ops[opx]));
imsz = imm_size(ofs);
mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
emit_modregrm(st, mods, sreg, dreg);
if (dreg == RSP || dreg == R12)
emit_sib(st, SIB_SCALE_1, dreg, dreg);
emit_imm(st, ofs, imsz);
if (opx == 0) {
imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
emit_imm(st, imm, imsz);
}
}
/*
* emit bswap %<dreg>
*/
static void
emit_be2le_48(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
{
uint32_t rop;
const uint8_t ops = 0x0F;
const uint8_t mods = 1;
rop = (imm == 64) ? EBPF_ALU64 : BPF_ALU;
emit_rex(st, rop, 0, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, mods, dreg);
}
/*
* emit mov %<sreg>, %<dreg>
*/
static void
emit_mov_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
uint32_t dreg)
{
const uint8_t ops = 0x89;
/* if operands are 32-bit, then it can be used to clear upper 32-bit */
if (sreg != dreg || BPF_CLASS(op) == BPF_ALU) {
emit_rex(st, op, sreg, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, sreg, dreg);
}
}
/*
* emit mov <imm>, %<dreg>
*/
static void
emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
{
const uint8_t ops = 0xC7;
if (imm == 0) {
/* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */
op = BPF_CLASS(op) | BPF_XOR | BPF_X;
emit_alu_reg(st, op, dreg, dreg);
return;
}
emit_rex(st, op, 0, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, 0, dreg);
emit_imm(st, imm, sizeof(imm));
}
/*
* emit mov <imm64>, %<dreg>
*/
static void
emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
uint32_t imm1)
{
const uint8_t ops = 0xB8;
if (imm1 == 0) {
emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, dreg, imm0);
return;
}
emit_rex(st, EBPF_ALU64, 0, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, 0, dreg);
emit_imm(st, imm0, sizeof(imm0));
emit_imm(st, imm1, sizeof(imm1));
}
/*
* emit one of:
* add %<sreg>, %<dreg>
* and %<sreg>, %<dreg>
* or %<sreg>, %<dreg>
* sub %<sreg>, %<dreg>
* xor %<sreg>, %<dreg>
*/
static void
emit_alu_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
uint32_t dreg)
{
uint32_t bop;
static const uint8_t ops[] = {
[GET_BPF_OP(BPF_ADD)] = 0x01,
[GET_BPF_OP(BPF_AND)] = 0x21,
[GET_BPF_OP(BPF_OR)] = 0x09,
[GET_BPF_OP(BPF_SUB)] = 0x29,
[GET_BPF_OP(BPF_XOR)] = 0x31,
};
bop = GET_BPF_OP(op);
emit_rex(st, op, sreg, dreg);
emit_bytes(st, &ops[bop], sizeof(ops[bop]));
emit_modregrm(st, MOD_DIRECT, sreg, dreg);
}
static void
emit_shift(struct bpf_jit_state *st, uint32_t op, uint32_t dreg)
{
uint8_t mod;
uint32_t bop, opx;
static const uint8_t ops[] = {0xC1, 0xD3};
static const uint8_t mods[] = {
[GET_BPF_OP(BPF_LSH)] = 4,
[GET_BPF_OP(BPF_RSH)] = 5,
[GET_BPF_OP(EBPF_ARSH)] = 7,
};
bop = GET_BPF_OP(op);
mod = mods[bop];
opx = (BPF_SRC(op) == BPF_X);
emit_rex(st, op, 0, dreg);
emit_bytes(st, &ops[opx], sizeof(ops[opx]));
emit_modregrm(st, MOD_DIRECT, mod, dreg);
}
/*
* note that rax:rdx are implicitly used as source/destination registers,
* so some reg spillage is necessary.
* emit:
* mov %rax, %r11
* mov %rdx, %r10
* mov %<dreg>, %rax
* either:
* mov %<sreg>, %rdx
* OR
* mov <imm>, %rdx
* mul %rdx
* mov %r10, %rdx
* mov %rax, %<dreg>
* mov %r11, %rax
*/
static void
emit_mul(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
uint32_t imm)
{
const uint8_t ops = 0xF7;
const uint8_t mods = 4;
/* save rax & rdx */
emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, REG_TMP0);
emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RDX, REG_TMP1);
/* rax = dreg */
emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, dreg, RAX);
if (BPF_SRC(op) == BPF_X)
/* rdx = sreg */
emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X,
sreg == RAX ? REG_TMP0 : sreg, RDX);
else
/* rdx = imm */
emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, RDX, imm);
emit_rex(st, op, RAX, RDX);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, mods, RDX);
if (dreg != RDX)
/* restore rdx */
emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP1, RDX);
if (dreg != RAX) {
/* dreg = rax */
emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, RAX, dreg);
/* restore rax */
emit_mov_reg(st, EBPF_ALU64 | EBPF_MOV | BPF_X, REG_TMP0, RAX);
}
}
static void emit_constant(Value value)
{
emit_bytes(OP_CONSTANT, make_constant(value));
}
