/*
* emit mov <ofs>(%<sreg>), %<dreg>
* note that for non 64-bit ops, higher bits have to be cleared.
*/
static void
emit_ld_reg(struct bpf_jit_state *st, uint32_t op, uint32_t sreg, uint32_t dreg,
int32_t ofs)
{
uint32_t mods, opsz;
const uint8_t op32 = 0x8B;
const uint8_t op16[] = {0x0F, 0xB7};
const uint8_t op8[] = {0x0F, 0xB6};
emit_rex(st, op, dreg, sreg);
opsz = BPF_SIZE(op);
if (opsz == BPF_B)
emit_bytes(st, op8, sizeof(op8));
else if (opsz == BPF_H)
emit_bytes(st, op16, sizeof(op16));
else
emit_bytes(st, &op32, sizeof(op32));
mods = (imm_size(ofs) == 1) ? MOD_IDISP8 : MOD_IDISP32;
emit_modregrm(st, mods, dreg, sreg);
if (sreg == RSP || sreg == R12)
emit_sib(st, SIB_SCALE_1, sreg, sreg);
emit_imm(st, ofs, imm_size(ofs));
}
/*
* emit jmp <ofs>
* where 'ofs' is the target offset for the native code.
*/
static void
emit_abs_jmp(struct bpf_jit_state *st, int32_t ofs)
{
int32_t joff;
uint32_t imsz;
const uint8_t op8 = 0xEB;
const uint8_t op32 = 0xE9;
const int32_t sz8 = sizeof(op8) + sizeof(uint8_t);
const int32_t sz32 = sizeof(op32) + sizeof(uint32_t);
/* max possible jmp instruction size */
const int32_t iszm = RTE_MAX(sz8, sz32);
joff = ofs - st->sz;
imsz = RTE_MAX(imm_size(joff), imm_size(joff + iszm));
if (imsz == 1) {
emit_bytes(st, &op8, sizeof(op8));
joff -= sz8;
} else {
emit_bytes(st, &op32, sizeof(op32));
joff -= sz32;
}
emit_imm(st, joff, imsz);
}
/*
* emit one of:
* shl <imm>, %<dreg>
* shr <imm>, %<dreg>
* sar <imm>, %<dreg>
*/
static void
emit_shift_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg,
uint32_t imm)
{
emit_shift(st, op, dreg);
emit_imm(st, imm, imm_size(imm));
}
/*
* emit one of:
* add <imm>, %<dreg>
* and <imm>, %<dreg>
* or <imm>, %<dreg>
* sub <imm>, %<dreg>
* xor <imm>, %<dreg>
*/
static void
emit_alu_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
{
uint8_t mod, opcode;
uint32_t bop, imsz;
const uint8_t op8 = 0x83;
const uint8_t op32 = 0x81;
static const uint8_t mods[] = {
[GET_BPF_OP(BPF_ADD)] = 0,
[GET_BPF_OP(BPF_AND)] = 4,
[GET_BPF_OP(BPF_OR)] = 1,
[GET_BPF_OP(BPF_SUB)] = 5,
[GET_BPF_OP(BPF_XOR)] = 6,
};
bop = GET_BPF_OP(op);
mod = mods[bop];
imsz = imm_size(imm);
opcode = (imsz == 1) ? op8 : op32;
emit_rex(st, op, 0, dreg);
emit_bytes(st, &opcode, sizeof(opcode));
emit_modregrm(st, MOD_DIRECT, mod, dreg);
emit_imm(st, imm, imsz);
}
/*
* emit mov <imm64>, %<dreg>
*/
static void
emit_ld_imm64(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm0,
uint32_t imm1)
{
const uint8_t ops = 0xB8;
if (imm1 == 0) {
emit_mov_imm(st, EBPF_ALU64 | EBPF_MOV | BPF_K, dreg, imm0);
return;
}
emit_rex(st, EBPF_ALU64, 0, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, 0, dreg);
emit_imm(st, imm0, sizeof(imm0));
emit_imm(st, imm1, sizeof(imm1));
}
/*
* emit one of:
* mov %<sreg>, <ofs>(%<dreg>)
* mov <imm>, <ofs>(%<dreg>)
*/
static void
emit_st_common(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
uint32_t dreg, uint32_t imm, int32_t ofs)
{
uint32_t mods, imsz, opsz, opx;
const uint8_t prfx16 = 0x66;
/* 8 bit instruction opcodes */
static const uint8_t op8[] = {0xC6, 0x88};
/* 16/32/64 bit instruction opcodes */
static const uint8_t ops[] = {0xC7, 0x89};
/* is the instruction has immediate value or src reg? */
opx = (BPF_CLASS(op) == BPF_STX);
opsz = BPF_SIZE(op);
if (opsz == BPF_H)
emit_bytes(st, &prfx16, sizeof(prfx16));
emit_rex(st, op, sreg, dreg);
if (opsz == BPF_B)
emit_bytes(st, &op8[opx], sizeof(op8[opx]));
else
emit_bytes(st, &ops[opx], sizeof(ops[opx]));
imsz = imm_size(ofs);
mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
emit_modregrm(st, mods, sreg, dreg);
if (dreg == RSP || dreg == R12)
emit_sib(st, SIB_SCALE_1, dreg, dreg);
emit_imm(st, ofs, imsz);
if (opx == 0) {
imsz = RTE_MIN(bpf_size(opsz), sizeof(imm));
emit_imm(st, imm, imsz);
}
}
static void __emit_push_imm(struct buffer *buf, long imm)
{
unsigned char opc;
if (is_imm_8(imm))
opc = 0x6a;
else
opc = 0x68;
emit(buf, opc);
emit_imm(buf, imm);
}
/*
* emit ror <imm8>, %<dreg>
*/
static void
emit_ror_imm(struct bpf_jit_state *st, uint32_t dreg, uint32_t imm)
{
const uint8_t prfx = 0x66;
const uint8_t ops = 0xC1;
const uint8_t mods = 1;
emit_bytes(st, &prfx, sizeof(prfx));
emit_rex(st, BPF_ALU, 0, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, mods, dreg);
emit_imm(st, imm, imm_size(imm));
}
/*
* emit mov <imm>, %<dreg>
*/
static void
emit_mov_imm(struct bpf_jit_state *st, uint32_t op, uint32_t dreg, uint32_t imm)
{
const uint8_t ops = 0xC7;
if (imm == 0) {
/* replace 'mov 0, %<dst>' with 'xor %<dst>, %<dst>' */
op = BPF_CLASS(op) | BPF_XOR | BPF_X;
emit_alu_reg(st, op, dreg, dreg);
return;
}
emit_rex(st, op, 0, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, MOD_DIRECT, 0, dreg);
emit_imm(st, imm, sizeof(imm));
}
/*
* emit lock add %<sreg>, <ofs>(%<dreg>)
*/
static void
emit_st_xadd(struct bpf_jit_state *st, uint32_t op, uint32_t sreg,
uint32_t dreg, int32_t ofs)
{
uint32_t imsz, mods;
const uint8_t lck = 0xF0; /* lock prefix */
const uint8_t ops = 0x01; /* add opcode */
imsz = imm_size(ofs);
mods = (imsz == 1) ? MOD_IDISP8 : MOD_IDISP32;
emit_bytes(st, &lck, sizeof(lck));
emit_rex(st, op, sreg, dreg);
emit_bytes(st, &ops, sizeof(ops));
emit_modregrm(st, mods, sreg, dreg);
emit_imm(st, ofs, imsz);
}