static void
decode_bpf_code(uint16_t code)
{
uint16_t i = code & ~BPF_CLASS(code);
printxval(bpf_class, BPF_CLASS(code), "BPF_???");
switch (BPF_CLASS(code)) {
case BPF_LD:
case BPF_LDX:
tprints(" | ");
printxval(bpf_size, BPF_SIZE(code), "BPF_???");
tprints(" | ");
printxval(bpf_mode, BPF_MODE(code), "BPF_???");
break;
case BPF_ST:
case BPF_STX:
if (i)
tprintf(" | %#x /* %s */", i, "BPF_???");
break;
case BPF_ALU:
tprints(" | ");
printxval(bpf_src, BPF_SRC(code), "BPF_???");
tprints(" | ");
printxval(bpf_op_alu, BPF_OP(code), "BPF_???");
break;
case BPF_JMP:
tprints(" | ");
printxval(bpf_src, BPF_SRC(code), "BPF_???");
tprints(" | ");
printxval(bpf_op_jmp, BPF_OP(code), "BPF_???");
break;
case BPF_RET:
tprints(" | ");
printxval(bpf_rval, BPF_RVAL(code), "BPF_???");
i &= ~BPF_RVAL(code);
if (i)
tprintf(" | %#x /* %s */", i, "BPF_???");
break;
case BPF_MISC:
tprints(" | ");
printxval(bpf_miscop, BPF_MISCOP(code), "BPF_???");
i &= ~BPF_MISCOP(code);
if (i)
tprintf(" | %#x /* %s */", i, "BPF_???");
break;
}
}
void flattenGraphTraversal(struct flatten_item * graph, struct bpf_program * filter, int indice, int level)
{
int j;
struct bpf_insn *insn = &filter->bf_insns[ graph[indice].bf_instruct_link ];
if(level > 50)
return;
for(j = 0;j< level;j++)
{
printf(" ");
}
printf("(%d : %d) fnode (code:%u - k:%u)\n",level, indice, insn->code, insn->k);
/*printf("(%d) %s\n",level, bpf_image(insn,indice));*/
if(BPF_CLASS(insn->code) == BPF_JMP)
{
if(BPF_OP(insn->code) == BPF_JA)
{
flattenGraphTraversal(graph, filter, graph[indice].true_instruct, level+1);
}
else
{
flattenGraphTraversal(graph, filter, graph[indice].true_instruct, level+1);
flattenGraphTraversal(graph, filter, graph[indice].false_instruct, level+1);
}
}
else if(BPF_CLASS(insn->code) != BPF_RET)
{
flattenGraphTraversal(graph, filter, graph[indice].true_instruct, level+1);
}
}
static bpf_insn op2insn_alu(unsigned opcode)
{
/* Endian is a special case */
if (BPF_OP(opcode) == BPF_ALU_END) {
switch (opcode ^ BPF_CLASS_ALU ^ BPF_ALU_END) {
case BPF_SRC_LITTLE | (16 << 4):
return BPF_INS_LE16;
case BPF_SRC_LITTLE | (32 << 4):
return BPF_INS_LE32;
case BPF_SRC_LITTLE | (64 << 4):
return BPF_INS_LE64;
case BPF_SRC_BIG | (16 << 4):
return BPF_INS_BE16;
case BPF_SRC_BIG | (32 << 4):
return BPF_INS_BE32;
case BPF_SRC_BIG | (64 << 4):
return BPF_INS_BE64;
}
return BPF_INS_INVALID;
}
#define CASE(c) case BPF_ALU_##c: \
if (BPF_CLASS(opcode) == BPF_CLASS_ALU) \
return BPF_INS_##c; \
else \
return BPF_INS_##c##64;
switch (BPF_OP(opcode)) {
CASE(ADD);
CASE(SUB);
CASE(MUL);
CASE(DIV);
CASE(OR);
CASE(AND);
CASE(LSH);
CASE(RSH);
CASE(NEG);
CASE(MOD);
CASE(XOR);
CASE(MOV);
CASE(ARSH);
}
#undef CASE
return BPF_INS_INVALID;
}
/*
* cette fonction compare le graphe existant avec le nouveau graphe applati
*
* @param graph : le graphe applati
* @param filter : le graphe bpf correspondant
* @param indice : l'indice du noeud a comparer
* @param node : la liste de noeud pouvant contenir un noeud similaire
* @return : le nombre de noeud equivalent dans l'arbre
*
*/
unsigned int compareGraphExploreChild(struct flatten_item * graph, struct bpf_program * filter, int indice, struct filter_node * node,unsigned int * collide)
{
int i = graph[indice].bf_instruct_link;
unsigned int local_collide = 0, child_collide_true = 0, child_collide_false = 0;
unsigned int ret = 0;
struct filter_node * current_node = node;
/*recherche d'un noeud racine correspondant*/
for( ; current_node != NULL ; current_node = current_node->next)
{
if(current_node->item->code == filter->bf_insns[i].code && current_node->item->k == filter->bf_insns[i].k)
{
break;
}
}
/*pas de noeud correspondant, on devra insérer ici*/
if(current_node == NULL)
{
if(indice > 0 && node!= NULL)
{
/*on a un noeud pouvant causer un breakpoint*/
local_collide = 1;
}
*collide = local_collide;
return 0;
}
else
{
if(indice > 0 && node != NULL && node->next!=NULL)
{
local_collide = 1;
}
}
if(BPF_CLASS(filter->bf_insns[i].code) == BPF_JMP && BPF_OP(filter->bf_insns[i].code) != BPF_JA) /*noeud conditionnel*/
{
ret = 1 + compareGraphExploreChild(graph, filter, graph[indice].true_instruct, current_node->item->next_child_t,&child_collide_true)
+ compareGraphExploreChild(graph, filter, graph[indice].false_instruct, current_node->item->next_child_f, &child_collide_false);
}
else if(BPF_CLASS(filter->bf_insns[i].code) != BPF_RET) /*noeud normal*/
{
ret = 1 + compareGraphExploreChild(graph, filter, graph[indice].true_instruct, current_node->item->next_child_t,&child_collide_true);
}
/*else
{
on ne se soucie pas des noeuds terminaux, ils seront ajoutés quoi qu'il arrive, c'est necessaire pour la suppression
}*/
*collide = local_collide + (child_collide_true > child_collide_false)?child_collide_true:child_collide_false;
return ret;
}
/*
* Do constant-folding on binary operators.
* (Unary operators are handled elsewhere.)
*/
static void
fold_op(struct stmt *s, int v0, int v1)
{
bpf_u_int32 a, b;
a = vmap[v0].const_val;
b = vmap[v1].const_val;
switch (BPF_OP(s->code)) {
case BPF_ADD:
a += b;
break;
case BPF_SUB:
a -= b;
break;
case BPF_MUL:
a *= b;
break;
case BPF_DIV:
if (b == 0)
bpf_error("division by zero");
a /= b;
break;
case BPF_AND:
a &= b;
break;
case BPF_OR:
a |= b;
break;
case BPF_LSH:
a <<= b;
break;
case BPF_RSH:
a >>= b;
break;
default:
abort();
}
s->k = a;
s->code = BPF_LD|BPF_IMM;
done = 0;
}
/**
* sk_chk_filter - verify socket filter code
* @filter: filter to verify
* @flen: length of filter
*
* Check the user's filter code. If we let some ugly
* filter code slip through kaboom! The filter must contain
* no references or jumps that are out of range, no illegal instructions
* and no backward jumps. It must end with a RET instruction
*
* Returns 0 if the rule set is legal or a negative errno code if not.
*/
int sk_chk_filter(struct sock_filter *filter, int flen)
{
struct sock_filter *ftest;
int pc;
if (((unsigned int)flen >= (~0U / sizeof(struct sock_filter))) || flen == 0)
return -EINVAL;
/* check the filter code now */
for (pc = 0; pc < flen; pc++) {
/* all jumps are forward as they are not signed */
ftest = &filter[pc];
if (BPF_CLASS(ftest->code) == BPF_JMP) {
/* but they mustn't jump off the end */
if (BPF_OP(ftest->code) == BPF_JA) {
/*
* Note, the large ftest->k might cause loops.
* Compare this with conditional jumps below,
* where offsets are limited. --ANK (981016)
*/
if (ftest->k >= (unsigned)(flen-pc-1))
return -EINVAL;
} else {
/* for conditionals both must be safe */
if (pc + ftest->jt +1 >= flen ||
pc + ftest->jf +1 >= flen)
return -EINVAL;
}
}
/* check that memory operations use valid addresses. */
if (ftest->k >= BPF_MEMWORDS) {
/* but it might not be a memory operation... */
switch (ftest->code) {
case BPF_ST:
case BPF_STX:
case BPF_LD|BPF_MEM:
case BPF_LDX|BPF_MEM:
return -EINVAL;
}
}
}
/*
* The program must end with a return. We don't care where they
* jumped within the script (its always forwards) but in the end
* they _will_ hit this.
*/
return (BPF_CLASS(filter[flen - 1].code) == BPF_RET) ? 0 : -EINVAL;
}
/*
* cette fonction met a plat un graphe bpf
*
* @param graph : le graphe applati à remplir
* @param filter : le graphe bpf à convertir
* @param non_terminal_count : le nombre de noeud non terminaux dans le graphe applati
* @param terminal_count : le nombre de noeud terminaux dans le graphe
* @param i : l'indice de l'instruction dans le graphe bpf
* @param id : le prochain identifiant disponible pour ajouter un noeud
* @return : l'id du noeud ajouté dans l'appel de la fonction
*/
int flatteGraph(struct flatten_item * graph, struct bpf_program * filter,int non_terminal_count, int terminal_count, int i, int * id)
{
int current_id, indice;
if(BPF_CLASS(filter->bf_insns[i].code) == BPF_JMP) /*JUMP INSTRUCTION*/
{
current_id = (*id)++;
graph[current_id].bf_instruct_link = i;
if(BPF_OP(filter->bf_insns[i].code) == BPF_JA)
{
graph[current_id].true_instruct = flatteGraph(graph, filter, non_terminal_count, terminal_count, filter->bf_insns[i].k+i+1,id);
}
else
{
graph[current_id].true_instruct = flatteGraph(graph, filter, non_terminal_count, terminal_count, filter->bf_insns[i].jt+i+1, id);
graph[current_id].false_instruct = flatteGraph(graph, filter, non_terminal_count, terminal_count, filter->bf_insns[i].jf+i+1, id);
}
}
else if(BPF_CLASS(filter->bf_insns[i].code) != BPF_RET) /*OTHER INSTRUCTION*/
{
current_id = (*id)++;
graph[current_id].bf_instruct_link = i;
graph[current_id].true_instruct = flatteGraph(graph, filter, non_terminal_count, terminal_count, i+1,id);
}
else /*TERMINAL INSTRUCTION*/
{
for(indice = 0 ; (indice<terminal_count) && ( graph[non_terminal_count + indice].bf_instruct_link != -1) ; indice++)
{
if(graph[non_terminal_count + indice].bf_instruct_link == i)
{
graph[non_terminal_count + indice].true_instruct += 1;
return non_terminal_count + indice;
}
}
/*noeud terminal non encore recontré, on l'ajoute*/
current_id = non_terminal_count + indice;
graph[current_id].bf_instruct_link = i;
graph[current_id].true_instruct = 1;
graph[current_id].false_instruct = indice;
}
return current_id;
}
/*
* cette methode explore le graphe existant et compte le nombre de noeud NON TERMINAUX necessaire pour la mise a plat du graphe
*/
unsigned int countNoTerminalNodes(struct bpf_program * filter, int i)
{
if(BPF_CLASS(filter->bf_insns[i].code) == BPF_JMP)
{
if(BPF_OP(filter->bf_insns[i].code) == BPF_JA)
{
return 1 + countNoTerminalNodes(filter,filter->bf_insns[i].k+i+1);
}
else
{
return 1 + countNoTerminalNodes(filter,filter->bf_insns[i].jt+i+1)
+ countNoTerminalNodes(filter,filter->bf_insns[i].jf+i+1);
}
}
else if(BPF_CLASS(filter->bf_insns[i].code) != BPF_RET)
{
return 1 + countNoTerminalNodes(filter,i+1);
}
/*on ne compte pas les noeuds terminaux ici*/
return 0;
}
static bpf_insn op2insn_jmp(unsigned opcode)
{
#define CASE(c) case BPF_JUMP_##c: return BPF_INS_##c
switch (BPF_OP(opcode)) {
case BPF_JUMP_JA:
return BPF_INS_JMP;
CASE(JEQ);
CASE(JGT);
CASE(JGE);
CASE(JSET);
CASE(JNE);
CASE(JSGT);
CASE(JSGE);
CASE(CALL);
CASE(EXIT);
CASE(JLT);
CASE(JLE);
CASE(JSLT);
CASE(JSLE);
}
#undef CASE
return BPF_INS_INVALID;
}
static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
int oldproglen, struct jit_context *ctx)
{
struct bpf_insn *insn = bpf_prog->insnsi;
int insn_cnt = bpf_prog->len;
bool seen_exit = false;
u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
int i, cnt = 0;
int proglen = 0;
u8 *prog = temp;
emit_prologue(&prog, bpf_prog->aux->stack_depth,
bpf_prog_was_classic(bpf_prog));
for (i = 0; i < insn_cnt; i++, insn++) {
const s32 imm32 = insn->imm;
u32 dst_reg = insn->dst_reg;
u32 src_reg = insn->src_reg;
u8 b2 = 0, b3 = 0;
s64 jmp_offset;
u8 jmp_cond;
int ilen;
u8 *func;
switch (insn->code) {
/* ALU */
case BPF_ALU | BPF_ADD | BPF_X:
case BPF_ALU | BPF_SUB | BPF_X:
case BPF_ALU | BPF_AND | BPF_X:
case BPF_ALU | BPF_OR | BPF_X:
case BPF_ALU | BPF_XOR | BPF_X:
case BPF_ALU64 | BPF_ADD | BPF_X:
case BPF_ALU64 | BPF_SUB | BPF_X:
case BPF_ALU64 | BPF_AND | BPF_X:
case BPF_ALU64 | BPF_OR | BPF_X:
case BPF_ALU64 | BPF_XOR | BPF_X:
switch (BPF_OP(insn->code)) {
case BPF_ADD: b2 = 0x01; break;
case BPF_SUB: b2 = 0x29; break;
case BPF_AND: b2 = 0x21; break;
case BPF_OR: b2 = 0x09; break;
case BPF_XOR: b2 = 0x31; break;
}
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_2mod(0x48, dst_reg, src_reg));
else if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT1(add_2mod(0x40, dst_reg, src_reg));
EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));
break;
case BPF_ALU64 | BPF_MOV | BPF_X:
case BPF_ALU | BPF_MOV | BPF_X:
emit_mov_reg(&prog,
BPF_CLASS(insn->code) == BPF_ALU64,
dst_reg, src_reg);
break;
/* neg dst */
case BPF_ALU | BPF_NEG:
case BPF_ALU64 | BPF_NEG:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
EMIT2(0xF7, add_1reg(0xD8, dst_reg));
break;
case BPF_ALU | BPF_ADD | BPF_K:
case BPF_ALU | BPF_SUB | BPF_K:
case BPF_ALU | BPF_AND | BPF_K:
case BPF_ALU | BPF_OR | BPF_K:
case BPF_ALU | BPF_XOR | BPF_K:
case BPF_ALU64 | BPF_ADD | BPF_K:
case BPF_ALU64 | BPF_SUB | BPF_K:
case BPF_ALU64 | BPF_AND | BPF_K:
case BPF_ALU64 | BPF_OR | BPF_K:
case BPF_ALU64 | BPF_XOR | BPF_K:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
/*
* b3 holds 'normal' opcode, b2 short form only valid
* in case dst is eax/rax.
*/
switch (BPF_OP(insn->code)) {
case BPF_ADD:
b3 = 0xC0;
b2 = 0x05;
break;
case BPF_SUB:
b3 = 0xE8;
b2 = 0x2D;
break;
case BPF_AND:
b3 = 0xE0;
b2 = 0x25;
break;
case BPF_OR:
b3 = 0xC8;
b2 = 0x0D;
break;
case BPF_XOR:
b3 = 0xF0;
b2 = 0x35;
break;
}
if (is_imm8(imm32))
EMIT3(0x83, add_1reg(b3, dst_reg), imm32);
else if (is_axreg(dst_reg))
EMIT1_off32(b2, imm32);
else
EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32);
break;
case BPF_ALU64 | BPF_MOV | BPF_K:
case BPF_ALU | BPF_MOV | BPF_K:
emit_mov_imm32(&prog, BPF_CLASS(insn->code) == BPF_ALU64,
dst_reg, imm32);
break;
case BPF_LD | BPF_IMM | BPF_DW:
emit_mov_imm64(&prog, dst_reg, insn[1].imm, insn[0].imm);
insn++;
i++;
break;
/* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
case BPF_ALU | BPF_MOD | BPF_X:
case BPF_ALU | BPF_DIV | BPF_X:
case BPF_ALU | BPF_MOD | BPF_K:
case BPF_ALU | BPF_DIV | BPF_K:
case BPF_ALU64 | BPF_MOD | BPF_X:
case BPF_ALU64 | BPF_DIV | BPF_X:
case BPF_ALU64 | BPF_MOD | BPF_K:
case BPF_ALU64 | BPF_DIV | BPF_K:
EMIT1(0x50); /* push rax */
EMIT1(0x52); /* push rdx */
if (BPF_SRC(insn->code) == BPF_X)
/* mov r11, src_reg */
EMIT_mov(AUX_REG, src_reg);
else
/* mov r11, imm32 */
EMIT3_off32(0x49, 0xC7, 0xC3, imm32);
/* mov rax, dst_reg */
EMIT_mov(BPF_REG_0, dst_reg);
/*
* xor edx, edx
* equivalent to 'xor rdx, rdx', but one byte less
*/
EMIT2(0x31, 0xd2);
if (BPF_CLASS(insn->code) == BPF_ALU64)
/* div r11 */
EMIT3(0x49, 0xF7, 0xF3);
else
/* div r11d */
EMIT3(0x41, 0xF7, 0xF3);
if (BPF_OP(insn->code) == BPF_MOD)
/* mov r11, rdx */
EMIT3(0x49, 0x89, 0xD3);
else
/* mov r11, rax */
EMIT3(0x49, 0x89, 0xC3);
EMIT1(0x5A); /* pop rdx */
EMIT1(0x58); /* pop rax */
/* mov dst_reg, r11 */
EMIT_mov(dst_reg, AUX_REG);
break;
case BPF_ALU | BPF_MUL | BPF_K:
case BPF_ALU | BPF_MUL | BPF_X:
case BPF_ALU64 | BPF_MUL | BPF_K:
case BPF_ALU64 | BPF_MUL | BPF_X:
{
bool is64 = BPF_CLASS(insn->code) == BPF_ALU64;
if (dst_reg != BPF_REG_0)
EMIT1(0x50); /* push rax */
if (dst_reg != BPF_REG_3)
EMIT1(0x52); /* push rdx */
/* mov r11, dst_reg */
EMIT_mov(AUX_REG, dst_reg);
if (BPF_SRC(insn->code) == BPF_X)
emit_mov_reg(&prog, is64, BPF_REG_0, src_reg);
else
emit_mov_imm32(&prog, is64, BPF_REG_0, imm32);
if (is64)
EMIT1(add_1mod(0x48, AUX_REG));
else if (is_ereg(AUX_REG))
EMIT1(add_1mod(0x40, AUX_REG));
/* mul(q) r11 */
EMIT2(0xF7, add_1reg(0xE0, AUX_REG));
if (dst_reg != BPF_REG_3)
EMIT1(0x5A); /* pop rdx */
if (dst_reg != BPF_REG_0) {
/* mov dst_reg, rax */
EMIT_mov(dst_reg, BPF_REG_0);
EMIT1(0x58); /* pop rax */
}
break;
}
/* Shifts */
case BPF_ALU | BPF_LSH | BPF_K:
case BPF_ALU | BPF_RSH | BPF_K:
case BPF_ALU | BPF_ARSH | BPF_K:
case BPF_ALU64 | BPF_LSH | BPF_K:
case BPF_ALU64 | BPF_RSH | BPF_K:
case BPF_ALU64 | BPF_ARSH | BPF_K:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
switch (BPF_OP(insn->code)) {
case BPF_LSH: b3 = 0xE0; break;
case BPF_RSH: b3 = 0xE8; break;
case BPF_ARSH: b3 = 0xF8; break;
}
if (imm32 == 1)
EMIT2(0xD1, add_1reg(b3, dst_reg));
else
EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);
break;
case BPF_ALU | BPF_LSH | BPF_X:
case BPF_ALU | BPF_RSH | BPF_X:
case BPF_ALU | BPF_ARSH | BPF_X:
case BPF_ALU64 | BPF_LSH | BPF_X:
case BPF_ALU64 | BPF_RSH | BPF_X:
case BPF_ALU64 | BPF_ARSH | BPF_X:
/* Check for bad case when dst_reg == rcx */
if (dst_reg == BPF_REG_4) {
/* mov r11, dst_reg */
EMIT_mov(AUX_REG, dst_reg);
dst_reg = AUX_REG;
}
if (src_reg != BPF_REG_4) { /* common case */
EMIT1(0x51); /* push rcx */
/* mov rcx, src_reg */
EMIT_mov(BPF_REG_4, src_reg);
}
/* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
switch (BPF_OP(insn->code)) {
case BPF_LSH: b3 = 0xE0; break;
case BPF_RSH: b3 = 0xE8; break;
case BPF_ARSH: b3 = 0xF8; break;
}
EMIT2(0xD3, add_1reg(b3, dst_reg));
if (src_reg != BPF_REG_4)
EMIT1(0x59); /* pop rcx */
if (insn->dst_reg == BPF_REG_4)
/* mov dst_reg, r11 */
EMIT_mov(insn->dst_reg, AUX_REG);
break;
case BPF_ALU | BPF_END | BPF_FROM_BE:
switch (imm32) {
case 16:
/* Emit 'ror %ax, 8' to swap lower 2 bytes */
EMIT1(0x66);
if (is_ereg(dst_reg))
EMIT1(0x41);
EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8);
/* Emit 'movzwl eax, ax' */
if (is_ereg(dst_reg))
EMIT3(0x45, 0x0F, 0xB7);
else
EMIT2(0x0F, 0xB7);
EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
break;
case 32:
/* Emit 'bswap eax' to swap lower 4 bytes */
if (is_ereg(dst_reg))
EMIT2(0x41, 0x0F);
else
EMIT1(0x0F);
EMIT1(add_1reg(0xC8, dst_reg));
break;
case 64:
/* Emit 'bswap rax' to swap 8 bytes */
EMIT3(add_1mod(0x48, dst_reg), 0x0F,
add_1reg(0xC8, dst_reg));
break;
}
break;
case BPF_ALU | BPF_END | BPF_FROM_LE:
switch (imm32) {
case 16:
/*
* Emit 'movzwl eax, ax' to zero extend 16-bit
* into 64 bit
*/
if (is_ereg(dst_reg))
EMIT3(0x45, 0x0F, 0xB7);
else
EMIT2(0x0F, 0xB7);
EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
break;
case 32:
/* Emit 'mov eax, eax' to clear upper 32-bits */
if (is_ereg(dst_reg))
EMIT1(0x45);
EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg));
break;
case 64:
/* nop */
break;
}
break;
/* ST: *(u8*)(dst_reg + off) = imm */
case BPF_ST | BPF_MEM | BPF_B:
if (is_ereg(dst_reg))
EMIT2(0x41, 0xC6);
else
EMIT1(0xC6);
goto st;
case BPF_ST | BPF_MEM | BPF_H:
if (is_ereg(dst_reg))
EMIT3(0x66, 0x41, 0xC7);
else
EMIT2(0x66, 0xC7);
goto st;
case BPF_ST | BPF_MEM | BPF_W:
if (is_ereg(dst_reg))
EMIT2(0x41, 0xC7);
else
EMIT1(0xC7);
goto st;
case BPF_ST | BPF_MEM | BPF_DW:
EMIT2(add_1mod(0x48, dst_reg), 0xC7);
st: if (is_imm8(insn->off))
EMIT2(add_1reg(0x40, dst_reg), insn->off);
else
EMIT1_off32(add_1reg(0x80, dst_reg), insn->off);
EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
break;
/* STX: *(u8*)(dst_reg + off) = src_reg */
case BPF_STX | BPF_MEM | BPF_B:
/* Emit 'mov byte ptr [rax + off], al' */
if (is_ereg(dst_reg) || is_ereg(src_reg) ||
/* We have to add extra byte for x86 SIL, DIL regs */
src_reg == BPF_REG_1 || src_reg == BPF_REG_2)
EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);
else
EMIT1(0x88);
goto stx;
case BPF_STX | BPF_MEM | BPF_H:
if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);
else
EMIT2(0x66, 0x89);
goto stx;
case BPF_STX | BPF_MEM | BPF_W:
if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);
else
EMIT1(0x89);
goto stx;
case BPF_STX | BPF_MEM | BPF_DW:
EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);
stx: if (is_imm8(insn->off))
EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
else
EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
insn->off);
break;
/* LDX: dst_reg = *(u8*)(src_reg + off) */
case BPF_LDX | BPF_MEM | BPF_B:
/* Emit 'movzx rax, byte ptr [rax + off]' */
EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);
goto ldx;
case BPF_LDX | BPF_MEM | BPF_H:
/* Emit 'movzx rax, word ptr [rax + off]' */
EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);
goto ldx;
case BPF_LDX | BPF_MEM | BPF_W:
/* Emit 'mov eax, dword ptr [rax+0x14]' */
if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);
else
EMIT1(0x8B);
goto ldx;
case BPF_LDX | BPF_MEM | BPF_DW:
/* Emit 'mov rax, qword ptr [rax+0x14]' */
EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
ldx: /*
* If insn->off == 0 we can save one extra byte, but
* special case of x86 R13 which always needs an offset
* is not worth the hassle
*/
if (is_imm8(insn->off))
EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off);
else
EMIT1_off32(add_2reg(0x80, src_reg, dst_reg),
insn->off);
break;
/* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */
case BPF_STX | BPF_XADD | BPF_W:
/* Emit 'lock add dword ptr [rax + off], eax' */
if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01);
else
EMIT2(0xF0, 0x01);
goto xadd;
case BPF_STX | BPF_XADD | BPF_DW:
EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01);
xadd: if (is_imm8(insn->off))
EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
else
EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
insn->off);
break;
/* call */
case BPF_JMP | BPF_CALL:
func = (u8 *) __bpf_call_base + imm32;
jmp_offset = func - (image + addrs[i]);
if (!imm32 || !is_simm32(jmp_offset)) {
pr_err("unsupported BPF func %d addr %p image %p\n",
imm32, func, image);
return -EINVAL;
}
EMIT1_off32(0xE8, jmp_offset);
break;
case BPF_JMP | BPF_TAIL_CALL:
emit_bpf_tail_call(&prog);
break;
/* cond jump */
case BPF_JMP | BPF_JEQ | BPF_X:
case BPF_JMP | BPF_JNE | BPF_X:
case BPF_JMP | BPF_JGT | BPF_X:
case BPF_JMP | BPF_JLT | BPF_X:
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP | BPF_JLE | BPF_X:
case BPF_JMP | BPF_JSGT | BPF_X:
case BPF_JMP | BPF_JSLT | BPF_X:
case BPF_JMP | BPF_JSGE | BPF_X:
case BPF_JMP | BPF_JSLE | BPF_X:
/* cmp dst_reg, src_reg */
EMIT3(add_2mod(0x48, dst_reg, src_reg), 0x39,
add_2reg(0xC0, dst_reg, src_reg));
goto emit_cond_jmp;
case BPF_JMP | BPF_JSET | BPF_X:
/* test dst_reg, src_reg */
EMIT3(add_2mod(0x48, dst_reg, src_reg), 0x85,
add_2reg(0xC0, dst_reg, src_reg));
goto emit_cond_jmp;
case BPF_JMP | BPF_JSET | BPF_K:
/* test dst_reg, imm32 */
EMIT1(add_1mod(0x48, dst_reg));
EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);
goto emit_cond_jmp;
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JLT | BPF_K:
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP | BPF_JLE | BPF_K:
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP | BPF_JSLT | BPF_K:
case BPF_JMP | BPF_JSGE | BPF_K:
case BPF_JMP | BPF_JSLE | BPF_K:
/* cmp dst_reg, imm8/32 */
EMIT1(add_1mod(0x48, dst_reg));
if (is_imm8(imm32))
EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);
else
EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32);
emit_cond_jmp: /* Convert BPF opcode to x86 */
switch (BPF_OP(insn->code)) {
case BPF_JEQ:
jmp_cond = X86_JE;
break;
case BPF_JSET:
case BPF_JNE:
jmp_cond = X86_JNE;
break;
case BPF_JGT:
/* GT is unsigned '>', JA in x86 */
jmp_cond = X86_JA;
break;
case BPF_JLT:
/* LT is unsigned '<', JB in x86 */
jmp_cond = X86_JB;
break;
case BPF_JGE:
/* GE is unsigned '>=', JAE in x86 */
jmp_cond = X86_JAE;
break;
case BPF_JLE:
/* LE is unsigned '<=', JBE in x86 */
jmp_cond = X86_JBE;
break;
case BPF_JSGT:
/* Signed '>', GT in x86 */
jmp_cond = X86_JG;
break;
case BPF_JSLT:
/* Signed '<', LT in x86 */
jmp_cond = X86_JL;
break;
case BPF_JSGE:
/* Signed '>=', GE in x86 */
jmp_cond = X86_JGE;
break;
case BPF_JSLE:
/* Signed '<=', LE in x86 */
jmp_cond = X86_JLE;
break;
default: /* to silence GCC warning */
return -EFAULT;
}
jmp_offset = addrs[i + insn->off] - addrs[i];
if (is_imm8(jmp_offset)) {
EMIT2(jmp_cond, jmp_offset);
} else if (is_simm32(jmp_offset)) {
EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
} else {
pr_err("cond_jmp gen bug %llx\n", jmp_offset);
return -EFAULT;
}
break;
case BPF_JMP | BPF_JA:
if (insn->off == -1)
/* -1 jmp instructions will always jump
* backwards two bytes. Explicitly handling
* this case avoids wasting too many passes
* when there are long sequences of replaced
* dead code.
*/
jmp_offset = -2;
else
jmp_offset = addrs[i + insn->off] - addrs[i];
if (!jmp_offset)
/* Optimize out nop jumps */
break;
emit_jmp:
if (is_imm8(jmp_offset)) {
EMIT2(0xEB, jmp_offset);
} else if (is_simm32(jmp_offset)) {
EMIT1_off32(0xE9, jmp_offset);
} else {
pr_err("jmp gen bug %llx\n", jmp_offset);
return -EFAULT;
}
break;
case BPF_JMP | BPF_EXIT:
if (seen_exit) {
jmp_offset = ctx->cleanup_addr - addrs[i];
goto emit_jmp;
}
seen_exit = true;
/* Update cleanup_addr */
ctx->cleanup_addr = proglen;
/* mov rbx, qword ptr [rbp+0] */
EMIT4(0x48, 0x8B, 0x5D, 0);
/* mov r13, qword ptr [rbp+8] */
EMIT4(0x4C, 0x8B, 0x6D, 8);
/* mov r14, qword ptr [rbp+16] */
EMIT4(0x4C, 0x8B, 0x75, 16);
/* mov r15, qword ptr [rbp+24] */
EMIT4(0x4C, 0x8B, 0x7D, 24);
/* add rbp, AUX_STACK_SPACE */
EMIT4(0x48, 0x83, 0xC5, AUX_STACK_SPACE);
EMIT1(0xC9); /* leave */
EMIT1(0xC3); /* ret */
break;
default:
/*
* By design x86-64 JIT should support all BPF instructions.
* This error will be seen if new instruction was added
* to the interpreter, but not to the JIT, or if there is
* junk in bpf_prog.
*/
pr_err("bpf_jit: unknown opcode %02x\n", insn->code);
return -EINVAL;
}
ilen = prog - temp;
if (ilen > BPF_MAX_INSN_SIZE) {
pr_err("bpf_jit: fatal insn size error\n");
return -EFAULT;
}
if (image) {
if (unlikely(proglen + ilen > oldproglen)) {
pr_err("bpf_jit: fatal error\n");
return -EFAULT;
}
memcpy(image + proglen, temp, ilen);
}
proglen += ilen;
addrs[i] = proglen;
prog = temp;
}
return proglen;
}
static char *bpf_dump(const struct sock_filter bpf, int n)
{
int v;
const char *fmt, *op;
static char image[256];
char operand[64];
v = bpf.k;
switch (bpf.code) {
default:
op = "unimp";
fmt = "0x%x";
v = bpf.code;
break;
case BPF_RET | BPF_K:
op = op_table[BPF_RET];
fmt = "#0x%x";
break;
case BPF_RET | BPF_A:
op = op_table[BPF_RET];
fmt = "a";
break;
case BPF_RET | BPF_X:
op = op_table[BPF_RET];
fmt = "x";
break;
case BPF_LD_W | BPF_ABS:
op = op_table[BPF_LD_W];
fmt = bpf_dump_linux_k(bpf.k);
break;
case BPF_LD_H | BPF_ABS:
op = op_table[BPF_LD_H];
fmt = bpf_dump_linux_k(bpf.k);
break;
case BPF_LD_B | BPF_ABS:
op = op_table[BPF_LD_B];
fmt = bpf_dump_linux_k(bpf.k);
break;
case BPF_LD_W | BPF_LEN:
op = op_table[BPF_LD_W];
fmt = "#len";
break;
case BPF_LD_W | BPF_IND:
op = op_table[BPF_LD_W];
fmt = "[x + %d]";
break;
case BPF_LD_H | BPF_IND:
op = op_table[BPF_LD_H];
fmt = "[x + %d]";
break;
case BPF_LD_B | BPF_IND:
op = op_table[BPF_LD_B];
fmt = "[x + %d]";
break;
case BPF_LD | BPF_IMM:
op = op_table[BPF_LD_W];
fmt = "#0x%x";
break;
case BPF_LDX | BPF_IMM:
op = op_table[BPF_LDX];
fmt = "#0x%x";
break;
case BPF_LDX_B | BPF_MSH:
op = op_table[BPF_LDX_B];
fmt = "4*([%d]&0xf)";
break;
case BPF_LD | BPF_MEM:
op = op_table[BPF_LD_W];
fmt = "M[%d]";
break;
case BPF_LDX | BPF_MEM:
op = op_table[BPF_LDX];
fmt = "M[%d]";
break;
case BPF_ST:
op = op_table[BPF_ST];
fmt = "M[%d]";
break;
case BPF_STX:
op = op_table[BPF_STX];
fmt = "M[%d]";
break;
case BPF_JMP_JA:
op = op_table[BPF_JMP_JA];
fmt = "%d";
v = n + 1 + bpf.k;
break;
case BPF_JMP_JGT | BPF_K:
op = op_table[BPF_JMP_JGT];
fmt = "#0x%x";
break;
case BPF_JMP_JGE | BPF_K:
op = op_table[BPF_JMP_JGE];
fmt = "#0x%x";
break;
case BPF_JMP_JEQ | BPF_K:
op = op_table[BPF_JMP_JEQ];
fmt = "#0x%x";
break;
case BPF_JMP_JSET | BPF_K:
op = op_table[BPF_JMP_JSET];
fmt = "#0x%x";
break;
case BPF_JMP_JGT | BPF_X:
op = op_table[BPF_JMP_JGT];
fmt = "x";
break;
case BPF_JMP_JGE | BPF_X:
op = op_table[BPF_JMP_JGE];
fmt = "x";
break;
case BPF_JMP_JEQ | BPF_X:
op = op_table[BPF_JMP_JEQ];
fmt = "x";
break;
case BPF_JMP_JSET | BPF_X:
op = op_table[BPF_JMP_JSET];
fmt = "x";
break;
case BPF_ALU_ADD | BPF_X:
op = op_table[BPF_ALU_ADD];
fmt = "x";
break;
case BPF_ALU_SUB | BPF_X:
op = op_table[BPF_ALU_SUB];
fmt = "x";
break;
case BPF_ALU_MUL | BPF_X:
op = op_table[BPF_ALU_MUL];
fmt = "x";
break;
case BPF_ALU_DIV | BPF_X:
op = op_table[BPF_ALU_DIV];
fmt = "x";
break;
case BPF_ALU_MOD | BPF_X:
op = op_table[BPF_ALU_MOD];
fmt = "x";
break;
case BPF_ALU_AND | BPF_X:
op = op_table[BPF_ALU_AND];
fmt = "x";
break;
case BPF_ALU_OR | BPF_X:
op = op_table[BPF_ALU_OR];
fmt = "x";
break;
case BPF_ALU_XOR | BPF_X:
op = op_table[BPF_ALU_XOR];
fmt = "x";
break;
case BPF_ALU_LSH | BPF_X:
op = op_table[BPF_ALU_LSH];
fmt = "x";
break;
case BPF_ALU_RSH | BPF_X:
op = op_table[BPF_ALU_RSH];
fmt = "x";
break;
case BPF_ALU_ADD | BPF_K:
op = op_table[BPF_ALU_ADD];
fmt = "#%d";
break;
case BPF_ALU_SUB | BPF_K:
op = op_table[BPF_ALU_SUB];
fmt = "#%d";
break;
case BPF_ALU_MUL | BPF_K:
op = op_table[BPF_ALU_MUL];
fmt = "#%d";
break;
case BPF_ALU_DIV | BPF_K:
op = op_table[BPF_ALU_DIV];
fmt = "#%d";
break;
case BPF_ALU_MOD | BPF_K:
op = op_table[BPF_ALU_MOD];
fmt = "#%d";
break;
case BPF_ALU_AND | BPF_K:
op = op_table[BPF_ALU_AND];
fmt = "#0x%x";
break;
case BPF_ALU_OR | BPF_K:
op = op_table[BPF_ALU_OR];
fmt = "#0x%x";
break;
case BPF_ALU_XOR | BPF_K:
op = op_table[BPF_ALU_XOR];
fmt = "#0x%x";
break;
case BPF_ALU_LSH | BPF_K:
op = op_table[BPF_ALU_LSH];
fmt = "#%d";
break;
case BPF_ALU_RSH | BPF_K:
op = op_table[BPF_ALU_RSH];
fmt = "#%d";
break;
case BPF_ALU_NEG:
op = op_table[BPF_ALU_NEG];
fmt = "";
break;
case BPF_MISC_TAX:
op = op_table[BPF_MISC_TAX];
fmt = "";
break;
case BPF_MISC_TXA:
op = op_table[BPF_MISC_TXA];
fmt = "";
break;
}
slprintf_nocheck(operand, sizeof(operand), fmt, v);
slprintf_nocheck(image, sizeof(image),
(BPF_CLASS(bpf.code) == BPF_JMP &&
BPF_OP(bpf.code) != BPF_JA) ?
" L%d: %s %s, L%d, L%d" : " L%d: %s %s",
n, op, operand, n + 1 + bpf.jt, n + 1 + bpf.jf);
return image;
}
static int do_jit(struct bpf_program *bpf_prog, int *addrs, u8 *image,
int oldproglen)
{
struct bpf_insn *insn = bpf_prog->insns;
int insn_cnt = bpf_prog->insn_cnt;
u8 temp[64];
int i;
int proglen = 0;
u8 *prog = temp;
int stacksize = 512;
EMIT1(0x55); /* push rbp */
EMIT3(0x48, 0x89, 0xE5); /* mov rbp,rsp */
/* sub rsp, stacksize */
EMIT3_off32(0x48, 0x81, 0xEC, stacksize);
/* mov qword ptr [rbp-X],rbx */
EMIT3_off32(0x48, 0x89, 0x9D, -stacksize);
/* mov qword ptr [rbp-X],r13 */
EMIT3_off32(0x4C, 0x89, 0xAD, -stacksize + 8);
/* mov qword ptr [rbp-X],r14 */
EMIT3_off32(0x4C, 0x89, 0xB5, -stacksize + 16);
/* mov qword ptr [rbp-X],r15 */
EMIT3_off32(0x4C, 0x89, 0xBD, -stacksize + 24);
for (i = 0; i < insn_cnt; i++, insn++) {
const __s32 K = insn->imm;
__u32 a_reg = insn->a_reg;
__u32 x_reg = insn->x_reg;
u8 b1 = 0, b2 = 0, b3 = 0;
u8 jmp_cond;
__s64 jmp_offset;
int ilen;
u8 *func;
switch (insn->code) {
/* ALU */
case BPF_ALU | BPF_ADD | BPF_X:
case BPF_ALU | BPF_SUB | BPF_X:
case BPF_ALU | BPF_AND | BPF_X:
case BPF_ALU | BPF_OR | BPF_X:
case BPF_ALU | BPF_XOR | BPF_X:
b1 = 0x48;
b3 = 0xC0;
switch (BPF_OP(insn->code)) {
case BPF_ADD: b2 = 0x01; break;
case BPF_SUB: b2 = 0x29; break;
case BPF_AND: b2 = 0x21; break;
case BPF_OR: b2 = 0x09; break;
case BPF_XOR: b2 = 0x31; break;
}
EMIT3(add_2mod(b1, a_reg, x_reg), b2,
add_2reg(b3, a_reg, x_reg));
break;
/* mov A, X */
case BPF_ALU | BPF_MOV | BPF_X:
EMIT_mov(a_reg, x_reg);
break;
/* neg A */
case BPF_ALU | BPF_NEG | BPF_X:
EMIT3(add_1mod(0x48, a_reg), 0xF7,
add_1reg(0xD8, a_reg));
break;
case BPF_ALU | BPF_ADD | BPF_K:
case BPF_ALU | BPF_SUB | BPF_K:
case BPF_ALU | BPF_AND | BPF_K:
case BPF_ALU | BPF_OR | BPF_K:
b1 = add_1mod(0x48, a_reg);
switch (BPF_OP(insn->code)) {
case BPF_ADD: b3 = 0xC0; break;
case BPF_SUB: b3 = 0xE8; break;
case BPF_AND: b3 = 0xE0; break;
case BPF_OR: b3 = 0xC8; break;
}
if (is_imm8(K))
EMIT4(b1, 0x83, add_1reg(b3, a_reg), K);
else
EMIT3_off32(b1, 0x81, add_1reg(b3, a_reg), K);
break;
case BPF_ALU | BPF_MOV | BPF_K:
/* 'mov rax, imm32' sign extends imm32.
* possible optimization: if imm32 is positive,
* use 'mov eax, imm32' (which zero-extends imm32)
* to save 2 bytes */
b1 = add_1mod(0x48, a_reg);
b2 = 0xC7;
b3 = 0xC0;
EMIT3_off32(b1, b2, add_1reg(b3, a_reg), K);
break;
/* A %= X
* A /= X */
case BPF_ALU | BPF_MOD | BPF_X:
case BPF_ALU | BPF_DIV | BPF_X:
EMIT1(0x50); /* push rax */
EMIT1(0x52); /* push rdx */
/* mov r9, X */
EMIT_mov(AUX_REG, x_reg);
/* mov rax, A */
EMIT_mov(R0, a_reg);
/* xor rdx, rdx */
EMIT3(0x48, 0x31, 0xd2);
/* if X==0, skip divide, make A=0 */
/* cmp r9, 0 */
EMIT4(0x49, 0x83, 0xF9, 0x00);
/* je .+3 */
EMIT2(X86_JE, 3);
/* div r9 */
EMIT3(0x49, 0xF7, 0xF1);
if (BPF_OP(insn->code) == BPF_MOD) {
/* mov r9, rdx */
EMIT3(0x49, 0x89, 0xD1);
} else {
/* mov r9, rax */
EMIT3(0x49, 0x89, 0xC1);
}
EMIT1(0x5A); /* pop rdx */
EMIT1(0x58); /* pop rax */
/* mov A, r9 */
EMIT_mov(a_reg, AUX_REG);
break;
/* shifts */
case BPF_ALU | BPF_LSH | BPF_K:
case BPF_ALU | BPF_RSH | BPF_K:
case BPF_ALU | BPF_ARSH | BPF_K:
b1 = add_1mod(0x48, a_reg);
switch (BPF_OP(insn->code)) {
case BPF_LSH: b3 = 0xE0; break;
case BPF_RSH: b3 = 0xE8; break;
case BPF_ARSH: b3 = 0xF8; break;
}
EMIT4(b1, 0xC1, add_1reg(b3, a_reg), K);
break;
case BPF_ALU | BPF_BSWAP32 | BPF_X:
/* emit 'bswap eax' to swap lower 4-bytes */
if (is_ereg(a_reg))
EMIT2(0x41, 0x0F);
else
EMIT1(0x0F);
EMIT1(add_1reg(0xC8, a_reg));
break;
case BPF_ALU | BPF_BSWAP64 | BPF_X:
/* emit 'bswap rax' to swap 8-bytes */
EMIT3(add_1mod(0x48, a_reg), 0x0F,
add_1reg(0xC8, a_reg));
break;
/* ST: *(u8*)(a_reg + off) = imm */
case BPF_ST | BPF_REL | BPF_B:
if (is_ereg(a_reg))
EMIT2(0x41, 0xC6);
else
EMIT1(0xC6);
goto st;
case BPF_ST | BPF_REL | BPF_H:
if (is_ereg(a_reg))
EMIT3(0x66, 0x41, 0xC7);
else
EMIT2(0x66, 0xC7);
goto st;
case BPF_ST | BPF_REL | BPF_W:
if (is_ereg(a_reg))
EMIT2(0x41, 0xC7);
else
EMIT1(0xC7);
goto st;
case BPF_ST | BPF_REL | BPF_DW:
EMIT2(add_1mod(0x48, a_reg), 0xC7);
st: if (is_imm8(insn->off))
EMIT2(add_1reg(0x40, a_reg), insn->off);
else
EMIT1_off32(add_1reg(0x80, a_reg), insn->off);
EMIT(K, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
break;
/* STX: *(u8*)(a_reg + off) = x_reg */
case BPF_STX | BPF_REL | BPF_B:
/* emit 'mov byte ptr [rax + off], al' */
if (is_ereg(a_reg) || is_ereg(x_reg) ||
/* have to add extra byte for x86 SIL, DIL regs */
x_reg == R1 || x_reg == R2)
EMIT2(add_2mod(0x40, a_reg, x_reg), 0x88);
else
EMIT1(0x88);
goto stx;
case BPF_STX | BPF_REL | BPF_H:
if (is_ereg(a_reg) || is_ereg(x_reg))
EMIT3(0x66, add_2mod(0x40, a_reg, x_reg), 0x89);
else
EMIT2(0x66, 0x89);
goto stx;
case BPF_STX | BPF_REL | BPF_W:
if (is_ereg(a_reg) || is_ereg(x_reg))
EMIT2(add_2mod(0x40, a_reg, x_reg), 0x89);
else
EMIT1(0x89);
goto stx;
case BPF_STX | BPF_REL | BPF_DW:
EMIT2(add_2mod(0x48, a_reg, x_reg), 0x89);
stx: if (is_imm8(insn->off))
EMIT2(add_2reg(0x40, a_reg, x_reg), insn->off);
else
EMIT1_off32(add_2reg(0x80, a_reg, x_reg),
insn->off);
break;
/* LDX: a_reg = *(u8*)(x_reg + off) */
case BPF_LDX | BPF_REL | BPF_B:
/* emit 'movzx rax, byte ptr [rax + off]' */
EMIT3(add_2mod(0x48, x_reg, a_reg), 0x0F, 0xB6);
goto ldx;
case BPF_LDX | BPF_REL | BPF_H:
/* emit 'movzx rax, word ptr [rax + off]' */
EMIT3(add_2mod(0x48, x_reg, a_reg), 0x0F, 0xB7);
goto ldx;
case BPF_LDX | BPF_REL | BPF_W:
/* emit 'mov eax, dword ptr [rax+0x14]' */
if (is_ereg(a_reg) || is_ereg(x_reg))
EMIT2(add_2mod(0x40, x_reg, a_reg), 0x8B);
else
EMIT1(0x8B);
goto ldx;
case BPF_LDX | BPF_REL | BPF_DW:
/* emit 'mov rax, qword ptr [rax+0x14]' */
EMIT2(add_2mod(0x48, x_reg, a_reg), 0x8B);
ldx: /* if insn->off == 0 we can save one extra byte, but
* special case of x86 R13 which always needs an offset
* is not worth the pain */
if (is_imm8(insn->off))
EMIT2(add_2reg(0x40, x_reg, a_reg), insn->off);
else
EMIT1_off32(add_2reg(0x80, x_reg, a_reg),
insn->off);
break;
/* STX XADD: lock *(u8*)(a_reg + off) += x_reg */
case BPF_STX | BPF_XADD | BPF_B:
/* emit 'lock add byte ptr [rax + off], al' */
if (is_ereg(a_reg) || is_ereg(x_reg) ||
/* have to add extra byte for x86 SIL, DIL regs */
x_reg == R1 || x_reg == R2)
EMIT3(0xF0, add_2mod(0x40, a_reg, x_reg), 0x00);
else
EMIT2(0xF0, 0x00);
goto xadd;
case BPF_STX | BPF_XADD | BPF_H:
if (is_ereg(a_reg) || is_ereg(x_reg))
EMIT4(0x66, 0xF0, add_2mod(0x40, a_reg, x_reg),
0x01);
else
EMIT3(0x66, 0xF0, 0x01);
goto xadd;
case BPF_STX | BPF_XADD | BPF_W:
if (is_ereg(a_reg) || is_ereg(x_reg))
EMIT3(0xF0, add_2mod(0x40, a_reg, x_reg), 0x01);
else
EMIT2(0xF0, 0x01);
goto xadd;
case BPF_STX | BPF_XADD | BPF_DW:
EMIT3(0xF0, add_2mod(0x48, a_reg, x_reg), 0x01);
xadd: if (is_imm8(insn->off))
EMIT2(add_2reg(0x40, a_reg, x_reg), insn->off);
else
EMIT1_off32(add_2reg(0x80, a_reg, x_reg),
insn->off);
break;
/* call */
case BPF_JMP | BPF_CALL:
func = select_bpf_func(bpf_prog, K);
jmp_offset = func - (image + addrs[i]);
if (!func || !is_simm32(jmp_offset)) {
pr_err("unsupported bpf func %d addr %p image %p\n",
K, func, image);
return -EINVAL;
}
EMIT1_off32(0xE8, jmp_offset);
break;
/* cond jump */
case BPF_JMP | BPF_JEQ | BPF_X:
case BPF_JMP | BPF_JNE | BPF_X:
case BPF_JMP | BPF_JGT | BPF_X:
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP | BPF_JSGT | BPF_X:
case BPF_JMP | BPF_JSGE | BPF_X:
/* emit 'cmp a_reg, x_reg' insn */
b1 = 0x48;
b2 = 0x39;
b3 = 0xC0;
EMIT3(add_2mod(b1, a_reg, x_reg), b2,
add_2reg(b3, a_reg, x_reg));
goto emit_jump;
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP | BPF_JSGE | BPF_K:
/* emit 'cmp a_reg, imm8/32' */
EMIT1(add_1mod(0x48, a_reg));
if (is_imm8(K))
EMIT3(0x83, add_1reg(0xF8, a_reg), K);
else
EMIT2_off32(0x81, add_1reg(0xF8, a_reg), K);
emit_jump: /* convert BPF opcode to x86 */
switch (BPF_OP(insn->code)) {
case BPF_JEQ:
jmp_cond = X86_JE;
break;
case BPF_JNE:
jmp_cond = X86_JNE;
break;
case BPF_JGT:
/* GT is unsigned '>', JA in x86 */
jmp_cond = X86_JA;
break;
case BPF_JGE:
/* GE is unsigned '>=', JAE in x86 */
jmp_cond = X86_JAE;
break;
case BPF_JSGT:
/* signed '>', GT in x86 */
jmp_cond = X86_JG;
break;
case BPF_JSGE:
/* signed '>=', GE in x86 */
jmp_cond = X86_JGE;
break;
default: /* to silence gcc warning */
return -EFAULT;
}
jmp_offset = addrs[i + insn->off] - addrs[i];
if (is_imm8(jmp_offset)) {
EMIT2(jmp_cond, jmp_offset);
} else if (is_simm32(jmp_offset)) {
EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
} else {
pr_err("cond_jmp gen bug %llx\n", jmp_offset);
return -EFAULT;
}
break;
case BPF_JMP | BPF_JA | BPF_X:
jmp_offset = addrs[i + insn->off] - addrs[i];
if (is_imm8(jmp_offset)) {
EMIT2(0xEB, jmp_offset);
} else if (is_simm32(jmp_offset)) {
EMIT1_off32(0xE9, jmp_offset);
} else {
pr_err("jmp gen bug %llx\n", jmp_offset);
return -EFAULT;
}
break;
case BPF_RET | BPF_K:
/* mov rbx, qword ptr [rbp-X] */
EMIT3_off32(0x48, 0x8B, 0x9D, -stacksize);
/* mov r13, qword ptr [rbp-X] */
EMIT3_off32(0x4C, 0x8B, 0xAD, -stacksize + 8);
/* mov r14, qword ptr [rbp-X] */
EMIT3_off32(0x4C, 0x8B, 0xB5, -stacksize + 16);
/* mov r15, qword ptr [rbp-X] */
EMIT3_off32(0x4C, 0x8B, 0xBD, -stacksize + 24);
EMIT1(0xC9); /* leave */
EMIT1(0xC3); /* ret */
break;
default:
/*pr_debug_bpf_insn(insn, NULL);*/
pr_err("bpf_jit: unknown opcode %02x\n", insn->code);
return -EINVAL;
}
ilen = prog - temp;
if (image) {
if (proglen + ilen > oldproglen)
return -2;
memcpy(image + proglen, temp, ilen);
}
proglen += ilen;
addrs[i] = proglen;
prog = temp;
}
return proglen;
}
int bpf_validate(const struct sock_fprog *bpf)
{
uint32_t i, from;
const struct sock_filter *p;
if (!bpf)
return 0;
if (bpf->len < 1)
return 0;
for (i = 0; i < bpf->len; ++i) {
p = &bpf->filter[i];
switch (BPF_CLASS(p->code)) {
/*
* Check that memory operations use valid addresses.
*/
case BPF_LD:
case BPF_LDX:
switch (BPF_MODE(p->code)) {
case BPF_IMM:
break;
case BPF_ABS:
case BPF_IND:
case BPF_MSH:
/*
* There's no maximum packet data size
* in userland. The runtime packet length
* check suffices.
*/
break;
case BPF_MEM:
if (p->k >= BPF_MEMWORDS)
return 0;
break;
case BPF_LEN:
break;
default:
return 0;
}
break;
case BPF_ST:
case BPF_STX:
if (p->k >= BPF_MEMWORDS)
return 0;
break;
case BPF_ALU:
switch (BPF_OP(p->code)) {
case BPF_ADD:
case BPF_SUB:
case BPF_MUL:
case BPF_OR:
case BPF_AND:
case BPF_LSH:
case BPF_RSH:
case BPF_NEG:
break;
case BPF_DIV:
/*
* Check for constant division by 0.
*/
if (BPF_RVAL(p->code) == BPF_K && p->k == 0)
return 0;
break;
default:
return 0;
}
break;
case BPF_JMP:
/*
* Check that jumps are within the code block,
* and that unconditional branches don't go
* backwards as a result of an overflow.
* Unconditional branches have a 32-bit offset,
* so they could overflow; we check to make
* sure they don't. Conditional branches have
* an 8-bit offset, and the from address is <=
* BPF_MAXINSNS, and we assume that BPF_MAXINSNS
* is sufficiently small that adding 255 to it
* won't overflow.
*
* We know that len is <= BPF_MAXINSNS, and we
* assume that BPF_MAXINSNS is < the maximum size
* of a u_int, so that i + 1 doesn't overflow.
*
* For userland, we don't know that the from
* or len are <= BPF_MAXINSNS, but we know that
* from <= len, and, except on a 64-bit system,
* it's unlikely that len, if it truly reflects
* the size of the program we've been handed,
* will be anywhere near the maximum size of
* a u_int. We also don't check for backward
* branches, as we currently support them in
* userland for the protochain operation.
*/
from = i + 1;
switch (BPF_OP(p->code)) {
case BPF_JA:
if (from + p->k >= bpf->len)
return 0;
break;
case BPF_JEQ:
case BPF_JGT:
case BPF_JGE:
case BPF_JSET:
if (from + p->jt >= bpf->len ||
from + p->jf >= bpf->len)
return 0;
break;
default:
return 0;
}
break;
case BPF_RET:
break;
case BPF_MISC:
break;
default:
return 0;
}
}
return BPF_CLASS(bpf->filter[bpf->len - 1].code) == BPF_RET;
}
static char *bpf_dump(const struct sock_filter bpf, int n)
{
int v;
const char *fmt, *op;
static char image[256];
char operand[64];
v = bpf.k;
switch (bpf.code) {
default:
op = "unimp";
fmt = "0x%x";
v = bpf.code;
break;
case BPF_RET | BPF_K:
op = "ret";
fmt = "#0x%x";
break;
case BPF_RET | BPF_A:
op = "ret";
fmt = "";
break;
case BPF_LD | BPF_W | BPF_ABS:
op = "ld";
fmt = "[%d]";
break;
case BPF_LD | BPF_H | BPF_ABS:
op = "ldh";
fmt = "[%d]";
break;
case BPF_LD | BPF_B | BPF_ABS:
op = "ldb";
fmt = "[%d]";
break;
case BPF_LD | BPF_W | BPF_LEN:
op = "ld";
fmt = "#pktlen";
break;
case BPF_LD | BPF_W | BPF_IND:
op = "ld";
fmt = "[x + %d]";
break;
case BPF_LD | BPF_H | BPF_IND:
op = "ldh";
fmt = "[x + %d]";
break;
case BPF_LD | BPF_B | BPF_IND:
op = "ldb";
fmt = "[x + %d]";
break;
case BPF_LD | BPF_IMM:
op = "ld";
fmt = "#0x%x";
break;
case BPF_LDX | BPF_IMM:
op = "ldx";
fmt = "#0x%x";
break;
case BPF_LDX | BPF_MSH | BPF_B:
op = "ldxb";
fmt = "4*([%d]&0xf)";
break;
case BPF_LD | BPF_MEM:
op = "ld";
fmt = "M[%d]";
break;
case BPF_LDX | BPF_MEM:
op = "ldx";
fmt = "M[%d]";
break;
case BPF_ST:
op = "st";
fmt = "M[%d]";
break;
case BPF_STX:
op = "stx";
fmt = "M[%d]";
break;
case BPF_JMP | BPF_JA:
op = "ja";
fmt = "%d";
v = n + 1 + bpf.k;
break;
case BPF_JMP | BPF_JGT | BPF_K:
op = "jgt";
fmt = "#0x%x";
break;
case BPF_JMP | BPF_JGE | BPF_K:
op = "jge";
fmt = "#0x%x";
break;
case BPF_JMP | BPF_JEQ | BPF_K:
op = "jeq";
fmt = "#0x%x";
break;
case BPF_JMP | BPF_JSET | BPF_K:
op = "jset";
fmt = "#0x%x";
break;
case BPF_JMP | BPF_JGT | BPF_X:
op = "jgt";
fmt = "x";
break;
case BPF_JMP | BPF_JGE | BPF_X:
op = "jge";
fmt = "x";
break;
case BPF_JMP | BPF_JEQ | BPF_X:
op = "jeq";
fmt = "x";
break;
case BPF_JMP | BPF_JSET | BPF_X:
op = "jset";
fmt = "x";
break;
case BPF_ALU | BPF_ADD | BPF_X:
op = "add";
fmt = "x";
break;
case BPF_ALU | BPF_SUB | BPF_X:
op = "sub";
fmt = "x";
break;
case BPF_ALU | BPF_MUL | BPF_X:
op = "mul";
fmt = "x";
break;
case BPF_ALU | BPF_DIV | BPF_X:
op = "div";
fmt = "x";
break;
case BPF_ALU | BPF_AND | BPF_X:
op = "and";
fmt = "x";
break;
case BPF_ALU | BPF_OR | BPF_X:
op = "or";
fmt = "x";
break;
case BPF_ALU | BPF_LSH | BPF_X:
op = "lsh";
fmt = "x";
break;
case BPF_ALU | BPF_RSH | BPF_X:
op = "rsh";
fmt = "x";
break;
case BPF_ALU | BPF_ADD | BPF_K:
op = "add";
fmt = "#%d";
break;
case BPF_ALU | BPF_SUB | BPF_K:
op = "sub";
fmt = "#%d";
break;
case BPF_ALU | BPF_MUL | BPF_K:
op = "mul";
fmt = "#%d";
break;
case BPF_ALU | BPF_DIV | BPF_K:
op = "div";
fmt = "#%d";
break;
case BPF_ALU | BPF_AND | BPF_K:
op = "and";
fmt = "#0x%x";
break;
case BPF_ALU | BPF_OR | BPF_K:
op = "or";
fmt = "#0x%x";
break;
case BPF_ALU | BPF_LSH | BPF_K:
op = "lsh";
fmt = "#%d";
break;
case BPF_ALU | BPF_RSH | BPF_K:
op = "rsh";
fmt = "#%d";
break;
case BPF_ALU | BPF_NEG:
op = "neg";
fmt = "";
break;
case BPF_MISC | BPF_TAX:
op = "tax";
fmt = "";
break;
case BPF_MISC | BPF_TXA:
op = "txa";
fmt = "";
break;
}
slprintf(operand, sizeof(operand), fmt, v);
slprintf(image, sizeof(image),
(BPF_CLASS(bpf.code) == BPF_JMP &&
BPF_OP(bpf.code) != BPF_JA) ?
" L%d: %s %s, L%d, L%d" : " L%d: %s %s",
n, op, operand, n + 1 + bpf.jt, n + 1 + bpf.jf);
return image;
}
static void
opt_peep(struct block *b)
{
struct slist *s;
struct slist *next, *last;
int val;
s = b->stmts;
if (s == 0)
return;
last = s;
for (/*empty*/; /*empty*/; s = next) {
/*
* Skip over nops.
*/
s = this_op(s);
if (s == 0)
break; /* nothing left in the block */
/*
* Find the next real instruction after that one
* (skipping nops).
*/
next = this_op(s->next);
if (next == 0)
break; /* no next instruction */
last = next;
/*
* st M[k] --> st M[k]
* ldx M[k] tax
*/
if (s->s.code == BPF_ST &&
next->s.code == (BPF_LDX|BPF_MEM) &&
s->s.k == next->s.k) {
done = 0;
next->s.code = BPF_MISC|BPF_TAX;
}
/*
* ld #k --> ldx #k
* tax txa
*/
if (s->s.code == (BPF_LD|BPF_IMM) &&
next->s.code == (BPF_MISC|BPF_TAX)) {
s->s.code = BPF_LDX|BPF_IMM;
next->s.code = BPF_MISC|BPF_TXA;
done = 0;
}
/*
* This is an ugly special case, but it happens
* when you say tcp[k] or udp[k] where k is a constant.
*/
if (s->s.code == (BPF_LD|BPF_IMM)) {
struct slist *add, *tax, *ild;
/*
* Check that X isn't used on exit from this
* block (which the optimizer might cause).
* We know the code generator won't generate
* any local dependencies.
*/
if (ATOMELEM(b->out_use, X_ATOM))
continue;
/*
* Check that the instruction following the ldi
* is an addx, or it's an ldxms with an addx
* following it (with 0 or more nops between the
* ldxms and addx).
*/
if (next->s.code != (BPF_LDX|BPF_MSH|BPF_B))
add = next;
else
add = this_op(next->next);
if (add == 0 || add->s.code != (BPF_ALU|BPF_ADD|BPF_X))
continue;
/*
* Check that a tax follows that (with 0 or more
* nops between them).
*/
tax = this_op(add->next);
if (tax == 0 || tax->s.code != (BPF_MISC|BPF_TAX))
continue;
/*
* Check that an ild follows that (with 0 or more
* nops between them).
*/
ild = this_op(tax->next);
if (ild == 0 || BPF_CLASS(ild->s.code) != BPF_LD ||
BPF_MODE(ild->s.code) != BPF_IND)
continue;
/*
* We want to turn this sequence:
*
* (004) ldi #0x2 {s}
* (005) ldxms [14] {next} -- optional
* (006) addx {add}
* (007) tax {tax}
* (008) ild [x+0] {ild}
*
* into this sequence:
*
* (004) nop
* (005) ldxms [14]
* (006) nop
* (007) nop
* (008) ild [x+2]
*
* XXX We need to check that X is not
* subsequently used, because we want to change
* what'll be in it after this sequence.
*
* We know we can eliminate the accumulator
* modifications earlier in the sequence since
* it is defined by the last stmt of this sequence
* (i.e., the last statement of the sequence loads
* a value into the accumulator, so we can eliminate
* earlier operations on the accumulator).
*/
ild->s.k += s->s.k;
s->s.code = NOP;
add->s.code = NOP;
tax->s.code = NOP;
done = 0;
}
}
/*
* If the comparison at the end of a block is an equality
* comparison against a constant, and nobody uses the value
* we leave in the A register at the end of a block, and
* the operation preceding the comparison is an arithmetic
* operation, we can sometime optimize it away.
*/
if (b->s.code == (BPF_JMP|BPF_JEQ|BPF_K) &&
!ATOMELEM(b->out_use, A_ATOM)) {
/*
* We can optimize away certain subtractions of the
* X register.
*/
if (last->s.code == (BPF_ALU|BPF_SUB|BPF_X)) {
val = b->val[X_ATOM];
if (vmap[val].is_const) {
/*
* If we have a subtract to do a comparison,
* and the X register is a known constant,
* we can merge this value into the
* comparison:
*
* sub x -> nop
* jeq #y jeq #(x+y)
*/
b->s.k += vmap[val].const_val;
last->s.code = NOP;
done = 0;
} else if (b->s.k == 0) {
/*
* If the X register isn't a constant,
* and the comparison in the test is
* against 0, we can compare with the
* X register, instead:
*
* sub x -> nop
* jeq #0 jeq x
*/
last->s.code = NOP;
b->s.code = BPF_JMP|BPF_JEQ|BPF_X;
done = 0;
}
}
/*
* Likewise, a constant subtract can be simplified:
*
* sub #x -> nop
* jeq #y -> jeq #(x+y)
*/
else if (last->s.code == (BPF_ALU|BPF_SUB|BPF_K)) {
last->s.code = NOP;
b->s.k += last->s.k;
done = 0;
}
/*
* And, similarly, a constant AND can be simplified
* if we're testing against 0, i.e.:
*
* and #k nop
* jeq #0 -> jset #k
*/
else if (last->s.code == (BPF_ALU|BPF_AND|BPF_K) &&
b->s.k == 0) {
b->s.k = last->s.k;
b->s.code = BPF_JMP|BPF_K|BPF_JSET;
last->s.code = NOP;
done = 0;
opt_not(b);
}
}
/*
* jset #0 -> never
* jset #ffffffff -> always
*/
if (b->s.code == (BPF_JMP|BPF_K|BPF_JSET)) {
if (b->s.k == 0)
JT(b) = JF(b);
if (b->s.k == (int)0xffffffff)
JF(b) = JT(b);
}
/*
* If we're comparing against the index register, and the index
* register is a known constant, we can just compare against that
* constant.
*/
val = b->val[X_ATOM];
if (vmap[val].is_const && BPF_SRC(b->s.code) == BPF_X) {
bpf_int32 v = vmap[val].const_val;
b->s.code &= ~BPF_X;
b->s.k = v;
}
/*
* If the accumulator is a known constant, we can compute the
* comparison result.
*/
val = b->val[A_ATOM];
if (vmap[val].is_const && BPF_SRC(b->s.code) == BPF_K) {
bpf_int32 v = vmap[val].const_val;
switch (BPF_OP(b->s.code)) {
case BPF_JEQ:
v = v == b->s.k;
break;
case BPF_JGT:
v = (unsigned)v > (unsigned)b->s.k;
break;
case BPF_JGE:
v = (unsigned)v >= (unsigned)b->s.k;
break;
case BPF_JSET:
v &= b->s.k;
break;
default:
abort();
}
if (JF(b) != JT(b))
done = 0;
if (v)
JF(b) = JT(b);
else
JT(b) = JF(b);
}
}
/*
* Compute the symbolic value of expression of 's', and update
* anything it defines in the value table 'val'. If 'alter' is true,
* do various optimizations. This code would be cleaner if symbolic
* evaluation and code transformations weren't folded together.
*/
static void
opt_stmt(struct stmt *s, int val[], int alter)
{
int op;
int v;
switch (s->code) {
case BPF_LD|BPF_ABS|BPF_W:
case BPF_LD|BPF_ABS|BPF_H:
case BPF_LD|BPF_ABS|BPF_B:
v = F(s->code, s->k, 0L);
vstore(s, &val[A_ATOM], v, alter);
break;
case BPF_LD|BPF_IND|BPF_W:
case BPF_LD|BPF_IND|BPF_H:
case BPF_LD|BPF_IND|BPF_B:
v = val[X_ATOM];
if (alter && vmap[v].is_const) {
s->code = BPF_LD|BPF_ABS|BPF_SIZE(s->code);
s->k += vmap[v].const_val;
v = F(s->code, s->k, 0L);
done = 0;
}
else
v = F(s->code, s->k, v);
vstore(s, &val[A_ATOM], v, alter);
break;
case BPF_LD|BPF_LEN:
v = F(s->code, 0L, 0L);
vstore(s, &val[A_ATOM], v, alter);
break;
case BPF_LD|BPF_IMM:
v = K(s->k);
vstore(s, &val[A_ATOM], v, alter);
break;
case BPF_LDX|BPF_IMM:
v = K(s->k);
vstore(s, &val[X_ATOM], v, alter);
break;
case BPF_LDX|BPF_MSH|BPF_B:
v = F(s->code, s->k, 0L);
vstore(s, &val[X_ATOM], v, alter);
break;
case BPF_ALU|BPF_NEG:
if (alter && vmap[val[A_ATOM]].is_const) {
s->code = BPF_LD|BPF_IMM;
s->k = -vmap[val[A_ATOM]].const_val;
val[A_ATOM] = K(s->k);
}
else
val[A_ATOM] = F(s->code, val[A_ATOM], 0L);
break;
case BPF_ALU|BPF_ADD|BPF_K:
case BPF_ALU|BPF_SUB|BPF_K:
case BPF_ALU|BPF_MUL|BPF_K:
case BPF_ALU|BPF_DIV|BPF_K:
case BPF_ALU|BPF_AND|BPF_K:
case BPF_ALU|BPF_OR|BPF_K:
case BPF_ALU|BPF_LSH|BPF_K:
case BPF_ALU|BPF_RSH|BPF_K:
op = BPF_OP(s->code);
if (alter) {
if (s->k == 0) {
/* don't optimize away "sub #0"
* as it may be needed later to
* fixup the generated math code */
if (op == BPF_ADD ||
op == BPF_LSH || op == BPF_RSH ||
op == BPF_OR) {
s->code = NOP;
break;
}
if (op == BPF_MUL || op == BPF_AND) {
s->code = BPF_LD|BPF_IMM;
val[A_ATOM] = K(s->k);
break;
}
}
if (vmap[val[A_ATOM]].is_const) {
fold_op(s, val[A_ATOM], K(s->k));
val[A_ATOM] = K(s->k);
break;
}
}
val[A_ATOM] = F(s->code, val[A_ATOM], K(s->k));
break;
case BPF_ALU|BPF_ADD|BPF_X:
case BPF_ALU|BPF_SUB|BPF_X:
case BPF_ALU|BPF_MUL|BPF_X:
case BPF_ALU|BPF_DIV|BPF_X:
case BPF_ALU|BPF_AND|BPF_X:
case BPF_ALU|BPF_OR|BPF_X:
case BPF_ALU|BPF_LSH|BPF_X:
case BPF_ALU|BPF_RSH|BPF_X:
op = BPF_OP(s->code);
if (alter && vmap[val[X_ATOM]].is_const) {
if (vmap[val[A_ATOM]].is_const) {
fold_op(s, val[A_ATOM], val[X_ATOM]);
val[A_ATOM] = K(s->k);
}
else {
s->code = BPF_ALU|BPF_K|op;
s->k = vmap[val[X_ATOM]].const_val;
done = 0;
val[A_ATOM] =
F(s->code, val[A_ATOM], K(s->k));
}
break;
}
/*
* Check if we're doing something to an accumulator
* that is 0, and simplify. This may not seem like
* much of a simplification but it could open up further
* optimizations.
* XXX We could also check for mul by 1, etc.
*/
if (alter && vmap[val[A_ATOM]].is_const
&& vmap[val[A_ATOM]].const_val == 0) {
if (op == BPF_ADD || op == BPF_OR) {
s->code = BPF_MISC|BPF_TXA;
vstore(s, &val[A_ATOM], val[X_ATOM], alter);
break;
}
else if (op == BPF_MUL || op == BPF_DIV ||
op == BPF_AND || op == BPF_LSH || op == BPF_RSH) {
s->code = BPF_LD|BPF_IMM;
s->k = 0;
vstore(s, &val[A_ATOM], K(s->k), alter);
break;
}
else if (op == BPF_NEG) {
s->code = NOP;
break;
}
}
val[A_ATOM] = F(s->code, val[A_ATOM], val[X_ATOM]);
break;
case BPF_MISC|BPF_TXA:
vstore(s, &val[A_ATOM], val[X_ATOM], alter);
break;
case BPF_LD|BPF_MEM:
v = val[s->k];
if (alter && vmap[v].is_const) {
s->code = BPF_LD|BPF_IMM;
s->k = vmap[v].const_val;
done = 0;
}
vstore(s, &val[A_ATOM], v, alter);
break;
case BPF_MISC|BPF_TAX:
vstore(s, &val[X_ATOM], val[A_ATOM], alter);
break;
case BPF_LDX|BPF_MEM:
v = val[s->k];
if (alter && vmap[v].is_const) {
s->code = BPF_LDX|BPF_IMM;
s->k = vmap[v].const_val;
done = 0;
}
vstore(s, &val[X_ATOM], v, alter);
break;
case BPF_ST:
vstore(s, &val[s->k], val[A_ATOM], alter);
break;
case BPF_STX:
vstore(s, &val[s->k], val[X_ATOM], alter);
break;
}
}
/*
* Function that does the real stuff.
*/
bpf_filter_func
bpf_jit_compile(struct bpf_insn *prog, u_int nins, size_t *size)
{
bpf_bin_stream stream;
struct bpf_insn *ins;
int flags, fret, fpkt, fmem, fjmp, fadk;
int save_esp;
u_int i, pass;
/*
* NOTE: Do not modify the name of this variable, as it's used by
* the macros to emit code.
*/
emit_func emitm;
flags = bpf_jit_optimize(prog, nins);
fret = (flags & BPF_JIT_FRET) != 0;
fpkt = (flags & BPF_JIT_FPKT) != 0;
fmem = (flags & BPF_JIT_FMEM) != 0;
fjmp = (flags & BPF_JIT_FJMP) != 0;
fadk = (flags & BPF_JIT_FADK) != 0;
save_esp = (fpkt || fmem || fadk); /* Stack is used. */
if (fret)
nins = 1;
memset(&stream, 0, sizeof(stream));
/* Allocate the reference table for the jumps. */
if (fjmp) {
#ifdef _KERNEL
stream.refs = malloc((nins + 1) * sizeof(u_int), M_BPFJIT,
M_NOWAIT | M_ZERO);
#else
stream.refs = calloc(nins + 1, sizeof(u_int));
#endif
if (stream.refs == NULL)
return (NULL);
}
/*
* The first pass will emit the lengths of the instructions
* to create the reference table.
*/
emitm = emit_length;
for (pass = 0; pass < 2; pass++) {
ins = prog;
/* Create the procedure header. */
if (save_esp) {
PUSH(EBP);
MOVrd(ESP, EBP);
}
if (fmem)
SUBib(BPF_MEMWORDS * sizeof(uint32_t), ESP);
if (save_esp)
PUSH(ESI);
if (fpkt) {
PUSH(EDI);
PUSH(EBX);
MOVodd(8, EBP, EBX);
MOVodd(16, EBP, EDI);
}
for (i = 0; i < nins; i++) {
stream.bpf_pc++;
switch (ins->code) {
default:
#ifdef _KERNEL
return (NULL);
#else
abort();
#endif
case BPF_RET|BPF_K:
MOVid(ins->k, EAX);
if (save_esp) {
if (fpkt) {
POP(EBX);
POP(EDI);
}
POP(ESI);
LEAVE();
}
RET();
break;
case BPF_RET|BPF_A:
if (save_esp) {
if (fpkt) {
POP(EBX);
POP(EDI);
}
POP(ESI);
LEAVE();
}
RET();
break;
case BPF_LD|BPF_W|BPF_ABS:
MOVid(ins->k, ESI);
CMPrd(EDI, ESI);
JAb(12);
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int32_t), ECX);
JAEb(7);
ZEROrd(EAX);
POP(EBX);
POP(EDI);
POP(ESI);
LEAVE();
RET();
MOVobd(EBX, ESI, EAX);
BSWAP(EAX);
break;
case BPF_LD|BPF_H|BPF_ABS:
ZEROrd(EAX);
MOVid(ins->k, ESI);
CMPrd(EDI, ESI);
JAb(12);
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int16_t), ECX);
JAEb(5);
POP(EBX);
POP(EDI);
POP(ESI);
LEAVE();
RET();
MOVobw(EBX, ESI, AX);
SWAP_AX();
break;
case BPF_LD|BPF_B|BPF_ABS:
ZEROrd(EAX);
MOVid(ins->k, ESI);
CMPrd(EDI, ESI);
JBb(5);
POP(EBX);
POP(EDI);
POP(ESI);
LEAVE();
RET();
MOVobb(EBX, ESI, AL);
break;
case BPF_LD|BPF_W|BPF_LEN:
if (save_esp)
MOVodd(12, EBP, EAX);
else {
MOVrd(ESP, ECX);
MOVodd(12, ECX, EAX);
}
break;
case BPF_LDX|BPF_W|BPF_LEN:
if (save_esp)
MOVodd(12, EBP, EDX);
else {
MOVrd(ESP, ECX);
MOVodd(12, ECX, EDX);
}
break;
case BPF_LD|BPF_W|BPF_IND:
CMPrd(EDI, EDX);
JAb(27);
MOVid(ins->k, ESI);
MOVrd(EDI, ECX);
SUBrd(EDX, ECX);
CMPrd(ESI, ECX);
JBb(14);
ADDrd(EDX, ESI);
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int32_t), ECX);
JAEb(7);
ZEROrd(EAX);
POP(EBX);
POP(EDI);
POP(ESI);
LEAVE();
RET();
MOVobd(EBX, ESI, EAX);
BSWAP(EAX);
break;
case BPF_LD|BPF_H|BPF_IND:
ZEROrd(EAX);
CMPrd(EDI, EDX);
JAb(27);
MOVid(ins->k, ESI);
MOVrd(EDI, ECX);
SUBrd(EDX, ECX);
CMPrd(ESI, ECX);
JBb(14);
ADDrd(EDX, ESI);
MOVrd(EDI, ECX);
SUBrd(ESI, ECX);
CMPid(sizeof(int16_t), ECX);
JAEb(5);
POP(EBX);
POP(EDI);
POP(ESI);
LEAVE();
RET();
MOVobw(EBX, ESI, AX);
SWAP_AX();
break;
case BPF_LD|BPF_B|BPF_IND:
ZEROrd(EAX);
CMPrd(EDI, EDX);
JAEb(13);
MOVid(ins->k, ESI);
MOVrd(EDI, ECX);
SUBrd(EDX, ECX);
CMPrd(ESI, ECX);
JAb(5);
POP(EBX);
POP(EDI);
POP(ESI);
LEAVE();
RET();
ADDrd(EDX, ESI);
MOVobb(EBX, ESI, AL);
break;
case BPF_LDX|BPF_MSH|BPF_B:
MOVid(ins->k, ESI);
CMPrd(EDI, ESI);
JBb(7);
ZEROrd(EAX);
POP(EBX);
POP(EDI);
POP(ESI);
LEAVE();
RET();
ZEROrd(EDX);
MOVobb(EBX, ESI, DL);
ANDib(0x0f, DL);
SHLib(2, EDX);
break;
case BPF_LD|BPF_IMM:
MOVid(ins->k, EAX);
break;
case BPF_LDX|BPF_IMM:
MOVid(ins->k, EDX);
break;
case BPF_LD|BPF_MEM:
MOVrd(EBP, ECX);
MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVobd(ECX, ESI, EAX);
break;
case BPF_LDX|BPF_MEM:
MOVrd(EBP, ECX);
MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVobd(ECX, ESI, EDX);
break;
case BPF_ST:
/*
* XXX this command and the following could
* be optimized if the previous instruction
* was already of this type
*/
MOVrd(EBP, ECX);
MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVomd(EAX, ECX, ESI);
break;
case BPF_STX:
MOVrd(EBP, ECX);
MOVid(((int)ins->k - BPF_MEMWORDS) *
sizeof(uint32_t), ESI);
MOVomd(EDX, ECX, ESI);
break;
case BPF_JMP|BPF_JA:
JUMP(ins->k);
break;
case BPF_JMP|BPF_JGT|BPF_K:
case BPF_JMP|BPF_JGE|BPF_K:
case BPF_JMP|BPF_JEQ|BPF_K:
case BPF_JMP|BPF_JSET|BPF_K:
case BPF_JMP|BPF_JGT|BPF_X:
case BPF_JMP|BPF_JGE|BPF_X:
case BPF_JMP|BPF_JEQ|BPF_X:
case BPF_JMP|BPF_JSET|BPF_X:
if (ins->jt == ins->jf) {
JUMP(ins->jt);
break;
}
switch (ins->code) {
case BPF_JMP|BPF_JGT|BPF_K:
CMPid(ins->k, EAX);
JCC(JA, JBE);
break;
case BPF_JMP|BPF_JGE|BPF_K:
CMPid(ins->k, EAX);
JCC(JAE, JB);
break;
case BPF_JMP|BPF_JEQ|BPF_K:
CMPid(ins->k, EAX);
JCC(JE, JNE);
break;
case BPF_JMP|BPF_JSET|BPF_K:
TESTid(ins->k, EAX);
JCC(JNE, JE);
break;
case BPF_JMP|BPF_JGT|BPF_X:
CMPrd(EDX, EAX);
JCC(JA, JBE);
break;
case BPF_JMP|BPF_JGE|BPF_X:
CMPrd(EDX, EAX);
JCC(JAE, JB);
break;
case BPF_JMP|BPF_JEQ|BPF_X:
CMPrd(EDX, EAX);
JCC(JE, JNE);
break;
case BPF_JMP|BPF_JSET|BPF_X:
TESTrd(EDX, EAX);
JCC(JNE, JE);
break;
}
break;
case BPF_ALU|BPF_ADD|BPF_X:
ADDrd(EDX, EAX);
break;
case BPF_ALU|BPF_SUB|BPF_X:
SUBrd(EDX, EAX);
break;
case BPF_ALU|BPF_MUL|BPF_X:
MOVrd(EDX, ECX);
MULrd(EDX);
MOVrd(ECX, EDX);
break;
case BPF_ALU|BPF_DIV|BPF_X:
case BPF_ALU|BPF_MOD|BPF_X:
TESTrd(EDX, EDX);
if (save_esp) {
if (fpkt) {
JNEb(7);
ZEROrd(EAX);
POP(EBX);
POP(EDI);
} else {
JNEb(5);
ZEROrd(EAX);
}
POP(ESI);
LEAVE();
} else {
JNEb(3);
ZEROrd(EAX);
}
RET();
MOVrd(EDX, ECX);
ZEROrd(EDX);
DIVrd(ECX);
if (BPF_OP(ins->code) == BPF_MOD)
MOVrd(EDX, EAX);
MOVrd(ECX, EDX);
break;
case BPF_ALU|BPF_AND|BPF_X:
ANDrd(EDX, EAX);
break;
case BPF_ALU|BPF_OR|BPF_X:
ORrd(EDX, EAX);
break;
case BPF_ALU|BPF_XOR|BPF_X:
XORrd(EDX, EAX);
break;
case BPF_ALU|BPF_LSH|BPF_X:
MOVrd(EDX, ECX);
SHL_CLrb(EAX);
break;
case BPF_ALU|BPF_RSH|BPF_X:
MOVrd(EDX, ECX);
SHR_CLrb(EAX);
break;
case BPF_ALU|BPF_ADD|BPF_K:
ADD_EAXi(ins->k);
break;
case BPF_ALU|BPF_SUB|BPF_K:
SUB_EAXi(ins->k);
break;
case BPF_ALU|BPF_MUL|BPF_K:
MOVrd(EDX, ECX);
MOVid(ins->k, EDX);
MULrd(EDX);
MOVrd(ECX, EDX);
break;
case BPF_ALU|BPF_DIV|BPF_K:
case BPF_ALU|BPF_MOD|BPF_K:
MOVrd(EDX, ECX);
ZEROrd(EDX);
MOVid(ins->k, ESI);
DIVrd(ESI);
if (BPF_OP(ins->code) == BPF_MOD)
MOVrd(EDX, EAX);
MOVrd(ECX, EDX);
break;
case BPF_ALU|BPF_AND|BPF_K:
ANDid(ins->k, EAX);
break;
case BPF_ALU|BPF_OR|BPF_K:
ORid(ins->k, EAX);
break;
case BPF_ALU|BPF_XOR|BPF_K:
XORid(ins->k, EAX);
break;
case BPF_ALU|BPF_LSH|BPF_K:
SHLib((ins->k) & 0xff, EAX);
break;
case BPF_ALU|BPF_RSH|BPF_K:
SHRib((ins->k) & 0xff, EAX);
break;
case BPF_ALU|BPF_NEG:
NEGd(EAX);
break;
case BPF_MISC|BPF_TAX:
MOVrd(EAX, EDX);
break;
case BPF_MISC|BPF_TXA:
MOVrd(EDX, EAX);
break;
}
ins++;
}
if (pass > 0)
continue;
*size = stream.cur_ip;
#ifdef _KERNEL
stream.ibuf = malloc(*size, M_BPFJIT, M_NOWAIT);
if (stream.ibuf == NULL)
break;
#else
stream.ibuf = mmap(NULL, *size, PROT_READ | PROT_WRITE,
MAP_ANON, -1, 0);
if (stream.ibuf == MAP_FAILED) {
stream.ibuf = NULL;
break;
}
#endif
/*
* Modify the reference table to contain the offsets and
* not the lengths of the instructions.
*/
if (fjmp)
for (i = 1; i < nins + 1; i++)
stream.refs[i] += stream.refs[i - 1];
/* Reset the counters. */
stream.cur_ip = 0;
stream.bpf_pc = 0;
/* The second pass creates the actual code. */
emitm = emit_code;
}
/*
* The reference table is needed only during compilation,
* now we can free it.
*/
if (fjmp)
#ifdef _KERNEL
free(stream.refs, M_BPFJIT);
#else
free(stream.refs);
#endif
#ifndef _KERNEL
if (stream.ibuf != NULL &&
mprotect(stream.ibuf, *size, PROT_READ | PROT_EXEC) != 0) {
munmap(stream.ibuf, *size);
stream.ibuf = NULL;
}
#endif
return ((bpf_filter_func)(void *)stream.ibuf);
}
static int do_jit(struct bpf_prog *bpf_prog, int *addrs, u8 *image,
int oldproglen, struct jit_context *ctx)
{
struct bpf_insn *insn = bpf_prog->insnsi;
int insn_cnt = bpf_prog->len;
bool seen_ld_abs = ctx->seen_ld_abs | (oldproglen == 0);
bool seen_exit = false;
u8 temp[BPF_MAX_INSN_SIZE + BPF_INSN_SAFETY];
int i, cnt = 0;
int proglen = 0;
u8 *prog = temp;
emit_prologue(&prog);
if (seen_ld_abs)
emit_load_skb_data_hlen(&prog);
for (i = 0; i < insn_cnt; i++, insn++) {
const s32 imm32 = insn->imm;
u32 dst_reg = insn->dst_reg;
u32 src_reg = insn->src_reg;
u8 b1 = 0, b2 = 0, b3 = 0;
s64 jmp_offset;
u8 jmp_cond;
bool reload_skb_data;
int ilen;
u8 *func;
switch (insn->code) {
/* ALU */
case BPF_ALU | BPF_ADD | BPF_X:
case BPF_ALU | BPF_SUB | BPF_X:
case BPF_ALU | BPF_AND | BPF_X:
case BPF_ALU | BPF_OR | BPF_X:
case BPF_ALU | BPF_XOR | BPF_X:
case BPF_ALU64 | BPF_ADD | BPF_X:
case BPF_ALU64 | BPF_SUB | BPF_X:
case BPF_ALU64 | BPF_AND | BPF_X:
case BPF_ALU64 | BPF_OR | BPF_X:
case BPF_ALU64 | BPF_XOR | BPF_X:
switch (BPF_OP(insn->code)) {
case BPF_ADD: b2 = 0x01; break;
case BPF_SUB: b2 = 0x29; break;
case BPF_AND: b2 = 0x21; break;
case BPF_OR: b2 = 0x09; break;
case BPF_XOR: b2 = 0x31; break;
}
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_2mod(0x48, dst_reg, src_reg));
else if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT1(add_2mod(0x40, dst_reg, src_reg));
EMIT2(b2, add_2reg(0xC0, dst_reg, src_reg));
break;
/* mov dst, src */
case BPF_ALU64 | BPF_MOV | BPF_X:
EMIT_mov(dst_reg, src_reg);
break;
/* mov32 dst, src */
case BPF_ALU | BPF_MOV | BPF_X:
if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT1(add_2mod(0x40, dst_reg, src_reg));
EMIT2(0x89, add_2reg(0xC0, dst_reg, src_reg));
break;
/* neg dst */
case BPF_ALU | BPF_NEG:
case BPF_ALU64 | BPF_NEG:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
EMIT2(0xF7, add_1reg(0xD8, dst_reg));
break;
case BPF_ALU | BPF_ADD | BPF_K:
case BPF_ALU | BPF_SUB | BPF_K:
case BPF_ALU | BPF_AND | BPF_K:
case BPF_ALU | BPF_OR | BPF_K:
case BPF_ALU | BPF_XOR | BPF_K:
case BPF_ALU64 | BPF_ADD | BPF_K:
case BPF_ALU64 | BPF_SUB | BPF_K:
case BPF_ALU64 | BPF_AND | BPF_K:
case BPF_ALU64 | BPF_OR | BPF_K:
case BPF_ALU64 | BPF_XOR | BPF_K:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
switch (BPF_OP(insn->code)) {
case BPF_ADD: b3 = 0xC0; break;
case BPF_SUB: b3 = 0xE8; break;
case BPF_AND: b3 = 0xE0; break;
case BPF_OR: b3 = 0xC8; break;
case BPF_XOR: b3 = 0xF0; break;
}
if (is_imm8(imm32))
EMIT3(0x83, add_1reg(b3, dst_reg), imm32);
else
EMIT2_off32(0x81, add_1reg(b3, dst_reg), imm32);
break;
case BPF_ALU64 | BPF_MOV | BPF_K:
/* optimization: if imm32 is positive,
* use 'mov eax, imm32' (which zero-extends imm32)
* to save 2 bytes
*/
if (imm32 < 0) {
/* 'mov rax, imm32' sign extends imm32 */
b1 = add_1mod(0x48, dst_reg);
b2 = 0xC7;
b3 = 0xC0;
EMIT3_off32(b1, b2, add_1reg(b3, dst_reg), imm32);
break;
}
case BPF_ALU | BPF_MOV | BPF_K:
/* optimization: if imm32 is zero, use 'xor <dst>,<dst>'
* to save 3 bytes.
*/
if (imm32 == 0) {
if (is_ereg(dst_reg))
EMIT1(add_2mod(0x40, dst_reg, dst_reg));
b2 = 0x31; /* xor */
b3 = 0xC0;
EMIT2(b2, add_2reg(b3, dst_reg, dst_reg));
break;
}
/* mov %eax, imm32 */
if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
EMIT1_off32(add_1reg(0xB8, dst_reg), imm32);
break;
case BPF_LD | BPF_IMM | BPF_DW:
if (insn[1].code != 0 || insn[1].src_reg != 0 ||
insn[1].dst_reg != 0 || insn[1].off != 0) {
/* verifier must catch invalid insns */
pr_err("invalid BPF_LD_IMM64 insn\n");
return -EINVAL;
}
/* optimization: if imm64 is zero, use 'xor <dst>,<dst>'
* to save 7 bytes.
*/
if (insn[0].imm == 0 && insn[1].imm == 0) {
b1 = add_2mod(0x48, dst_reg, dst_reg);
b2 = 0x31; /* xor */
b3 = 0xC0;
EMIT3(b1, b2, add_2reg(b3, dst_reg, dst_reg));
insn++;
i++;
break;
}
/* movabsq %rax, imm64 */
EMIT2(add_1mod(0x48, dst_reg), add_1reg(0xB8, dst_reg));
EMIT(insn[0].imm, 4);
EMIT(insn[1].imm, 4);
insn++;
i++;
break;
/* dst %= src, dst /= src, dst %= imm32, dst /= imm32 */
case BPF_ALU | BPF_MOD | BPF_X:
case BPF_ALU | BPF_DIV | BPF_X:
case BPF_ALU | BPF_MOD | BPF_K:
case BPF_ALU | BPF_DIV | BPF_K:
case BPF_ALU64 | BPF_MOD | BPF_X:
case BPF_ALU64 | BPF_DIV | BPF_X:
case BPF_ALU64 | BPF_MOD | BPF_K:
case BPF_ALU64 | BPF_DIV | BPF_K:
EMIT1(0x50); /* push rax */
EMIT1(0x52); /* push rdx */
if (BPF_SRC(insn->code) == BPF_X)
/* mov r11, src_reg */
EMIT_mov(AUX_REG, src_reg);
else
/* mov r11, imm32 */
EMIT3_off32(0x49, 0xC7, 0xC3, imm32);
/* mov rax, dst_reg */
EMIT_mov(BPF_REG_0, dst_reg);
/* xor edx, edx
* equivalent to 'xor rdx, rdx', but one byte less
*/
EMIT2(0x31, 0xd2);
if (BPF_SRC(insn->code) == BPF_X) {
/* if (src_reg == 0) return 0 */
/* cmp r11, 0 */
EMIT4(0x49, 0x83, 0xFB, 0x00);
/* jne .+9 (skip over pop, pop, xor and jmp) */
EMIT2(X86_JNE, 1 + 1 + 2 + 5);
EMIT1(0x5A); /* pop rdx */
EMIT1(0x58); /* pop rax */
EMIT2(0x31, 0xc0); /* xor eax, eax */
/* jmp cleanup_addr
* addrs[i] - 11, because there are 11 bytes
* after this insn: div, mov, pop, pop, mov
*/
jmp_offset = ctx->cleanup_addr - (addrs[i] - 11);
EMIT1_off32(0xE9, jmp_offset);
}
if (BPF_CLASS(insn->code) == BPF_ALU64)
/* div r11 */
EMIT3(0x49, 0xF7, 0xF3);
else
/* div r11d */
EMIT3(0x41, 0xF7, 0xF3);
if (BPF_OP(insn->code) == BPF_MOD)
/* mov r11, rdx */
EMIT3(0x49, 0x89, 0xD3);
else
/* mov r11, rax */
EMIT3(0x49, 0x89, 0xC3);
EMIT1(0x5A); /* pop rdx */
EMIT1(0x58); /* pop rax */
/* mov dst_reg, r11 */
EMIT_mov(dst_reg, AUX_REG);
break;
case BPF_ALU | BPF_MUL | BPF_K:
case BPF_ALU | BPF_MUL | BPF_X:
case BPF_ALU64 | BPF_MUL | BPF_K:
case BPF_ALU64 | BPF_MUL | BPF_X:
EMIT1(0x50); /* push rax */
EMIT1(0x52); /* push rdx */
/* mov r11, dst_reg */
EMIT_mov(AUX_REG, dst_reg);
if (BPF_SRC(insn->code) == BPF_X)
/* mov rax, src_reg */
EMIT_mov(BPF_REG_0, src_reg);
else
/* mov rax, imm32 */
EMIT3_off32(0x48, 0xC7, 0xC0, imm32);
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, AUX_REG));
else if (is_ereg(AUX_REG))
EMIT1(add_1mod(0x40, AUX_REG));
/* mul(q) r11 */
EMIT2(0xF7, add_1reg(0xE0, AUX_REG));
/* mov r11, rax */
EMIT_mov(AUX_REG, BPF_REG_0);
EMIT1(0x5A); /* pop rdx */
EMIT1(0x58); /* pop rax */
/* mov dst_reg, r11 */
EMIT_mov(dst_reg, AUX_REG);
break;
/* shifts */
case BPF_ALU | BPF_LSH | BPF_K:
case BPF_ALU | BPF_RSH | BPF_K:
case BPF_ALU | BPF_ARSH | BPF_K:
case BPF_ALU64 | BPF_LSH | BPF_K:
case BPF_ALU64 | BPF_RSH | BPF_K:
case BPF_ALU64 | BPF_ARSH | BPF_K:
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
switch (BPF_OP(insn->code)) {
case BPF_LSH: b3 = 0xE0; break;
case BPF_RSH: b3 = 0xE8; break;
case BPF_ARSH: b3 = 0xF8; break;
}
EMIT3(0xC1, add_1reg(b3, dst_reg), imm32);
break;
case BPF_ALU | BPF_LSH | BPF_X:
case BPF_ALU | BPF_RSH | BPF_X:
case BPF_ALU | BPF_ARSH | BPF_X:
case BPF_ALU64 | BPF_LSH | BPF_X:
case BPF_ALU64 | BPF_RSH | BPF_X:
case BPF_ALU64 | BPF_ARSH | BPF_X:
/* check for bad case when dst_reg == rcx */
if (dst_reg == BPF_REG_4) {
/* mov r11, dst_reg */
EMIT_mov(AUX_REG, dst_reg);
dst_reg = AUX_REG;
}
if (src_reg != BPF_REG_4) { /* common case */
EMIT1(0x51); /* push rcx */
/* mov rcx, src_reg */
EMIT_mov(BPF_REG_4, src_reg);
}
/* shl %rax, %cl | shr %rax, %cl | sar %rax, %cl */
if (BPF_CLASS(insn->code) == BPF_ALU64)
EMIT1(add_1mod(0x48, dst_reg));
else if (is_ereg(dst_reg))
EMIT1(add_1mod(0x40, dst_reg));
switch (BPF_OP(insn->code)) {
case BPF_LSH: b3 = 0xE0; break;
case BPF_RSH: b3 = 0xE8; break;
case BPF_ARSH: b3 = 0xF8; break;
}
EMIT2(0xD3, add_1reg(b3, dst_reg));
if (src_reg != BPF_REG_4)
EMIT1(0x59); /* pop rcx */
if (insn->dst_reg == BPF_REG_4)
/* mov dst_reg, r11 */
EMIT_mov(insn->dst_reg, AUX_REG);
break;
case BPF_ALU | BPF_END | BPF_FROM_BE:
switch (imm32) {
case 16:
/* emit 'ror %ax, 8' to swap lower 2 bytes */
EMIT1(0x66);
if (is_ereg(dst_reg))
EMIT1(0x41);
EMIT3(0xC1, add_1reg(0xC8, dst_reg), 8);
/* emit 'movzwl eax, ax' */
if (is_ereg(dst_reg))
EMIT3(0x45, 0x0F, 0xB7);
else
EMIT2(0x0F, 0xB7);
EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
break;
case 32:
/* emit 'bswap eax' to swap lower 4 bytes */
if (is_ereg(dst_reg))
EMIT2(0x41, 0x0F);
else
EMIT1(0x0F);
EMIT1(add_1reg(0xC8, dst_reg));
break;
case 64:
/* emit 'bswap rax' to swap 8 bytes */
EMIT3(add_1mod(0x48, dst_reg), 0x0F,
add_1reg(0xC8, dst_reg));
break;
}
break;
case BPF_ALU | BPF_END | BPF_FROM_LE:
switch (imm32) {
case 16:
/* emit 'movzwl eax, ax' to zero extend 16-bit
* into 64 bit
*/
if (is_ereg(dst_reg))
EMIT3(0x45, 0x0F, 0xB7);
else
EMIT2(0x0F, 0xB7);
EMIT1(add_2reg(0xC0, dst_reg, dst_reg));
break;
case 32:
/* emit 'mov eax, eax' to clear upper 32-bits */
if (is_ereg(dst_reg))
EMIT1(0x45);
EMIT2(0x89, add_2reg(0xC0, dst_reg, dst_reg));
break;
case 64:
/* nop */
break;
}
break;
/* ST: *(u8*)(dst_reg + off) = imm */
case BPF_ST | BPF_MEM | BPF_B:
if (is_ereg(dst_reg))
EMIT2(0x41, 0xC6);
else
EMIT1(0xC6);
goto st;
case BPF_ST | BPF_MEM | BPF_H:
if (is_ereg(dst_reg))
EMIT3(0x66, 0x41, 0xC7);
else
EMIT2(0x66, 0xC7);
goto st;
case BPF_ST | BPF_MEM | BPF_W:
if (is_ereg(dst_reg))
EMIT2(0x41, 0xC7);
else
EMIT1(0xC7);
goto st;
case BPF_ST | BPF_MEM | BPF_DW:
EMIT2(add_1mod(0x48, dst_reg), 0xC7);
st: if (is_imm8(insn->off))
EMIT2(add_1reg(0x40, dst_reg), insn->off);
else
EMIT1_off32(add_1reg(0x80, dst_reg), insn->off);
EMIT(imm32, bpf_size_to_x86_bytes(BPF_SIZE(insn->code)));
break;
/* STX: *(u8*)(dst_reg + off) = src_reg */
case BPF_STX | BPF_MEM | BPF_B:
/* emit 'mov byte ptr [rax + off], al' */
if (is_ereg(dst_reg) || is_ereg(src_reg) ||
/* have to add extra byte for x86 SIL, DIL regs */
src_reg == BPF_REG_1 || src_reg == BPF_REG_2)
EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x88);
else
EMIT1(0x88);
goto stx;
case BPF_STX | BPF_MEM | BPF_H:
if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT3(0x66, add_2mod(0x40, dst_reg, src_reg), 0x89);
else
EMIT2(0x66, 0x89);
goto stx;
case BPF_STX | BPF_MEM | BPF_W:
if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT2(add_2mod(0x40, dst_reg, src_reg), 0x89);
else
EMIT1(0x89);
goto stx;
case BPF_STX | BPF_MEM | BPF_DW:
EMIT2(add_2mod(0x48, dst_reg, src_reg), 0x89);
stx: if (is_imm8(insn->off))
EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
else
EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
insn->off);
break;
/* LDX: dst_reg = *(u8*)(src_reg + off) */
case BPF_LDX | BPF_MEM | BPF_B:
/* emit 'movzx rax, byte ptr [rax + off]' */
EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB6);
goto ldx;
case BPF_LDX | BPF_MEM | BPF_H:
/* emit 'movzx rax, word ptr [rax + off]' */
EMIT3(add_2mod(0x48, src_reg, dst_reg), 0x0F, 0xB7);
goto ldx;
case BPF_LDX | BPF_MEM | BPF_W:
/* emit 'mov eax, dword ptr [rax+0x14]' */
if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT2(add_2mod(0x40, src_reg, dst_reg), 0x8B);
else
EMIT1(0x8B);
goto ldx;
case BPF_LDX | BPF_MEM | BPF_DW:
/* emit 'mov rax, qword ptr [rax+0x14]' */
EMIT2(add_2mod(0x48, src_reg, dst_reg), 0x8B);
ldx: /* if insn->off == 0 we can save one extra byte, but
* special case of x86 r13 which always needs an offset
* is not worth the hassle
*/
if (is_imm8(insn->off))
EMIT2(add_2reg(0x40, src_reg, dst_reg), insn->off);
else
EMIT1_off32(add_2reg(0x80, src_reg, dst_reg),
insn->off);
break;
/* STX XADD: lock *(u32*)(dst_reg + off) += src_reg */
case BPF_STX | BPF_XADD | BPF_W:
/* emit 'lock add dword ptr [rax + off], eax' */
if (is_ereg(dst_reg) || is_ereg(src_reg))
EMIT3(0xF0, add_2mod(0x40, dst_reg, src_reg), 0x01);
else
EMIT2(0xF0, 0x01);
goto xadd;
case BPF_STX | BPF_XADD | BPF_DW:
EMIT3(0xF0, add_2mod(0x48, dst_reg, src_reg), 0x01);
xadd: if (is_imm8(insn->off))
EMIT2(add_2reg(0x40, dst_reg, src_reg), insn->off);
else
EMIT1_off32(add_2reg(0x80, dst_reg, src_reg),
insn->off);
break;
/* call */
case BPF_JMP | BPF_CALL:
func = (u8 *) __bpf_call_base + imm32;
jmp_offset = func - (image + addrs[i]);
if (seen_ld_abs) {
reload_skb_data = bpf_helper_changes_skb_data(func);
if (reload_skb_data) {
EMIT1(0x57); /* push %rdi */
jmp_offset += 22; /* pop, mov, sub, mov */
} else {
EMIT2(0x41, 0x52); /* push %r10 */
EMIT2(0x41, 0x51); /* push %r9 */
/* need to adjust jmp offset, since
* pop %r9, pop %r10 take 4 bytes after call insn
*/
jmp_offset += 4;
}
}
if (!imm32 || !is_simm32(jmp_offset)) {
pr_err("unsupported bpf func %d addr %p image %p\n",
imm32, func, image);
return -EINVAL;
}
EMIT1_off32(0xE8, jmp_offset);
if (seen_ld_abs) {
if (reload_skb_data) {
EMIT1(0x5F); /* pop %rdi */
emit_load_skb_data_hlen(&prog);
} else {
EMIT2(0x41, 0x59); /* pop %r9 */
EMIT2(0x41, 0x5A); /* pop %r10 */
}
}
break;
case BPF_JMP | BPF_CALL | BPF_X:
emit_bpf_tail_call(&prog);
break;
/* cond jump */
case BPF_JMP | BPF_JEQ | BPF_X:
case BPF_JMP | BPF_JNE | BPF_X:
case BPF_JMP | BPF_JGT | BPF_X:
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP | BPF_JSGT | BPF_X:
case BPF_JMP | BPF_JSGE | BPF_X:
/* cmp dst_reg, src_reg */
EMIT3(add_2mod(0x48, dst_reg, src_reg), 0x39,
add_2reg(0xC0, dst_reg, src_reg));
goto emit_cond_jmp;
case BPF_JMP | BPF_JSET | BPF_X:
/* test dst_reg, src_reg */
EMIT3(add_2mod(0x48, dst_reg, src_reg), 0x85,
add_2reg(0xC0, dst_reg, src_reg));
goto emit_cond_jmp;
case BPF_JMP | BPF_JSET | BPF_K:
/* test dst_reg, imm32 */
EMIT1(add_1mod(0x48, dst_reg));
EMIT2_off32(0xF7, add_1reg(0xC0, dst_reg), imm32);
goto emit_cond_jmp;
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP | BPF_JSGE | BPF_K:
/* cmp dst_reg, imm8/32 */
EMIT1(add_1mod(0x48, dst_reg));
if (is_imm8(imm32))
EMIT3(0x83, add_1reg(0xF8, dst_reg), imm32);
else
EMIT2_off32(0x81, add_1reg(0xF8, dst_reg), imm32);
emit_cond_jmp: /* convert BPF opcode to x86 */
switch (BPF_OP(insn->code)) {
case BPF_JEQ:
jmp_cond = X86_JE;
break;
case BPF_JSET:
case BPF_JNE:
jmp_cond = X86_JNE;
break;
case BPF_JGT:
/* GT is unsigned '>', JA in x86 */
jmp_cond = X86_JA;
break;
case BPF_JGE:
/* GE is unsigned '>=', JAE in x86 */
jmp_cond = X86_JAE;
break;
case BPF_JSGT:
/* signed '>', GT in x86 */
jmp_cond = X86_JG;
break;
case BPF_JSGE:
/* signed '>=', GE in x86 */
jmp_cond = X86_JGE;
break;
default: /* to silence gcc warning */
return -EFAULT;
}
jmp_offset = addrs[i + insn->off] - addrs[i];
if (is_imm8(jmp_offset)) {
EMIT2(jmp_cond, jmp_offset);
} else if (is_simm32(jmp_offset)) {
EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset);
} else {
pr_err("cond_jmp gen bug %llx\n", jmp_offset);
return -EFAULT;
}
break;
case BPF_JMP | BPF_JA:
jmp_offset = addrs[i + insn->off] - addrs[i];
if (!jmp_offset)
/* optimize out nop jumps */
break;
emit_jmp:
if (is_imm8(jmp_offset)) {
EMIT2(0xEB, jmp_offset);
} else if (is_simm32(jmp_offset)) {
EMIT1_off32(0xE9, jmp_offset);
} else {
pr_err("jmp gen bug %llx\n", jmp_offset);
return -EFAULT;
}
break;
case BPF_LD | BPF_IND | BPF_W:
func = sk_load_word;
goto common_load;
case BPF_LD | BPF_ABS | BPF_W:
func = CHOOSE_LOAD_FUNC(imm32, sk_load_word);
common_load:
ctx->seen_ld_abs = seen_ld_abs = true;
jmp_offset = func - (image + addrs[i]);
if (!func || !is_simm32(jmp_offset)) {
pr_err("unsupported bpf func %d addr %p image %p\n",
imm32, func, image);
return -EINVAL;
}
if (BPF_MODE(insn->code) == BPF_ABS) {
/* mov %esi, imm32 */
EMIT1_off32(0xBE, imm32);
} else {
/* mov %rsi, src_reg */
EMIT_mov(BPF_REG_2, src_reg);
if (imm32) {
if (is_imm8(imm32))
/* add %esi, imm8 */
EMIT3(0x83, 0xC6, imm32);
else
/* add %esi, imm32 */
EMIT2_off32(0x81, 0xC6, imm32);
}
}
/* skb pointer is in R6 (%rbx), it will be copied into
* %rdi if skb_copy_bits() call is necessary.
* sk_load_* helpers also use %r10 and %r9d.
* See bpf_jit.S
*/
EMIT1_off32(0xE8, jmp_offset); /* call */
break;
case BPF_LD | BPF_IND | BPF_H:
func = sk_load_half;
goto common_load;
case BPF_LD | BPF_ABS | BPF_H:
func = CHOOSE_LOAD_FUNC(imm32, sk_load_half);
goto common_load;
case BPF_LD | BPF_IND | BPF_B:
func = sk_load_byte;
goto common_load;
case BPF_LD | BPF_ABS | BPF_B:
func = CHOOSE_LOAD_FUNC(imm32, sk_load_byte);
goto common_load;
case BPF_JMP | BPF_EXIT:
if (seen_exit) {
jmp_offset = ctx->cleanup_addr - addrs[i];
goto emit_jmp;
}
seen_exit = true;
/* update cleanup_addr */
ctx->cleanup_addr = proglen;
/* mov rbx, qword ptr [rbp-X] */
EMIT3_off32(0x48, 0x8B, 0x9D, -STACKSIZE);
/* mov r13, qword ptr [rbp-X] */
EMIT3_off32(0x4C, 0x8B, 0xAD, -STACKSIZE + 8);
/* mov r14, qword ptr [rbp-X] */
EMIT3_off32(0x4C, 0x8B, 0xB5, -STACKSIZE + 16);
/* mov r15, qword ptr [rbp-X] */
EMIT3_off32(0x4C, 0x8B, 0xBD, -STACKSIZE + 24);
EMIT1(0xC9); /* leave */
EMIT1(0xC3); /* ret */
break;
default:
/* By design x64 JIT should support all BPF instructions
* This error will be seen if new instruction was added
* to interpreter, but not to JIT
* or if there is junk in bpf_prog
*/
pr_err("bpf_jit: unknown opcode %02x\n", insn->code);
return -EINVAL;
}
ilen = prog - temp;
if (ilen > BPF_MAX_INSN_SIZE) {
pr_err("bpf_jit_compile fatal insn size error\n");
return -EFAULT;
}
if (image) {
if (unlikely(proglen + ilen > oldproglen)) {
pr_err("bpf_jit_compile fatal error\n");
return -EFAULT;
}
memcpy(image + proglen, temp, ilen);
}
proglen += ilen;
addrs[i] = proglen;
prog = temp;
}
return proglen;
}
static int disassemble(RAsm *a, RAsmOp *r_op, const ut8 *buf, int len) {
const char *op, *fmt;
RBpfSockFilter * f = (RBpfSockFilter*)buf;
int val = f->k;
char vbuf[256];
switch (f->code) {
case BPF_RET | BPF_K:
op = r_bpf_op_table[BPF_RET];
fmt = "%#x";
break;
case BPF_RET | BPF_A:
op = r_bpf_op_table[BPF_RET];
fmt = "a";
break;
case BPF_RET | BPF_X:
op = r_bpf_op_table[BPF_RET];
fmt = "x";
break;
case BPF_MISC_TAX:
op = r_bpf_op_table[BPF_MISC_TAX];
fmt = "";
break;
case BPF_MISC_TXA:
op = r_bpf_op_table[BPF_MISC_TXA];
fmt = "";
break;
case BPF_ST:
op = r_bpf_op_table[BPF_ST];
fmt = "M[%d]";
break;
case BPF_STX:
op = r_bpf_op_table[BPF_STX];
fmt = "M[%d]";
break;
case BPF_LD_W | BPF_ABS:
op = r_bpf_op_table[BPF_LD_W];
fmt = "[%d]";
break;
case BPF_LD_H | BPF_ABS:
op = r_bpf_op_table[BPF_LD_H];
fmt = "[%d]";
break;
case BPF_LD_B | BPF_ABS:
op = r_bpf_op_table[BPF_LD_B];
fmt = "[%d]";
break;
case BPF_LD_W | BPF_LEN:
op = r_bpf_op_table[BPF_LD_W];
fmt = "len";
break;
case BPF_LD_W | BPF_IND:
op = r_bpf_op_table[BPF_LD_W];
fmt = "[x+%d]";
break;
case BPF_LD_H | BPF_IND:
op = r_bpf_op_table[BPF_LD_H];
fmt = "[x+%d]";
break;
case BPF_LD_B | BPF_IND:
op = r_bpf_op_table[BPF_LD_B];
fmt = "[x+%d]";
break;
case BPF_LD | BPF_IMM:
op = r_bpf_op_table[BPF_LD_W];
fmt = "%#x";
break;
case BPF_LDX | BPF_IMM:
op = r_bpf_op_table[BPF_LDX];
fmt = "%#x";
break;
case BPF_LDX | BPF_LEN:
op = r_bpf_op_table[BPF_LDX];
fmt = "len";
break;
case BPF_LDX | BPF_ABS:
op = r_bpf_op_table[BPF_LDX];
fmt = "[%d]";
break;
case BPF_LDX_B | BPF_MSH:
op = r_bpf_op_table[BPF_LDX_B];
fmt = "4*([%d]&0xf)";
break;
case BPF_LD | BPF_MEM:
op = r_bpf_op_table[BPF_LD_W];
fmt = "M[%d]";
break;
case BPF_LDX | BPF_MEM:
op = r_bpf_op_table[BPF_LDX];
fmt = "M[%d]";
break;
case BPF_JMP_JA:
op = r_bpf_op_table[BPF_JMP_JA];
fmt = "%d";
val = a->pc + 8 + f->k * 8;
break;
case BPF_JMP_JGT | BPF_X:
op = r_bpf_op_table[BPF_JMP_JGT];
fmt = "x";
break;
case BPF_JMP_JGT | BPF_K:
op = r_bpf_op_table[BPF_JMP_JGT];
fmt = "%#x";
break;
case BPF_JMP_JGE | BPF_X:
op = r_bpf_op_table[BPF_JMP_JGE];
fmt = "x";
break;
case BPF_JMP_JGE | BPF_K:
op = r_bpf_op_table[BPF_JMP_JGE];
fmt = "%#x";
break;
case BPF_JMP_JEQ | BPF_X:
op = r_bpf_op_table[BPF_JMP_JEQ];
fmt = "x";
break;
case BPF_JMP_JEQ | BPF_K:
op = r_bpf_op_table[BPF_JMP_JEQ];
fmt = "%#x";
break;
case BPF_JMP_JSET | BPF_X:
op = r_bpf_op_table[BPF_JMP_JSET];
fmt = "x";
break;
case BPF_JMP_JSET | BPF_K:
op = r_bpf_op_table[BPF_JMP_JSET];
fmt = "%#x";
break;
case BPF_ALU_NEG:
op = r_bpf_op_table[BPF_ALU_NEG];
fmt = "";
break;
case BPF_ALU_LSH | BPF_X:
op = r_bpf_op_table[BPF_ALU_LSH];
fmt = "x";
break;
case BPF_ALU_LSH | BPF_K:
op = r_bpf_op_table[BPF_ALU_LSH];
fmt = "%d";
break;
case BPF_ALU_RSH | BPF_X:
op = r_bpf_op_table[BPF_ALU_RSH];
fmt = "x";
break;
case BPF_ALU_RSH | BPF_K:
op = r_bpf_op_table[BPF_ALU_RSH];
fmt = "%d";
break;
case BPF_ALU_ADD | BPF_X:
op = r_bpf_op_table[BPF_ALU_ADD];
fmt = "x";
break;
case BPF_ALU_ADD | BPF_K:
op = r_bpf_op_table[BPF_ALU_ADD];
fmt = "%d";
break;
case BPF_ALU_SUB | BPF_X:
op = r_bpf_op_table[BPF_ALU_SUB];
fmt = "x";
break;
case BPF_ALU_SUB | BPF_K:
op = r_bpf_op_table[BPF_ALU_SUB];
fmt = "%d";
break;
case BPF_ALU_MUL | BPF_X:
op = r_bpf_op_table[BPF_ALU_MUL];
fmt = "x";
break;
case BPF_ALU_MUL | BPF_K:
op = r_bpf_op_table[BPF_ALU_MUL];
fmt = "%d";
break;
case BPF_ALU_DIV | BPF_X:
op = r_bpf_op_table[BPF_ALU_DIV];
fmt = "x";
break;
case BPF_ALU_DIV | BPF_K:
op = r_bpf_op_table[BPF_ALU_DIV];
fmt = "%d";
break;
case BPF_ALU_MOD | BPF_X:
op = r_bpf_op_table[BPF_ALU_MOD];
fmt = "x";
break;
case BPF_ALU_MOD | BPF_K:
op = r_bpf_op_table[BPF_ALU_MOD];
fmt = "%d";
break;
case BPF_ALU_AND | BPF_X:
op = r_bpf_op_table[BPF_ALU_AND];
fmt = "x";
break;
case BPF_ALU_AND | BPF_K:
op = r_bpf_op_table[BPF_ALU_AND];
fmt = "%#x";
break;
case BPF_ALU_OR | BPF_X:
op = r_bpf_op_table[BPF_ALU_OR];
fmt = "x";
break;
case BPF_ALU_OR | BPF_K:
op = r_bpf_op_table[BPF_ALU_OR];
fmt = "%#x";
break;
case BPF_ALU_XOR | BPF_X:
op = r_bpf_op_table[BPF_ALU_XOR];
fmt = "x";
break;
case BPF_ALU_XOR | BPF_K:
op = r_bpf_op_table[BPF_ALU_XOR];
fmt = "%#x";
break;
default:
op = "invalid";
fmt = "%#x";
val = f->code;
break;
}
memset(vbuf, 0, sizeof(vbuf));
snprintf(vbuf, sizeof(vbuf), fmt, val);
vbuf[sizeof(vbuf) - 1] = 0;
if ((BPF_CLASS(f->code) == BPF_JMP && BPF_OP(f->code) != BPF_JA))
sprintf (r_op->buf_asm, "%s %s, 0x%08"PFMT64x", 0x%08"PFMT64x"", op, vbuf,
a->pc + 8 + f->jt * 8, a->pc + 8 + f->jf * 8);
else
sprintf (r_op->buf_asm, "%s %s", op, vbuf);
return r_op->size = 8;
}
void updateWorkingGraph(struct flatten_item * graph, struct bpf_program * filter, int indice,
struct filter_node ** where_to_insert, struct filter_node *** next_available_node,
struct filter_item *** next_available_item, struct filter_item * parent, struct filter_item ** terminal_nodes)
{
int i = graph[indice].bf_instruct_link;
struct filter_node * node = (*where_to_insert);
if(BPF_CLASS(filter->bf_insns[i].code) != BPF_RET)
{
/*recherche d'un noeud racine correspondant*/
for( ; node != NULL ; node = node->next)
{
if(node->item->code == filter->bf_insns[i].code && node->item->k == filter->bf_insns[i].k)
{
break;
}
}
/*pas de noeud racine correspondant, on l'ajoute*/
if(node == NULL)
{
/*on insere le nouveau noeud*/
(**next_available_item)->code = filter->bf_insns[i].code;
(**next_available_item)->k = filter->bf_insns[i].k;
(**next_available_item)->next_child_t = NULL;
(**next_available_item)->next_child_f = NULL;
(**next_available_item)->parent = parent;
(**next_available_node)->item = (**next_available_item);
(**next_available_node)->next = (*where_to_insert);
(*where_to_insert) = (**next_available_node);
node = (**next_available_node);
(*next_available_node) ++;
(*next_available_item) ++;
}
if(BPF_CLASS(filter->bf_insns[i].code) == BPF_JMP && BPF_OP(filter->bf_insns[i].code) != BPF_JA) /*noeud conditionnel*/
{
updateWorkingGraph(graph, filter, graph[indice].true_instruct, &node->item->next_child_t,next_available_node,next_available_item, node->item, terminal_nodes);
updateWorkingGraph(graph, filter, graph[indice].false_instruct, &node->item->next_child_f,next_available_node,next_available_item, node->item, terminal_nodes);
}
else /*tous les autres*/
{
updateWorkingGraph(graph, filter, graph[indice].true_instruct, &node->item->next_child_t,next_available_node,next_available_item, node->item, terminal_nodes);
}
}
else /*les noeud terminaux sont obligatoirement inséré*/
{
/*faire le lien avec le terminal node correspondant */
(**next_available_node)->item = terminal_nodes[graph[indice].false_instruct];
(**next_available_node)->next = (*where_to_insert);
(*where_to_insert) = (**next_available_node);
(*next_available_node) ++;
/*ajouter le parent au noeud terminal*/
if(terminal_nodes[graph[indice].false_instruct]->parent == NULL)
{
terminal_nodes[graph[indice].false_instruct]->parent = parent;
}
else
{
(**next_available_node)->item = parent;
(**next_available_node)->next = terminal_nodes[graph[indice].false_instruct]->next_child_t;
terminal_nodes[graph[indice].false_instruct]->next_child_t = (**next_available_node);
(*next_available_node) ++;
}
}
}
/* Assemble the body code between the prologue & epilogue */
static int bpf_jit_build_body(struct bpf_prog *fp, u32 *image,
struct codegen_context *ctx,
u32 *addrs, bool extra_pass)
{
const struct bpf_insn *insn = fp->insnsi;
int flen = fp->len;
int i, ret;
/* Start of epilogue code - will only be valid 2nd pass onwards */
u32 exit_addr = addrs[flen];
for (i = 0; i < flen; i++) {
u32 code = insn[i].code;
u32 dst_reg = b2p[insn[i].dst_reg];
u32 src_reg = b2p[insn[i].src_reg];
s16 off = insn[i].off;
s32 imm = insn[i].imm;
bool func_addr_fixed;
u64 func_addr;
u64 imm64;
u32 true_cond;
u32 tmp_idx;
/*
* addrs[] maps a BPF bytecode address into a real offset from
* the start of the body code.
*/
addrs[i] = ctx->idx * 4;
/*
* As an optimization, we note down which non-volatile registers
* are used so that we can only save/restore those in our
* prologue and epilogue. We do this here regardless of whether
* the actual BPF instruction uses src/dst registers or not
* (for instance, BPF_CALL does not use them). The expectation
* is that those instructions will have src_reg/dst_reg set to
* 0. Even otherwise, we just lose some prologue/epilogue
* optimization but everything else should work without
* any issues.
*/
if (dst_reg >= BPF_PPC_NVR_MIN && dst_reg < 32)
bpf_set_seen_register(ctx, insn[i].dst_reg);
if (src_reg >= BPF_PPC_NVR_MIN && src_reg < 32)
bpf_set_seen_register(ctx, insn[i].src_reg);
switch (code) {
/*
* Arithmetic operations: ADD/SUB/MUL/DIV/MOD/NEG
*/
case BPF_ALU | BPF_ADD | BPF_X: /* (u32) dst += (u32) src */
case BPF_ALU64 | BPF_ADD | BPF_X: /* dst += src */
PPC_ADD(dst_reg, dst_reg, src_reg);
goto bpf_alu32_trunc;
case BPF_ALU | BPF_SUB | BPF_X: /* (u32) dst -= (u32) src */
case BPF_ALU64 | BPF_SUB | BPF_X: /* dst -= src */
PPC_SUB(dst_reg, dst_reg, src_reg);
goto bpf_alu32_trunc;
case BPF_ALU | BPF_ADD | BPF_K: /* (u32) dst += (u32) imm */
case BPF_ALU | BPF_SUB | BPF_K: /* (u32) dst -= (u32) imm */
case BPF_ALU64 | BPF_ADD | BPF_K: /* dst += imm */
case BPF_ALU64 | BPF_SUB | BPF_K: /* dst -= imm */
if (BPF_OP(code) == BPF_SUB)
imm = -imm;
if (imm) {
if (imm >= -32768 && imm < 32768)
PPC_ADDI(dst_reg, dst_reg, IMM_L(imm));
else {
PPC_LI32(b2p[TMP_REG_1], imm);
PPC_ADD(dst_reg, dst_reg, b2p[TMP_REG_1]);
}
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_MUL | BPF_X: /* (u32) dst *= (u32) src */
case BPF_ALU64 | BPF_MUL | BPF_X: /* dst *= src */
if (BPF_CLASS(code) == BPF_ALU)
PPC_MULW(dst_reg, dst_reg, src_reg);
else
PPC_MULD(dst_reg, dst_reg, src_reg);
goto bpf_alu32_trunc;
case BPF_ALU | BPF_MUL | BPF_K: /* (u32) dst *= (u32) imm */
case BPF_ALU64 | BPF_MUL | BPF_K: /* dst *= imm */
if (imm >= -32768 && imm < 32768)
PPC_MULI(dst_reg, dst_reg, IMM_L(imm));
else {
PPC_LI32(b2p[TMP_REG_1], imm);
if (BPF_CLASS(code) == BPF_ALU)
PPC_MULW(dst_reg, dst_reg,
b2p[TMP_REG_1]);
else
PPC_MULD(dst_reg, dst_reg,
b2p[TMP_REG_1]);
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_DIV | BPF_X: /* (u32) dst /= (u32) src */
case BPF_ALU | BPF_MOD | BPF_X: /* (u32) dst %= (u32) src */
if (BPF_OP(code) == BPF_MOD) {
PPC_DIVWU(b2p[TMP_REG_1], dst_reg, src_reg);
PPC_MULW(b2p[TMP_REG_1], src_reg,
b2p[TMP_REG_1]);
PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
} else
PPC_DIVWU(dst_reg, dst_reg, src_reg);
goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_DIV | BPF_X: /* dst /= src */
case BPF_ALU64 | BPF_MOD | BPF_X: /* dst %= src */
if (BPF_OP(code) == BPF_MOD) {
PPC_DIVD(b2p[TMP_REG_1], dst_reg, src_reg);
PPC_MULD(b2p[TMP_REG_1], src_reg,
b2p[TMP_REG_1]);
PPC_SUB(dst_reg, dst_reg, b2p[TMP_REG_1]);
} else
PPC_DIVD(dst_reg, dst_reg, src_reg);
break;
case BPF_ALU | BPF_MOD | BPF_K: /* (u32) dst %= (u32) imm */
case BPF_ALU | BPF_DIV | BPF_K: /* (u32) dst /= (u32) imm */
case BPF_ALU64 | BPF_MOD | BPF_K: /* dst %= imm */
case BPF_ALU64 | BPF_DIV | BPF_K: /* dst /= imm */
if (imm == 0)
return -EINVAL;
else if (imm == 1)
goto bpf_alu32_trunc;
PPC_LI32(b2p[TMP_REG_1], imm);
switch (BPF_CLASS(code)) {
case BPF_ALU:
if (BPF_OP(code) == BPF_MOD) {
PPC_DIVWU(b2p[TMP_REG_2], dst_reg,
b2p[TMP_REG_1]);
PPC_MULW(b2p[TMP_REG_1],
b2p[TMP_REG_1],
b2p[TMP_REG_2]);
PPC_SUB(dst_reg, dst_reg,
b2p[TMP_REG_1]);
} else
PPC_DIVWU(dst_reg, dst_reg,
b2p[TMP_REG_1]);
break;
case BPF_ALU64:
if (BPF_OP(code) == BPF_MOD) {
PPC_DIVD(b2p[TMP_REG_2], dst_reg,
b2p[TMP_REG_1]);
PPC_MULD(b2p[TMP_REG_1],
b2p[TMP_REG_1],
b2p[TMP_REG_2]);
PPC_SUB(dst_reg, dst_reg,
b2p[TMP_REG_1]);
} else
PPC_DIVD(dst_reg, dst_reg,
b2p[TMP_REG_1]);
break;
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_NEG: /* (u32) dst = -dst */
case BPF_ALU64 | BPF_NEG: /* dst = -dst */
PPC_NEG(dst_reg, dst_reg);
goto bpf_alu32_trunc;
/*
* Logical operations: AND/OR/XOR/[A]LSH/[A]RSH
*/
case BPF_ALU | BPF_AND | BPF_X: /* (u32) dst = dst & src */
case BPF_ALU64 | BPF_AND | BPF_X: /* dst = dst & src */
PPC_AND(dst_reg, dst_reg, src_reg);
goto bpf_alu32_trunc;
case BPF_ALU | BPF_AND | BPF_K: /* (u32) dst = dst & imm */
case BPF_ALU64 | BPF_AND | BPF_K: /* dst = dst & imm */
if (!IMM_H(imm))
PPC_ANDI(dst_reg, dst_reg, IMM_L(imm));
else {
/* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm);
PPC_AND(dst_reg, dst_reg, b2p[TMP_REG_1]);
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_OR | BPF_X: /* dst = (u32) dst | (u32) src */
case BPF_ALU64 | BPF_OR | BPF_X: /* dst = dst | src */
PPC_OR(dst_reg, dst_reg, src_reg);
goto bpf_alu32_trunc;
case BPF_ALU | BPF_OR | BPF_K:/* dst = (u32) dst | (u32) imm */
case BPF_ALU64 | BPF_OR | BPF_K:/* dst = dst | imm */
if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
/* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm);
PPC_OR(dst_reg, dst_reg, b2p[TMP_REG_1]);
} else {
if (IMM_L(imm))
PPC_ORI(dst_reg, dst_reg, IMM_L(imm));
if (IMM_H(imm))
PPC_ORIS(dst_reg, dst_reg, IMM_H(imm));
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_XOR | BPF_X: /* (u32) dst ^= src */
case BPF_ALU64 | BPF_XOR | BPF_X: /* dst ^= src */
PPC_XOR(dst_reg, dst_reg, src_reg);
goto bpf_alu32_trunc;
case BPF_ALU | BPF_XOR | BPF_K: /* (u32) dst ^= (u32) imm */
case BPF_ALU64 | BPF_XOR | BPF_K: /* dst ^= imm */
if (imm < 0 && BPF_CLASS(code) == BPF_ALU64) {
/* Sign-extended */
PPC_LI32(b2p[TMP_REG_1], imm);
PPC_XOR(dst_reg, dst_reg, b2p[TMP_REG_1]);
} else {
if (IMM_L(imm))
PPC_XORI(dst_reg, dst_reg, IMM_L(imm));
if (IMM_H(imm))
PPC_XORIS(dst_reg, dst_reg, IMM_H(imm));
}
goto bpf_alu32_trunc;
case BPF_ALU | BPF_LSH | BPF_X: /* (u32) dst <<= (u32) src */
/* slw clears top 32 bits */
PPC_SLW(dst_reg, dst_reg, src_reg);
break;
case BPF_ALU64 | BPF_LSH | BPF_X: /* dst <<= src; */
PPC_SLD(dst_reg, dst_reg, src_reg);
break;
case BPF_ALU | BPF_LSH | BPF_K: /* (u32) dst <<== (u32) imm */
/* with imm 0, we still need to clear top 32 bits */
PPC_SLWI(dst_reg, dst_reg, imm);
break;
case BPF_ALU64 | BPF_LSH | BPF_K: /* dst <<== imm */
if (imm != 0)
PPC_SLDI(dst_reg, dst_reg, imm);
break;
case BPF_ALU | BPF_RSH | BPF_X: /* (u32) dst >>= (u32) src */
PPC_SRW(dst_reg, dst_reg, src_reg);
break;
case BPF_ALU64 | BPF_RSH | BPF_X: /* dst >>= src */
PPC_SRD(dst_reg, dst_reg, src_reg);
break;
case BPF_ALU | BPF_RSH | BPF_K: /* (u32) dst >>= (u32) imm */
PPC_SRWI(dst_reg, dst_reg, imm);
break;
case BPF_ALU64 | BPF_RSH | BPF_K: /* dst >>= imm */
if (imm != 0)
PPC_SRDI(dst_reg, dst_reg, imm);
break;
case BPF_ALU | BPF_ARSH | BPF_X: /* (s32) dst >>= src */
PPC_SRAW(dst_reg, dst_reg, src_reg);
goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_ARSH | BPF_X: /* (s64) dst >>= src */
PPC_SRAD(dst_reg, dst_reg, src_reg);
break;
case BPF_ALU | BPF_ARSH | BPF_K: /* (s32) dst >>= imm */
PPC_SRAWI(dst_reg, dst_reg, imm);
goto bpf_alu32_trunc;
case BPF_ALU64 | BPF_ARSH | BPF_K: /* (s64) dst >>= imm */
if (imm != 0)
PPC_SRADI(dst_reg, dst_reg, imm);
break;
/*
* MOV
*/
case BPF_ALU | BPF_MOV | BPF_X: /* (u32) dst = src */
case BPF_ALU64 | BPF_MOV | BPF_X: /* dst = src */
PPC_MR(dst_reg, src_reg);
goto bpf_alu32_trunc;
case BPF_ALU | BPF_MOV | BPF_K: /* (u32) dst = imm */
case BPF_ALU64 | BPF_MOV | BPF_K: /* dst = (s64) imm */
PPC_LI32(dst_reg, imm);
if (imm < 0)
goto bpf_alu32_trunc;
break;
bpf_alu32_trunc:
/* Truncate to 32-bits */
if (BPF_CLASS(code) == BPF_ALU)
PPC_RLWINM(dst_reg, dst_reg, 0, 0, 31);
break;
/*
* BPF_FROM_BE/LE
*/
case BPF_ALU | BPF_END | BPF_FROM_LE:
case BPF_ALU | BPF_END | BPF_FROM_BE:
#ifdef __BIG_ENDIAN__
if (BPF_SRC(code) == BPF_FROM_BE)
goto emit_clear;
#else /* !__BIG_ENDIAN__ */
if (BPF_SRC(code) == BPF_FROM_LE)
goto emit_clear;
#endif
switch (imm) {
case 16:
/* Rotate 8 bits left & mask with 0x0000ff00 */
PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 16, 23);
/* Rotate 8 bits right & insert LSB to reg */
PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 24, 31);
/* Move result back to dst_reg */
PPC_MR(dst_reg, b2p[TMP_REG_1]);
break;
case 32:
/*
* Rotate word left by 8 bits:
* 2 bytes are already in their final position
* -- byte 2 and 4 (of bytes 1, 2, 3 and 4)
*/
PPC_RLWINM(b2p[TMP_REG_1], dst_reg, 8, 0, 31);
/* Rotate 24 bits and insert byte 1 */
PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 0, 7);
/* Rotate 24 bits and insert byte 3 */
PPC_RLWIMI(b2p[TMP_REG_1], dst_reg, 24, 16, 23);
PPC_MR(dst_reg, b2p[TMP_REG_1]);
break;
case 64:
/*
* Way easier and faster(?) to store the value
* into stack and then use ldbrx
*
* ctx->seen will be reliable in pass2, but
* the instructions generated will remain the
* same across all passes
*/
PPC_STD(dst_reg, 1, bpf_jit_stack_local(ctx));
PPC_ADDI(b2p[TMP_REG_1], 1, bpf_jit_stack_local(ctx));
PPC_LDBRX(dst_reg, 0, b2p[TMP_REG_1]);
break;
}
break;
emit_clear:
switch (imm) {
case 16:
/* zero-extend 16 bits into 64 bits */
PPC_RLDICL(dst_reg, dst_reg, 0, 48);
break;
case 32:
/* zero-extend 32 bits into 64 bits */
PPC_RLDICL(dst_reg, dst_reg, 0, 32);
break;
case 64:
/* nop */
break;
}
break;
/*
* BPF_ST(X)
*/
case BPF_STX | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_B: /* *(u8 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
PPC_LI(b2p[TMP_REG_1], imm);
src_reg = b2p[TMP_REG_1];
}
PPC_STB(src_reg, dst_reg, off);
break;
case BPF_STX | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_H: /* (u16 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
PPC_LI(b2p[TMP_REG_1], imm);
src_reg = b2p[TMP_REG_1];
}
PPC_STH(src_reg, dst_reg, off);
break;
case BPF_STX | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_W: /* *(u32 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
PPC_LI32(b2p[TMP_REG_1], imm);
src_reg = b2p[TMP_REG_1];
}
PPC_STW(src_reg, dst_reg, off);
break;
case BPF_STX | BPF_MEM | BPF_DW: /* (u64 *)(dst + off) = src */
case BPF_ST | BPF_MEM | BPF_DW: /* *(u64 *)(dst + off) = imm */
if (BPF_CLASS(code) == BPF_ST) {
PPC_LI32(b2p[TMP_REG_1], imm);
src_reg = b2p[TMP_REG_1];
}
PPC_STD(src_reg, dst_reg, off);
break;
/*
* BPF_STX XADD (atomic_add)
*/
/* *(u32 *)(dst + off) += src */
case BPF_STX | BPF_XADD | BPF_W:
/* Get EA into TMP_REG_1 */
PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
tmp_idx = ctx->idx * 4;
/* load value from memory into TMP_REG_2 */
PPC_BPF_LWARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
/* add value from src_reg into this */
PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
/* store result back */
PPC_BPF_STWCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
/* we're done if this succeeded */
PPC_BCC_SHORT(COND_NE, tmp_idx);
break;
/* *(u64 *)(dst + off) += src */
case BPF_STX | BPF_XADD | BPF_DW:
PPC_ADDI(b2p[TMP_REG_1], dst_reg, off);
tmp_idx = ctx->idx * 4;
PPC_BPF_LDARX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1], 0);
PPC_ADD(b2p[TMP_REG_2], b2p[TMP_REG_2], src_reg);
PPC_BPF_STDCX(b2p[TMP_REG_2], 0, b2p[TMP_REG_1]);
PPC_BCC_SHORT(COND_NE, tmp_idx);
break;
/*
* BPF_LDX
*/
/* dst = *(u8 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_B:
PPC_LBZ(dst_reg, src_reg, off);
break;
/* dst = *(u16 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_H:
PPC_LHZ(dst_reg, src_reg, off);
break;
/* dst = *(u32 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_W:
PPC_LWZ(dst_reg, src_reg, off);
break;
/* dst = *(u64 *)(ul) (src + off) */
case BPF_LDX | BPF_MEM | BPF_DW:
PPC_LD(dst_reg, src_reg, off);
break;
/*
* Doubleword load
* 16 byte instruction that uses two 'struct bpf_insn'
*/
case BPF_LD | BPF_IMM | BPF_DW: /* dst = (u64) imm */
imm64 = ((u64)(u32) insn[i].imm) |
(((u64)(u32) insn[i+1].imm) << 32);
/* Adjust for two bpf instructions */
addrs[++i] = ctx->idx * 4;
PPC_LI64(dst_reg, imm64);
break;
/*
* Return/Exit
*/
case BPF_JMP | BPF_EXIT:
/*
* If this isn't the very last instruction, branch to
* the epilogue. If we _are_ the last instruction,
* we'll just fall through to the epilogue.
*/
if (i != flen - 1)
PPC_JMP(exit_addr);
/* else fall through to the epilogue */
break;
/*
* Call kernel helper or bpf function
*/
case BPF_JMP | BPF_CALL:
ctx->seen |= SEEN_FUNC;
ret = bpf_jit_get_func_addr(fp, &insn[i], extra_pass,
&func_addr, &func_addr_fixed);
if (ret < 0)
return ret;
if (func_addr_fixed)
bpf_jit_emit_func_call_hlp(image, ctx, func_addr);
else
bpf_jit_emit_func_call_rel(image, ctx, func_addr);
/* move return value from r3 to BPF_REG_0 */
PPC_MR(b2p[BPF_REG_0], 3);
break;
/*
* Jumps and branches
*/
case BPF_JMP | BPF_JA:
PPC_JMP(addrs[i + 1 + off]);
break;
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JGT | BPF_X:
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP | BPF_JSGT | BPF_X:
true_cond = COND_GT;
goto cond_branch;
case BPF_JMP | BPF_JLT | BPF_K:
case BPF_JMP | BPF_JLT | BPF_X:
case BPF_JMP | BPF_JSLT | BPF_K:
case BPF_JMP | BPF_JSLT | BPF_X:
true_cond = COND_LT;
goto cond_branch;
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP | BPF_JSGE | BPF_K:
case BPF_JMP | BPF_JSGE | BPF_X:
true_cond = COND_GE;
goto cond_branch;
case BPF_JMP | BPF_JLE | BPF_K:
case BPF_JMP | BPF_JLE | BPF_X:
case BPF_JMP | BPF_JSLE | BPF_K:
case BPF_JMP | BPF_JSLE | BPF_X:
true_cond = COND_LE;
goto cond_branch;
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JEQ | BPF_X:
true_cond = COND_EQ;
goto cond_branch;
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP | BPF_JNE | BPF_X:
true_cond = COND_NE;
goto cond_branch;
case BPF_JMP | BPF_JSET | BPF_K:
case BPF_JMP | BPF_JSET | BPF_X:
true_cond = COND_NE;
/* Fall through */
cond_branch:
switch (code) {
case BPF_JMP | BPF_JGT | BPF_X:
case BPF_JMP | BPF_JLT | BPF_X:
case BPF_JMP | BPF_JGE | BPF_X:
case BPF_JMP | BPF_JLE | BPF_X:
case BPF_JMP | BPF_JEQ | BPF_X:
case BPF_JMP | BPF_JNE | BPF_X:
/* unsigned comparison */
PPC_CMPLD(dst_reg, src_reg);
break;
case BPF_JMP | BPF_JSGT | BPF_X:
case BPF_JMP | BPF_JSLT | BPF_X:
case BPF_JMP | BPF_JSGE | BPF_X:
case BPF_JMP | BPF_JSLE | BPF_X:
/* signed comparison */
PPC_CMPD(dst_reg, src_reg);
break;
case BPF_JMP | BPF_JSET | BPF_X:
PPC_AND_DOT(b2p[TMP_REG_1], dst_reg, src_reg);
break;
case BPF_JMP | BPF_JNE | BPF_K:
case BPF_JMP | BPF_JEQ | BPF_K:
case BPF_JMP | BPF_JGT | BPF_K:
case BPF_JMP | BPF_JLT | BPF_K:
case BPF_JMP | BPF_JGE | BPF_K:
case BPF_JMP | BPF_JLE | BPF_K:
/*
* Need sign-extended load, so only positive
* values can be used as imm in cmpldi
*/
if (imm >= 0 && imm < 32768)
PPC_CMPLDI(dst_reg, imm);
else {
/* sign-extending load */
PPC_LI32(b2p[TMP_REG_1], imm);
/* ... but unsigned comparison */
PPC_CMPLD(dst_reg, b2p[TMP_REG_1]);
}
break;
case BPF_JMP | BPF_JSGT | BPF_K:
case BPF_JMP | BPF_JSLT | BPF_K:
case BPF_JMP | BPF_JSGE | BPF_K:
case BPF_JMP | BPF_JSLE | BPF_K:
/*
* signed comparison, so any 16-bit value
* can be used in cmpdi
*/
if (imm >= -32768 && imm < 32768)
PPC_CMPDI(dst_reg, imm);
else {
PPC_LI32(b2p[TMP_REG_1], imm);
PPC_CMPD(dst_reg, b2p[TMP_REG_1]);
}
break;
case BPF_JMP | BPF_JSET | BPF_K:
/* andi does not sign-extend the immediate */
if (imm >= 0 && imm < 32768)
/* PPC_ANDI is _only/always_ dot-form */
PPC_ANDI(b2p[TMP_REG_1], dst_reg, imm);
else {
PPC_LI32(b2p[TMP_REG_1], imm);
PPC_AND_DOT(b2p[TMP_REG_1], dst_reg,
b2p[TMP_REG_1]);
}
break;
}
PPC_BCC(true_cond, addrs[i + 1 + off]);
break;
/*
* Tail call
*/
case BPF_JMP | BPF_TAIL_CALL:
ctx->seen |= SEEN_TAILCALL;
bpf_jit_emit_tail_call(image, ctx, addrs[i + 1]);
break;
default:
/*
* The filter contains something cruel & unusual.
* We don't handle it, but also there shouldn't be
* anything missing from our list.
*/
pr_err_ratelimited("eBPF filter opcode %04x (@%d) unsupported\n",
code, i);
return -ENOTSUPP;
}
}
/* Set end-of-body-code address for exit. */
addrs[i] = ctx->idx * 4;
return 0;
}
char * filter_item_image(const struct filter_item * p)
{
int v;
const char *fmt, *op;
static char image[256];
char operand[64];
v = p->k;
switch (p->code) {
default:
op = "unimp";
fmt = "0x%x";
v = p->code;
break;
case BPF_RET|BPF_K:
op = "ret";
fmt = "#%d";
break;
case BPF_RET|BPF_A:
op = "ret";
fmt = "";
break;
case BPF_LD|BPF_W|BPF_ABS:
op = "ld";
fmt = "[%d]";
break;
case BPF_LD|BPF_H|BPF_ABS:
op = "ldh";
fmt = "[%d]";
break;
case BPF_LD|BPF_B|BPF_ABS:
op = "ldb";
fmt = "[%d]";
break;
case BPF_LD|BPF_W|BPF_LEN:
op = "ld";
fmt = "#pktlen";
break;
case BPF_LD|BPF_W|BPF_IND:
op = "ld";
fmt = "[x + %d]";
break;
case BPF_LD|BPF_H|BPF_IND:
op = "ldh";
fmt = "[x + %d]";
break;
case BPF_LD|BPF_B|BPF_IND:
op = "ldb";
fmt = "[x + %d]";
break;
case BPF_LD|BPF_IMM:
op = "ld";
fmt = "#0x%x";
break;
case BPF_LDX|BPF_IMM:
op = "ldx";
fmt = "#0x%x";
break;
case BPF_LDX|BPF_MSH|BPF_B:
op = "ldxb";
fmt = "4*([%d]&0xf)";
break;
case BPF_LD|BPF_MEM:
op = "ld";
fmt = "M[%d]";
break;
case BPF_LDX|BPF_MEM:
op = "ldx";
fmt = "M[%d]";
break;
case BPF_ST:
op = "st";
fmt = "M[%d]";
break;
case BPF_STX:
op = "stx";
fmt = "M[%d]";
break;
case BPF_JMP|BPF_JA:
op = "ja";
fmt = "long";
fmt = "%d";
/*v = n + 1 + p->k;*/
break;
case BPF_JMP|BPF_JGT|BPF_K:
op = "jgt";
fmt = "#0x%x";
break;
case BPF_JMP|BPF_JGE|BPF_K:
op = "jge";
fmt = "#0x%x";
break;
case BPF_JMP|BPF_JEQ|BPF_K:
op = "jeq";
fmt = "#0x%x";
break;
case BPF_JMP|BPF_JSET|BPF_K:
op = "jset";
fmt = "#0x%x";
break;
case BPF_JMP|BPF_JGT|BPF_X:
op = "jgt";
fmt = "x";
break;
case BPF_JMP|BPF_JGE|BPF_X:
op = "jge";
fmt = "x";
break;
case BPF_JMP|BPF_JEQ|BPF_X:
op = "jeq";
fmt = "x";
break;
case BPF_JMP|BPF_JSET|BPF_X:
op = "jset";
fmt = "x";
break;
case BPF_ALU|BPF_ADD|BPF_X:
op = "add";
fmt = "x";
break;
case BPF_ALU|BPF_SUB|BPF_X:
op = "sub";
fmt = "x";
break;
case BPF_ALU|BPF_MUL|BPF_X:
op = "mul";
fmt = "x";
break;
case BPF_ALU|BPF_DIV|BPF_X:
op = "div";
fmt = "x";
break;
case BPF_ALU|BPF_AND|BPF_X:
op = "and";
fmt = "x";
break;
case BPF_ALU|BPF_OR|BPF_X:
op = "or";
fmt = "x";
break;
case BPF_ALU|BPF_LSH|BPF_X:
op = "lsh";
fmt = "x";
break;
case BPF_ALU|BPF_RSH|BPF_X:
op = "rsh";
fmt = "x";
break;
case BPF_ALU|BPF_ADD|BPF_K:
op = "add";
fmt = "#%d";
break;
case BPF_ALU|BPF_SUB|BPF_K:
op = "sub";
fmt = "#%d";
break;
case BPF_ALU|BPF_MUL|BPF_K:
op = "mul";
fmt = "#%d";
break;
case BPF_ALU|BPF_DIV|BPF_K:
op = "div";
fmt = "#%d";
break;
case BPF_ALU|BPF_AND|BPF_K:
op = "and";
fmt = "#0x%x";
break;
case BPF_ALU|BPF_OR|BPF_K:
op = "or";
fmt = "#0x%x";
break;
case BPF_ALU|BPF_LSH|BPF_K:
op = "lsh";
fmt = "#%d";
break;
case BPF_ALU|BPF_RSH|BPF_K:
op = "rsh";
fmt = "#%d";
break;
case BPF_ALU|BPF_NEG:
op = "neg";
fmt = "";
break;
case BPF_MISC|BPF_TAX:
op = "tax";
fmt = "";
break;
case BPF_MISC|BPF_TXA:
op = "txa";
fmt = "";
break;
}
(void)snprintf(operand, sizeof operand, fmt, v);
(void)snprintf(image, sizeof image,
(BPF_CLASS(p->code) == BPF_JMP && BPF_OP(p->code) != BPF_JA) ?
/*"(%03d) %-8s %-16s jt %d\tjf %d" : "(%03d) %-8s %s",
n, op, operand, n + 1 + p->jt, n + 1 + p->jf);*/
"(%03d) %-8s %-16s jt\tjf" : "(%03d) %-8s %s",
0, op, operand/*, n + 1 + p->jt, n + 1 + p->jf*/);
return image;
}
int bpf_validate(struct bpf_insn *f, int len)
{
int i;
int from;
struct bpf_insn *p;
if (len < 1 || len > BPF_MAXINSNS)
return 0;
i = 0;
p = &f[i];
switch (BPF_CLASS(p->code)) {
/*
* Check that memory operations use valid addresses.
*/
case BPF_LD:
case BPF_LDX:
switch (BPF_MODE(p->code)) {
case BPF_IMM:
break;
case BPF_ABS:
case BPF_IND:
case BPF_MSH:
/*
* More strict check with actual packet length
* is done runtime.
*/
if (p->k >= bpf_maxbufsize)
return 0;
break;
case BPF_MEM:
if (p->k >= BPF_MEMWORDS)
return 0;
break;
case BPF_LEN:
break;
default:
return 0;
}
break;
case BPF_ST:
case BPF_STX:
if (p->k >= BPF_MEMWORDS)
return 0;
break;
case BPF_ALU:
switch (BPF_OP(p->code)) {
case BPF_ADD:
case BPF_SUB:
case BPF_OR:
case BPF_AND:
case BPF_LSH:
case BPF_RSH:
case BPF_NEG:
break;
case BPF_DIV:
/*
* Check for constant division by 0.
*/
if (BPF_RVAL(p->code) == BPF_K && p->k == 0)
return 0;
break;
default:
return 0;
}
break;
case BPF_JMP:
/*
* Check that jumps are forward, and within
* the code block.
*/
from = i + 1;
switch (BPF_OP(p->code)) {
case BPF_JA:
if (from + p->k < from || from + p->k >= len)
return 0;
break;
case BPF_JEQ:
case BPF_JGT:
case BPF_JGE:
case BPF_JSET:
if (from + p->jt >= len || from + p->jf >= len)
return 0;
break;
default:
return 0;
}
break;
case BPF_RET:
break;
case BPF_MISC:
break;
default:
return 0;
}
klee_merge();
i = 1;
p = &f[i];
switch (BPF_CLASS(p->code)) {
/*
* Check that memory operations use valid addresses.
*/
case BPF_LD:
case BPF_LDX:
switch (BPF_MODE(p->code)) {
case BPF_IMM:
break;
case BPF_ABS:
case BPF_IND:
case BPF_MSH:
/*
* More strict check with actual packet length
* is done runtime.
*/
if (p->k >= bpf_maxbufsize)
return 0;
break;
case BPF_MEM:
if (p->k >= BPF_MEMWORDS)
return 0;
break;
case BPF_LEN:
break;
default:
return 0;
}
break;
case BPF_ST:
case BPF_STX:
if (p->k >= BPF_MEMWORDS)
return 0;
break;
case BPF_ALU:
switch (BPF_OP(p->code)) {
case BPF_ADD:
case BPF_SUB:
case BPF_OR:
case BPF_AND:
case BPF_LSH:
case BPF_RSH:
case BPF_NEG:
break;
case BPF_DIV:
/*
* Check for constant division by 0.
*/
if (BPF_RVAL(p->code) == BPF_K && p->k == 0)
return 0;
break;
default:
return 0;
}
break;
case BPF_JMP:
/*
* Check that jumps are forward, and within
* the code block.
*/
from = i + 1;
switch (BPF_OP(p->code)) {
case BPF_JA:
if (from + p->k < from || from + p->k >= len)
return 0;
break;
case BPF_JEQ:
case BPF_JGT:
case BPF_JGE:
case BPF_JSET:
if (from + p->jt >= len || from + p->jf >= len)
return 0;
break;
default:
return 0;
}
break;
case BPF_RET:
break;
case BPF_MISC:
break;
default:
return 0;
}
klee_merge();
i = 2;
p = &f[i];
switch (BPF_CLASS(p->code)) {
/*
* Check that memory operations use valid addresses.
*/
case BPF_LD:
case BPF_LDX:
switch (BPF_MODE(p->code)) {
case BPF_IMM:
break;
case BPF_ABS:
case BPF_IND:
case BPF_MSH:
/*
* More strict check with actual packet length
* is done runtime.
*/
if (p->k >= bpf_maxbufsize)
return 0;
break;
case BPF_MEM:
if (p->k >= BPF_MEMWORDS)
return 0;
break;
case BPF_LEN:
break;
default:
return 0;
}
break;
case BPF_ST:
case BPF_STX:
if (p->k >= BPF_MEMWORDS)
return 0;
break;
case BPF_ALU:
switch (BPF_OP(p->code)) {
case BPF_ADD:
case BPF_SUB:
case BPF_OR:
case BPF_AND:
case BPF_LSH:
case BPF_RSH:
case BPF_NEG:
break;
case BPF_DIV:
/*
* Check for constant division by 0.
*/
if (BPF_RVAL(p->code) == BPF_K && p->k == 0)
return 0;
break;
default:
return 0;
}
break;
case BPF_JMP:
/*
* Check that jumps are forward, and within
* the code block.
*/
from = i + 1;
switch (BPF_OP(p->code)) {
case BPF_JA:
if (from + p->k < from || from + p->k >= len)
return 0;
break;
case BPF_JEQ:
case BPF_JGT:
case BPF_JGE:
case BPF_JSET:
if (from + p->jt >= len || from + p->jf >= len)
return 0;
break;
default:
return 0;
}
break;
case BPF_RET:
break;
case BPF_MISC:
break;
default:
return 0;
}
klee_merge();
return BPF_CLASS(f[len - 1].code) == BPF_RET;
}
