static void bpf_jit_build_prologue(struct bpf_prog *fp, u32 *image, struct codegen_context *ctx) { int i; const struct sock_filter *filter = fp->insns; if (ctx->seen & (SEEN_MEM | SEEN_DATAREF)) { /* Make stackframe */ if (ctx->seen & SEEN_DATAREF) { /* If we call any helpers (for loads), save LR */ EMIT(PPC_INST_MFLR | __PPC_RT(R0)); PPC_BPF_STL(0, 1, PPC_LR_STKOFF); /* Back up non-volatile regs. */ PPC_BPF_STL(r_D, 1, -(REG_SZ*(32-r_D))); PPC_BPF_STL(r_HL, 1, -(REG_SZ*(32-r_HL))); } if (ctx->seen & SEEN_MEM) { /* * Conditionally save regs r15-r31 as some will be used * for M[] data. */ for (i = r_M; i < (r_M+16); i++) { if (ctx->seen & (1 << (i-r_M))) PPC_BPF_STL(i, 1, -(REG_SZ*(32-i))); } } PPC_BPF_STLU(1, 1, -BPF_PPC_STACKFRAME); } if (ctx->seen & SEEN_DATAREF) { /* * If this filter needs to access skb data, * prepare r_D and r_HL: * r_HL = skb->len - skb->data_len * r_D = skb->data */ PPC_LWZ_OFFS(r_scratch1, r_skb, offsetof(struct sk_buff, data_len)); PPC_LWZ_OFFS(r_HL, r_skb, offsetof(struct sk_buff, len)); PPC_SUB(r_HL, r_HL, r_scratch1); PPC_LL_OFFS(r_D, r_skb, offsetof(struct sk_buff, data)); } if (ctx->seen & SEEN_XREG) { /* * TODO: Could also detect whether first instr. sets X and * avoid this (as below, with A). */ PPC_LI(r_X, 0); } /* make sure we dont leak kernel information to user */ if (bpf_needs_clear_a(&filter[0])) PPC_LI(r_A, 0); }
static void bpf_jit_build_prologue(u32 *image, struct codegen_context *ctx) { int i; /* * Initialize tail_call_cnt if we do tail calls. * Otherwise, put in NOPs so that it can be skipped when we are * invoked through a tail call. */ if (ctx->seen & SEEN_TAILCALL) { PPC_LI(b2p[TMP_REG_1], 0); /* this goes in the redzone */ PPC_BPF_STL(b2p[TMP_REG_1], 1, -(BPF_PPC_STACK_SAVE + 8)); } else { PPC_NOP(); PPC_NOP(); } #define BPF_TAILCALL_PROLOGUE_SIZE 8 if (bpf_has_stack_frame(ctx)) { /* * We need a stack frame, but we don't necessarily need to * save/restore LR unless we call other functions */ if (ctx->seen & SEEN_FUNC) { EMIT(PPC_INST_MFLR | __PPC_RT(R0)); PPC_BPF_STL(0, 1, PPC_LR_STKOFF); } PPC_BPF_STLU(1, 1, -(BPF_PPC_STACKFRAME + ctx->stack_size)); } /* * Back up non-volatile regs -- BPF registers 6-10 * If we haven't created our own stack frame, we save these * in the protected zone below the previous stack frame */ for (i = BPF_REG_6; i <= BPF_REG_10; i++) if (bpf_is_seen_register(ctx, i)) PPC_BPF_STL(b2p[i], 1, bpf_jit_stack_offsetof(ctx, b2p[i])); /* Setup frame pointer to point to the bpf stack area */ if (bpf_is_seen_register(ctx, BPF_REG_FP)) PPC_ADDI(b2p[BPF_REG_FP], 1, STACK_FRAME_MIN_SIZE + ctx->stack_size); }
static void bpf_jit_emit_tail_call(u32 *image, struct codegen_context *ctx, u32 out) { /* * By now, the eBPF program has already setup parameters in r3, r4 and r5 * r3/BPF_REG_1 - pointer to ctx -- passed as is to the next bpf program * r4/BPF_REG_2 - pointer to bpf_array * r5/BPF_REG_3 - index in bpf_array */ int b2p_bpf_array = b2p[BPF_REG_2]; int b2p_index = b2p[BPF_REG_3]; /* * if (index >= array->map.max_entries) * goto out; */ PPC_LWZ(b2p[TMP_REG_1], b2p_bpf_array, offsetof(struct bpf_array, map.max_entries)); PPC_RLWINM(b2p_index, b2p_index, 0, 0, 31); PPC_CMPLW(b2p_index, b2p[TMP_REG_1]); PPC_BCC(COND_GE, out); /* * if (tail_call_cnt > MAX_TAIL_CALL_CNT) * goto out; */ PPC_LD(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); PPC_CMPLWI(b2p[TMP_REG_1], MAX_TAIL_CALL_CNT); PPC_BCC(COND_GT, out); /* * tail_call_cnt++; */ PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], 1); PPC_BPF_STL(b2p[TMP_REG_1], 1, bpf_jit_stack_tailcallcnt(ctx)); /* prog = array->ptrs[index]; */ PPC_MULI(b2p[TMP_REG_1], b2p_index, 8); PPC_ADD(b2p[TMP_REG_1], b2p[TMP_REG_1], b2p_bpf_array); PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_array, ptrs)); /* * if (prog == NULL) * goto out; */ PPC_CMPLDI(b2p[TMP_REG_1], 0); PPC_BCC(COND_EQ, out); /* goto *(prog->bpf_func + prologue_size); */ PPC_LD(b2p[TMP_REG_1], b2p[TMP_REG_1], offsetof(struct bpf_prog, bpf_func)); #ifdef PPC64_ELF_ABI_v1 /* skip past the function descriptor */ PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], FUNCTION_DESCR_SIZE + BPF_TAILCALL_PROLOGUE_SIZE); #else PPC_ADDI(b2p[TMP_REG_1], b2p[TMP_REG_1], BPF_TAILCALL_PROLOGUE_SIZE); #endif PPC_MTCTR(b2p[TMP_REG_1]); /* tear down stack, restore NVRs, ... */ bpf_jit_emit_common_epilogue(image, ctx); PPC_BCTR(); /* out: */ }