void bpf2_jit_compile(struct bpf_program *prog) { struct bpf_binary_header *header = NULL; int proglen, oldproglen = 0; int *addrs; u8 *image = NULL; int pass; int i; if (!prog || !prog->cb || !prog->cb->jit_select_func) return; addrs = kmalloc(prog->insn_cnt * sizeof(*addrs), GFP_KERNEL); if (!addrs) return; for (proglen = 0, i = 0; i < prog->insn_cnt; i++) { proglen += 64; addrs[i] = proglen; } for (pass = 0; pass < 10; pass++) { proglen = do_jit(prog, addrs, image, oldproglen); if (proglen <= 0) { image = NULL; goto out; } if (image) { if (proglen != oldproglen) pr_err("bpf_jit: proglen=%d != oldproglen=%d\n", proglen, oldproglen); break; } if (proglen == oldproglen) { header = bpf_alloc_binary(proglen, &image); if (!header) goto out; } oldproglen = proglen; } if (image) { bpf_flush_icache(header, image + proglen); set_memory_ro((unsigned long)header, header->pages); } out: kfree(addrs); prog->jit_image = (void (*)(struct bpf_context *ctx))image; return; }
void bpf_int_jit_compile(struct bpf_prog *prog) { struct bpf_binary_header *header = NULL; int proglen, oldproglen = 0; struct jit_context ctx = {}; u8 *image = NULL; int *addrs; int pass; int i; if (!bpf_jit_enable) return; if (!prog || !prog->len) return; addrs = kmalloc(prog->len * sizeof(*addrs), GFP_KERNEL); if (!addrs) return; /* Before first pass, make a rough estimation of addrs[] * each bpf instruction is translated to less than 64 bytes */ for (proglen = 0, i = 0; i < prog->len; i++) { proglen += 64; addrs[i] = proglen; } ctx.cleanup_addr = proglen; /* JITed image shrinks with every pass and the loop iterates * until the image stops shrinking. Very large bpf programs * may converge on the last pass. In such case do one more * pass to emit the final image */ for (pass = 0; pass < 10 || image; pass++) { proglen = do_jit(prog, addrs, image, oldproglen, &ctx); if (proglen <= 0) { image = NULL; if (header) bpf_jit_binary_free(header); goto out; } if (image) { if (proglen != oldproglen) { pr_err("bpf_jit: proglen=%d != oldproglen=%d\n", proglen, oldproglen); goto out; } break; } if (proglen == oldproglen) { header = bpf_jit_binary_alloc(proglen, &image, 1, jit_fill_hole); if (!header) goto out; } oldproglen = proglen; } if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, proglen, pass + 1, image); if (image) { bpf_flush_icache(header, image + proglen); set_memory_ro((unsigned long)header, header->pages); prog->bpf_func = (void *)image; prog->jited = 1; } out: kfree(addrs); }
struct bpf_prog *bpf_int_jit_compile(struct bpf_prog *prog) { struct bpf_binary_header *header = NULL; struct bpf_prog *tmp, *orig_prog = prog; struct x64_jit_data *jit_data; int proglen, oldproglen = 0; struct jit_context ctx = {}; bool tmp_blinded = false; bool extra_pass = false; u8 *image = NULL; int *addrs; int pass; int i; if (!prog->jit_requested) return orig_prog; tmp = bpf_jit_blind_constants(prog); /* * If blinding was requested and we failed during blinding, * we must fall back to the interpreter. */ if (IS_ERR(tmp)) return orig_prog; if (tmp != prog) { tmp_blinded = true; prog = tmp; } jit_data = prog->aux->jit_data; if (!jit_data) { jit_data = kzalloc(sizeof(*jit_data), GFP_KERNEL); if (!jit_data) { prog = orig_prog; goto out; } prog->aux->jit_data = jit_data; } addrs = jit_data->addrs; if (addrs) { ctx = jit_data->ctx; oldproglen = jit_data->proglen; image = jit_data->image; header = jit_data->header; extra_pass = true; goto skip_init_addrs; } addrs = kmalloc(prog->len * sizeof(*addrs), GFP_KERNEL); if (!addrs) { prog = orig_prog; goto out_addrs; } /* * Before first pass, make a rough estimation of addrs[] * each BPF instruction is translated to less than 64 bytes */ for (proglen = 0, i = 0; i < prog->len; i++) { proglen += 64; addrs[i] = proglen; } ctx.cleanup_addr = proglen; skip_init_addrs: /* * JITed image shrinks with every pass and the loop iterates * until the image stops shrinking. Very large BPF programs * may converge on the last pass. In such case do one more * pass to emit the final image. */ for (pass = 0; pass < 20 || image; pass++) { proglen = do_jit(prog, addrs, image, oldproglen, &ctx); if (proglen <= 0) { out_image: image = NULL; if (header) bpf_jit_binary_free(header); prog = orig_prog; goto out_addrs; } if (image) { if (proglen != oldproglen) { pr_err("bpf_jit: proglen=%d != oldproglen=%d\n", proglen, oldproglen); goto out_image; } break; } if (proglen == oldproglen) { header = bpf_jit_binary_alloc(proglen, &image, 1, jit_fill_hole); if (!header) { prog = orig_prog; goto out_addrs; } } oldproglen = proglen; cond_resched(); } if (bpf_jit_enable > 1) bpf_jit_dump(prog->len, proglen, pass + 1, image); if (image) { if (!prog->is_func || extra_pass) { bpf_jit_binary_lock_ro(header); } else { jit_data->addrs = addrs; jit_data->ctx = ctx; jit_data->proglen = proglen; jit_data->image = image; jit_data->header = header; } prog->bpf_func = (void *)image; prog->jited = 1; prog->jited_len = proglen; } else { prog = orig_prog; } if (!image || !prog->is_func || extra_pass) { out_addrs: kfree(addrs); kfree(jit_data); prog->aux->jit_data = NULL; } out: if (tmp_blinded) bpf_jit_prog_release_other(prog, prog == orig_prog ? tmp : orig_prog); return prog; }