Lines Matching +full:pass +full:- +full:1
1 // SPDX-License-Identifier: GPL-2.0-only
12 #include <asm/asm-compat.h>
21 #include <asm/code-patching.h>
32 if (!exit_addr || is_offset_in_branch_range(exit_addr - (ctx->idx * 4))) { in bpf_jit_emit_exit_insn()
34 } else if (ctx->alt_exit_addr) { in bpf_jit_emit_exit_insn()
35 if (WARN_ON(!is_offset_in_branch_range((long)ctx->alt_exit_addr - (ctx->idx * 4)))) in bpf_jit_emit_exit_insn()
36 return -1; in bpf_jit_emit_exit_insn()
37 PPC_JMP(ctx->alt_exit_addr); in bpf_jit_emit_exit_insn()
39 ctx->alt_exit_addr = ctx->idx * 4; in bpf_jit_emit_exit_insn()
71 int pass; in bpf_int_jit_compile() local
84 if (!fp->jit_requested) in bpf_int_jit_compile()
96 jit_data = fp->aux->jit_data; in bpf_int_jit_compile()
103 fp->aux->jit_data = jit_data; in bpf_int_jit_compile()
106 flen = fp->len; in bpf_int_jit_compile()
107 addrs = jit_data->addrs; in bpf_int_jit_compile()
109 cgctx = jit_data->ctx; in bpf_int_jit_compile()
115 fimage = jit_data->fimage; in bpf_int_jit_compile()
116 fhdr = jit_data->fhdr; in bpf_int_jit_compile()
117 proglen = jit_data->proglen; in bpf_int_jit_compile()
118 hdr = jit_data->hdr; in bpf_int_jit_compile()
119 image = (void *)hdr + ((void *)fimage - (void *)fhdr); in bpf_int_jit_compile()
121 /* During extra pass, ensure index is reset before repopulating extable entries */ in bpf_int_jit_compile()
126 addrs = kcalloc(flen + 1, sizeof(*addrs), GFP_KERNEL); in bpf_int_jit_compile()
136 cgctx.stack_size = round_up(fp->aux->stack_depth, 16); in bpf_int_jit_compile()
138 /* Scouting faux-generate pass 0 */ in bpf_int_jit_compile()
146 * If we have seen a tail call, we need a second pass. in bpf_int_jit_compile()
148 * from bpf_jit_emit_tail_call() with a not yet stable ctx->seen. in bpf_int_jit_compile()
149 * We also need a second pass if we ended up with too large in bpf_int_jit_compile()
167 addrs[fp->len] = cgctx.idx * 4; in bpf_int_jit_compile()
170 fixup_len = fp->aux->num_exentries * BPF_FIXUP_LEN * 4; in bpf_int_jit_compile()
171 extable_len = fp->aux->num_exentries * sizeof(struct exception_table_entry); in bpf_int_jit_compile()
184 fp->aux->extable = (void *)fimage + FUNCTION_DESCR_SIZE + proglen + fixup_len; in bpf_int_jit_compile()
190 /* Code generation passes 1-2 */ in bpf_int_jit_compile()
191 for (pass = 1; pass < 3; pass++) { in bpf_int_jit_compile()
196 if (bpf_jit_build_body(fp, code_base, fcode_base, &cgctx, addrs, pass, in bpf_int_jit_compile()
198 bpf_arch_text_copy(&fhdr->size, &hdr->size, sizeof(hdr->size)); in bpf_int_jit_compile()
205 if (bpf_jit_enable > 1) in bpf_int_jit_compile()
206 pr_info("Pass %d: shrink = %d, seen = 0x%x\n", pass, in bpf_int_jit_compile()
207 proglen - (cgctx.idx * 4), cgctx.seen); in bpf_int_jit_compile()
210 if (bpf_jit_enable > 1) in bpf_int_jit_compile()
215 bpf_jit_dump(flen, proglen, pass, code_base); in bpf_int_jit_compile()
220 ((u64 *)image)[1] = local_paca->kernel_toc; in bpf_int_jit_compile()
223 fp->bpf_func = (void *)fimage; in bpf_int_jit_compile()
224 fp->jited = 1; in bpf_int_jit_compile()
225 fp->jited_len = proglen + FUNCTION_DESCR_SIZE; in bpf_int_jit_compile()
227 if (!fp->is_func || extra_pass) { in bpf_int_jit_compile()
236 fp->aux->jit_data = NULL; in bpf_int_jit_compile()
238 jit_data->addrs = addrs; in bpf_int_jit_compile()
239 jit_data->ctx = cgctx; in bpf_int_jit_compile()
240 jit_data->proglen = proglen; in bpf_int_jit_compile()
241 jit_data->fimage = fimage; in bpf_int_jit_compile()
242 jit_data->fhdr = fhdr; in bpf_int_jit_compile()
243 jit_data->hdr = hdr; in bpf_int_jit_compile()
257 int bpf_add_extable_entry(struct bpf_prog *fp, u32 *image, u32 *fimage, int pass, in bpf_add_extable_entry() argument
266 /* Populate extable entries only in the last pass */ in bpf_add_extable_entry()
267 if (pass != 2) in bpf_add_extable_entry()
270 if (!fp->aux->extable || in bpf_add_extable_entry()
271 WARN_ON_ONCE(ctx->exentry_idx >= fp->aux->num_exentries)) in bpf_add_extable_entry()
272 return -EINVAL; in bpf_add_extable_entry()
281 ex = (void *)fp->aux->extable - (void *)fimage + (void *)image; in bpf_add_extable_entry()
283 fixup = (void *)ex - in bpf_add_extable_entry()
284 (fp->aux->num_exentries * BPF_FIXUP_LEN * 4) + in bpf_add_extable_entry()
285 (ctx->exentry_idx * BPF_FIXUP_LEN * 4); in bpf_add_extable_entry()
289 fixup[1] = PPC_RAW_LI(dst_reg - 1, 0); /* clear higher 32-bit register too */ in bpf_add_extable_entry()
291 fixup[BPF_FIXUP_LEN - 1] = in bpf_add_extable_entry()
292 PPC_RAW_BRANCH((long)(pc + jmp_off) - (long)&fixup[BPF_FIXUP_LEN - 1]); in bpf_add_extable_entry()
294 ex_entry = &ex[ctx->exentry_idx]; in bpf_add_extable_entry()
296 offset = pc - (long)&ex_entry->insn; in bpf_add_extable_entry()
298 return -ERANGE; in bpf_add_extable_entry()
299 ex_entry->insn = offset; in bpf_add_extable_entry()
301 offset = (long)fixup - (long)&ex_entry->fixup; in bpf_add_extable_entry()
303 return -ERANGE; in bpf_add_extable_entry()
304 ex_entry->fixup = offset; in bpf_add_extable_entry()
306 ctx->exentry_idx++; in bpf_add_extable_entry()
315 return ERR_PTR(-EINVAL); in bpf_arch_text_copy()
330 return -EINVAL; in bpf_arch_text_invalidate()
341 if (fp->jited) { in bpf_jit_free()
342 struct powerpc_jit_data *jit_data = fp->aux->jit_data; in bpf_jit_free()
346 * If we fail the final pass of JIT (from jit_subprogs), in bpf_jit_free()
351 bpf_jit_binary_pack_finalize(fp, jit_data->fhdr, jit_data->hdr); in bpf_jit_free()
352 kvfree(jit_data->addrs); in bpf_jit_free()