Lines Matching +full:function +full:- +full:off
1 // SPDX-License-Identifier: GPL-2.0
73 __set_bit(reg, &ctx->flags); in bpf_to_rv_reg()
88 return test_bit(reg, &ctx->flags); in seen_reg()
95 __set_bit(RV_CTX_F_SEEN_S5, &ctx->flags); in mark_fp()
100 __set_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in mark_call()
105 return test_bit(RV_CTX_F_SEEN_CALL, &ctx->flags); in seen_call()
110 __set_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in mark_tail_call()
115 return test_bit(RV_CTX_F_SEEN_TAIL_CALL, &ctx->flags); in seen_tail_call()
123 __set_bit(RV_CTX_F_SEEN_S6, &ctx->flags); in rv_tail_call_reg()
131 return -(1L << 31) <= val && val < (1L << 31); in is_32b_int()
137 * auipc+jalr can reach any signed PC-relative offset in the range in in_auipc_jalr_range()
138 * [-2^31 - 2^11, 2^31 - 2^11). in in_auipc_jalr_range()
140 return (-(1L << 31) - (1L << 11)) <= val && in in_auipc_jalr_range()
141 val < ((1L << 31) - (1L << 11)); in in_auipc_jalr_range()
144 /* Emit fixed-length instructions for address */
151 u64 ip = (u64)(ctx->ro_insns + ctx->ninsns); in emit_addr()
152 s64 off = addr - ip; in emit_addr() local
153 s64 upper = (off + (1 << 11)) >> 12; in emit_addr()
154 s64 lower = off & 0xfff; in emit_addr()
156 if (extra_pass && !in_auipc_jalr_range(off)) { in emit_addr()
157 pr_err("bpf-jit: target offset 0x%llx is out of range\n", off); in emit_addr()
158 return -ERANGE; in emit_addr()
166 /* Emit variable-length instructions for 32-bit and 64-bit imm */
169 /* Note that the immediate from the add is sign-extended, in emit_imm()
173 * shift. The "Loading a 32-Bit constant" example from the in emit_imm()
174 * "Computer Organization and Design, RISC-V edition" book by in emit_imm()
180 /* Sign-extend lower 12 bits to 64 bits since immediates for li, addiw, in emit_imm()
212 int stack_adjust = ctx->stack_size, store_offset = stack_adjust - 8; in __build_epilogue()
216 store_offset -= 8; in __build_epilogue()
219 store_offset -= 8; in __build_epilogue()
222 store_offset -= 8; in __build_epilogue()
226 store_offset -= 8; in __build_epilogue()
230 store_offset -= 8; in __build_epilogue()
234 store_offset -= 8; in __build_epilogue()
238 store_offset -= 8; in __build_epilogue()
242 store_offset -= 8; in __build_epilogue()
301 rvoff -= 4; in emit_branch()
337 int tc_ninsn, off, start_insn = ctx->ninsns; in emit_bpf_tail_call() local
344 * if (index >= array->map.max_entries) in emit_bpf_tail_call()
347 tc_ninsn = insn ? ctx->offset[insn] - ctx->offset[insn - 1] : in emit_bpf_tail_call()
348 ctx->offset[0]; in emit_bpf_tail_call()
351 off = offsetof(struct bpf_array, map.max_entries); in emit_bpf_tail_call()
352 if (is_12b_check(off, insn)) in emit_bpf_tail_call()
353 return -1; in emit_bpf_tail_call()
354 emit(rv_lwu(RV_REG_T1, off, RV_REG_A1), ctx); in emit_bpf_tail_call()
355 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
356 emit_branch(BPF_JGE, RV_REG_A2, RV_REG_T1, off, ctx); in emit_bpf_tail_call()
358 /* if (--TCC < 0) in emit_bpf_tail_call()
361 emit_addi(RV_REG_TCC, tcc, -1, ctx); in emit_bpf_tail_call()
362 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
363 emit_branch(BPF_JSLT, RV_REG_TCC, RV_REG_ZERO, off, ctx); in emit_bpf_tail_call()
365 /* prog = array->ptrs[index]; in emit_bpf_tail_call()
371 off = offsetof(struct bpf_array, ptrs); in emit_bpf_tail_call()
372 if (is_12b_check(off, insn)) in emit_bpf_tail_call()
373 return -1; in emit_bpf_tail_call()
374 emit_ld(RV_REG_T2, off, RV_REG_T2, ctx); in emit_bpf_tail_call()
375 off = ninsns_rvoff(tc_ninsn - (ctx->ninsns - start_insn)); in emit_bpf_tail_call()
376 emit_branch(BPF_JEQ, RV_REG_T2, RV_REG_ZERO, off, ctx); in emit_bpf_tail_call()
378 /* goto *(prog->bpf_func + 4); */ in emit_bpf_tail_call()
379 off = offsetof(struct bpf_prog, bpf_func); in emit_bpf_tail_call()
380 if (is_12b_check(off, insn)) in emit_bpf_tail_call()
381 return -1; in emit_bpf_tail_call()
382 emit_ld(RV_REG_T3, off, RV_REG_T2, ctx); in emit_bpf_tail_call()
390 u8 code = insn->code; in init_regs()
399 *rd = bpf_to_rv_reg(insn->dst_reg, ctx); in init_regs()
405 *rs = bpf_to_rv_reg(insn->src_reg, ctx); in init_regs()
456 pr_err("bpf-jit: target offset 0x%llx is out of range\n", rvoff); in emit_jump_and_link()
457 return -ERANGE; in emit_jump_and_link()
468 s64 off = 0; in emit_call() local
471 if (addr && ctx->insns && ctx->ro_insns) { in emit_call()
476 ip = (u64)(long)(ctx->ro_insns + ctx->ninsns); in emit_call()
477 off = addr - ip; in emit_call()
480 return emit_jump_and_link(RV_REG_RA, off, fixed_addr, ctx); in emit_call()
483 static void emit_atomic(u8 rd, u8 rs, s16 off, s32 imm, bool is64, in emit_atomic() argument
489 if (off) { in emit_atomic()
490 if (is_12b_int(off)) { in emit_atomic()
491 emit_addi(RV_REG_T1, rd, off, ctx); in emit_atomic()
493 emit_imm(RV_REG_T1, off, ctx); in emit_atomic()
560 jmp_offset = ninsns_rvoff(-6); in emit_atomic()
573 off_t offset = FIELD_GET(BPF_FIXUP_OFFSET_MASK, ex->fixup); in ex_handler_bpf()
574 int regs_offset = FIELD_GET(BPF_FIXUP_REG_MASK, ex->fixup); in ex_handler_bpf()
577 regs->epc = (unsigned long)&ex->fixup - offset; in ex_handler_bpf()
592 if (!ctx->insns || !ctx->ro_insns || !ctx->prog->aux->extable || in add_exception_handler()
593 (BPF_MODE(insn->code) != BPF_PROBE_MEM && BPF_MODE(insn->code) != BPF_PROBE_MEMSX)) in add_exception_handler()
596 if (WARN_ON_ONCE(ctx->nexentries >= ctx->prog->aux->num_exentries)) in add_exception_handler()
597 return -EINVAL; in add_exception_handler()
599 if (WARN_ON_ONCE(insn_len > ctx->ninsns)) in add_exception_handler()
600 return -EINVAL; in add_exception_handler()
603 return -EINVAL; in add_exception_handler()
605 ex = &ctx->prog->aux->extable[ctx->nexentries]; in add_exception_handler()
606 pc = (unsigned long)&ctx->ro_insns[ctx->ninsns - insn_len]; in add_exception_handler()
614 ins_offset = pc - (long)&ex->insn; in add_exception_handler()
616 return -ERANGE; in add_exception_handler()
630 fixup_offset = (long)&ex->fixup - (pc + insn_len * sizeof(u16)); in add_exception_handler()
632 return -ERANGE; in add_exception_handler()
639 ex = (void *)ctx->insns + ((void *)ex - (void *)ctx->ro_insns); in add_exception_handler()
641 ex->insn = ins_offset; in add_exception_handler()
643 ex->fixup = FIELD_PREP(BPF_FIXUP_OFFSET_MASK, fixup_offset) | in add_exception_handler()
645 ex->type = EX_TYPE_BPF; in add_exception_handler()
647 ctx->nexentries++; in add_exception_handler()
665 rvoff = (s64)(target - ip); in gen_jump_or_nops()
678 return -ENOTSUPP; in bpf_arch_text_poke()
685 return -EFAULT; in bpf_arch_text_poke()
706 emit_sd(RV_REG_FP, -args_off, RV_REG_A0 + i, ctx); in store_args()
707 args_off -= 8; in store_args()
716 emit_ld(RV_REG_A0 + i, -args_off, RV_REG_FP, ctx); in restore_args()
717 args_off -= 8; in restore_args()
725 struct bpf_prog *p = l->link.prog; in invoke_bpf_prog()
728 if (l->cookie) { in invoke_bpf_prog()
729 emit_imm(RV_REG_T1, l->cookie, ctx); in invoke_bpf_prog()
730 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_T1, ctx); in invoke_bpf_prog()
732 emit_sd(RV_REG_FP, -run_ctx_off + cookie_off, RV_REG_ZERO, ctx); in invoke_bpf_prog()
738 emit_addi(RV_REG_A1, RV_REG_FP, -run_ctx_off, ctx); in invoke_bpf_prog()
746 branch_off = ctx->ninsns; in invoke_bpf_prog()
754 emit_addi(RV_REG_A0, RV_REG_FP, -args_off, ctx); in invoke_bpf_prog()
755 if (!p->jited) in invoke_bpf_prog()
756 /* arg2: progs[i]->insnsi for interpreter */ in invoke_bpf_prog()
757 emit_imm(RV_REG_A1, (const s64)p->insnsi, ctx); in invoke_bpf_prog()
758 ret = emit_call((const u64)p->bpf_func, true, ctx); in invoke_bpf_prog()
763 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); in invoke_bpf_prog()
764 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); in invoke_bpf_prog()
768 if (ctx->insns) { in invoke_bpf_prog()
769 int offset = ninsns_rvoff(ctx->ninsns - branch_off); in invoke_bpf_prog()
771 *(u32 *)(ctx->insns + branch_off) = insn; in invoke_bpf_prog()
779 emit_addi(RV_REG_A2, RV_REG_FP, -run_ctx_off, ctx); in invoke_bpf_prog()
793 int stack_size = 0, nregs = m->nr_args; in __arch_prepare_bpf_trampoline()
805 * 1. trampoline called from function entry in __arch_prepare_bpf_trampoline()
806 * -------------------------------------- in __arch_prepare_bpf_trampoline()
808 * function in __arch_prepare_bpf_trampoline()
810 * function in __arch_prepare_bpf_trampoline()
811 * FP - 8 [ T0 to traced func ] return address of traced in __arch_prepare_bpf_trampoline()
812 * function in __arch_prepare_bpf_trampoline()
813 * FP - 16 [ FP of traced func ] frame pointer of traced in __arch_prepare_bpf_trampoline()
814 * function in __arch_prepare_bpf_trampoline()
815 * -------------------------------------- in __arch_prepare_bpf_trampoline()
818 * -------------------------------------- in __arch_prepare_bpf_trampoline()
819 * FP - 8 [ RA to caller func ] return address to caller in __arch_prepare_bpf_trampoline()
820 * function in __arch_prepare_bpf_trampoline()
821 * FP - 16 [ FP of caller func ] frame pointer of caller in __arch_prepare_bpf_trampoline()
822 * function in __arch_prepare_bpf_trampoline()
823 * -------------------------------------- in __arch_prepare_bpf_trampoline()
825 * FP - retval_off [ return value ] BPF_TRAMP_F_CALL_ORIG or in __arch_prepare_bpf_trampoline()
829 * FP - args_off [ arg1 ] in __arch_prepare_bpf_trampoline()
831 * FP - nregs_off [ regs count ] in __arch_prepare_bpf_trampoline()
833 * FP - ip_off [ traced func ] BPF_TRAMP_F_IP_ARG in __arch_prepare_bpf_trampoline()
835 * FP - run_ctx_off [ bpf_tramp_run_ctx ] in __arch_prepare_bpf_trampoline()
837 * FP - sreg_off [ callee saved reg ] in __arch_prepare_bpf_trampoline()
843 return -ENOTSUPP; in __arch_prepare_bpf_trampoline()
846 for (i = 0; i < m->nr_args; i++) in __arch_prepare_bpf_trampoline()
847 if (m->arg_flags[i] & BTF_FMODEL_STRUCT_ARG) in __arch_prepare_bpf_trampoline()
848 nregs += round_up(m->arg_size[i], 8) / 8 - 1; in __arch_prepare_bpf_trampoline()
852 return -ENOTSUPP; in __arch_prepare_bpf_trampoline()
883 /* For the trampoline called from function entry, in __arch_prepare_bpf_trampoline()
884 * the frame of traced function and the frame of in __arch_prepare_bpf_trampoline()
887 emit_addi(RV_REG_SP, RV_REG_SP, -16, ctx); in __arch_prepare_bpf_trampoline()
892 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); in __arch_prepare_bpf_trampoline()
893 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_T0, ctx); in __arch_prepare_bpf_trampoline()
894 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
900 emit_addi(RV_REG_SP, RV_REG_SP, -stack_size, ctx); in __arch_prepare_bpf_trampoline()
901 emit_sd(RV_REG_SP, stack_size - 8, RV_REG_RA, ctx); in __arch_prepare_bpf_trampoline()
902 emit_sd(RV_REG_SP, stack_size - 16, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
907 emit_sd(RV_REG_FP, -sreg_off, RV_REG_S1, ctx); in __arch_prepare_bpf_trampoline()
909 /* store ip address of the traced function */ in __arch_prepare_bpf_trampoline()
912 emit_sd(RV_REG_FP, -ip_off, RV_REG_T1, ctx); in __arch_prepare_bpf_trampoline()
916 emit_sd(RV_REG_FP, -nregs_off, RV_REG_T1, ctx); in __arch_prepare_bpf_trampoline()
920 /* skip to actual body of traced function */ in __arch_prepare_bpf_trampoline()
931 for (i = 0; i < fentry->nr_links; i++) { in __arch_prepare_bpf_trampoline()
932 ret = invoke_bpf_prog(fentry->links[i], args_off, retval_off, run_ctx_off, in __arch_prepare_bpf_trampoline()
938 if (fmod_ret->nr_links) { in __arch_prepare_bpf_trampoline()
939 branches_off = kcalloc(fmod_ret->nr_links, sizeof(int), GFP_KERNEL); in __arch_prepare_bpf_trampoline()
941 return -ENOMEM; in __arch_prepare_bpf_trampoline()
944 emit_sd(RV_REG_FP, -retval_off, RV_REG_ZERO, ctx); in __arch_prepare_bpf_trampoline()
945 for (i = 0; i < fmod_ret->nr_links; i++) { in __arch_prepare_bpf_trampoline()
946 ret = invoke_bpf_prog(fmod_ret->links[i], args_off, retval_off, in __arch_prepare_bpf_trampoline()
950 emit_ld(RV_REG_T1, -retval_off, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
951 branches_off[i] = ctx->ninsns; in __arch_prepare_bpf_trampoline()
962 emit_sd(RV_REG_FP, -retval_off, RV_REG_A0, ctx); in __arch_prepare_bpf_trampoline()
963 emit_sd(RV_REG_FP, -(retval_off - 8), regmap[BPF_REG_0], ctx); in __arch_prepare_bpf_trampoline()
964 im->ip_after_call = ctx->insns + ctx->ninsns; in __arch_prepare_bpf_trampoline()
971 for (i = 0; ctx->insns && i < fmod_ret->nr_links; i++) { in __arch_prepare_bpf_trampoline()
972 offset = ninsns_rvoff(ctx->ninsns - branches_off[i]); in __arch_prepare_bpf_trampoline()
974 *(u32 *)(ctx->insns + branches_off[i]) = insn; in __arch_prepare_bpf_trampoline()
977 for (i = 0; i < fexit->nr_links; i++) { in __arch_prepare_bpf_trampoline()
978 ret = invoke_bpf_prog(fexit->links[i], args_off, retval_off, in __arch_prepare_bpf_trampoline()
985 im->ip_epilogue = ctx->insns + ctx->ninsns; in __arch_prepare_bpf_trampoline()
996 emit_ld(RV_REG_A0, -retval_off, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
997 emit_ld(regmap[BPF_REG_0], -(retval_off - 8), RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
1000 emit_ld(RV_REG_S1, -sreg_off, RV_REG_FP, ctx); in __arch_prepare_bpf_trampoline()
1003 /* trampoline called from function entry */ in __arch_prepare_bpf_trampoline()
1004 emit_ld(RV_REG_T0, stack_size - 8, RV_REG_SP, ctx); in __arch_prepare_bpf_trampoline()
1005 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); in __arch_prepare_bpf_trampoline()
1013 /* return to parent function */ in __arch_prepare_bpf_trampoline()
1016 /* return to traced function */ in __arch_prepare_bpf_trampoline()
1020 emit_ld(RV_REG_RA, stack_size - 8, RV_REG_SP, ctx); in __arch_prepare_bpf_trampoline()
1021 emit_ld(RV_REG_FP, stack_size - 16, RV_REG_SP, ctx); in __arch_prepare_bpf_trampoline()
1027 ret = ctx->ninsns; in __arch_prepare_bpf_trampoline()
1078 bool is64 = BPF_CLASS(insn->code) == BPF_ALU64 || in bpf_jit_emit_insn()
1079 BPF_CLASS(insn->code) == BPF_JMP; in bpf_jit_emit_insn()
1080 int s, e, rvoff, ret, i = insn - ctx->prog->insnsi; in bpf_jit_emit_insn()
1081 struct bpf_prog_aux *aux = ctx->prog->aux; in bpf_jit_emit_insn()
1082 u8 rd = -1, rs = -1, code = insn->code; in bpf_jit_emit_insn()
1083 s16 off = insn->off; in bpf_jit_emit_insn() local
1084 s32 imm = insn->imm; in bpf_jit_emit_insn()
1097 switch (insn->off) { in bpf_jit_emit_insn()
1103 emit_slli(RV_REG_T1, rs, 64 - insn->off, ctx); in bpf_jit_emit_insn()
1104 emit_srai(rd, RV_REG_T1, 64 - insn->off, ctx); in bpf_jit_emit_insn()
1110 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1118 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1128 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1134 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1140 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1146 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1152 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1157 if (off) in bpf_jit_emit_insn()
1161 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1166 if (off) in bpf_jit_emit_insn()
1170 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1176 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1182 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1188 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1192 /* dst = -dst */ in bpf_jit_emit_insn()
1196 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1208 if (!aux->verifier_zext) in bpf_jit_emit_insn()
1270 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1283 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1288 if (is_12b_int(-imm)) { in bpf_jit_emit_insn()
1289 emit_addi(rd, rd, -imm, ctx); in bpf_jit_emit_insn()
1294 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1305 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1316 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1327 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1335 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1341 if (off) in bpf_jit_emit_insn()
1347 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1353 if (off) in bpf_jit_emit_insn()
1359 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1366 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1376 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1386 if (!is64 && !aux->verifier_zext) in bpf_jit_emit_insn()
1390 /* JUMP off */ in bpf_jit_emit_insn()
1394 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
1402 /* IF (dst COND src) JUMP off */ in bpf_jit_emit_insn()
1425 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
1427 s = ctx->ninsns; in bpf_jit_emit_insn()
1432 e = ctx->ninsns; in bpf_jit_emit_insn()
1435 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
1440 rvoff -= 4; in bpf_jit_emit_insn()
1449 /* IF (dst COND imm) JUMP off */ in bpf_jit_emit_insn()
1470 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
1471 s = ctx->ninsns; in bpf_jit_emit_insn()
1485 e = ctx->ninsns; in bpf_jit_emit_insn()
1488 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
1494 rvoff = rv_offset(i, off, ctx); in bpf_jit_emit_insn()
1495 s = ctx->ninsns; in bpf_jit_emit_insn()
1503 * sign-extension is sufficient here and saves one instruction, in bpf_jit_emit_insn()
1508 e = ctx->ninsns; in bpf_jit_emit_insn()
1509 rvoff -= ninsns_rvoff(e - s); in bpf_jit_emit_insn()
1513 /* function call */ in bpf_jit_emit_insn()
1520 ret = bpf_jit_get_func_addr(ctx->prog, insn, extra_pass, in bpf_jit_emit_insn()
1529 if (insn->src_reg != BPF_PSEUDO_CALL) in bpf_jit_emit_insn()
1536 return -1; in bpf_jit_emit_insn()
1539 /* function return */ in bpf_jit_emit_insn()
1541 if (i == ctx->prog->len - 1) in bpf_jit_emit_insn()
1558 /* fixed-length insns for extra jit pass */ in bpf_jit_emit_insn()
1569 /* LDX: dst = *(unsigned size *)(src + off) */ in bpf_jit_emit_insn()
1578 /* LDSX: dst = *(signed size *)(src + off) */ in bpf_jit_emit_insn()
1589 sign_ext = BPF_MODE(insn->code) == BPF_MEMSX || in bpf_jit_emit_insn()
1590 BPF_MODE(insn->code) == BPF_PROBE_MEMSX; in bpf_jit_emit_insn()
1594 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1595 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1597 emit(rv_lb(rd, off, rs), ctx); in bpf_jit_emit_insn()
1599 emit(rv_lbu(rd, off, rs), ctx); in bpf_jit_emit_insn()
1600 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1604 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1606 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1611 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1614 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1615 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1617 emit(rv_lh(rd, off, rs), ctx); in bpf_jit_emit_insn()
1619 emit(rv_lhu(rd, off, rs), ctx); in bpf_jit_emit_insn()
1620 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1624 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1626 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1631 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1634 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1635 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1637 emit(rv_lw(rd, off, rs), ctx); in bpf_jit_emit_insn()
1639 emit(rv_lwu(rd, off, rs), ctx); in bpf_jit_emit_insn()
1640 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1644 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1646 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1651 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1654 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1655 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1656 emit_ld(rd, off, rs, ctx); in bpf_jit_emit_insn()
1657 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1661 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1663 insns_start = ctx->ninsns; in bpf_jit_emit_insn()
1665 insn_len = ctx->ninsns - insns_start; in bpf_jit_emit_insn()
1681 /* ST: *(size *)(dst + off) = imm */ in bpf_jit_emit_insn()
1684 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1685 emit(rv_sb(rd, off, RV_REG_T1), ctx); in bpf_jit_emit_insn()
1689 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
1696 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1697 emit(rv_sh(rd, off, RV_REG_T1), ctx); in bpf_jit_emit_insn()
1701 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
1707 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1708 emit_sw(rd, off, RV_REG_T1, ctx); in bpf_jit_emit_insn()
1712 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
1718 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1719 emit_sd(rd, off, RV_REG_T1, ctx); in bpf_jit_emit_insn()
1723 emit_imm(RV_REG_T2, off, ctx); in bpf_jit_emit_insn()
1728 /* STX: *(size *)(dst + off) = src */ in bpf_jit_emit_insn()
1730 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1731 emit(rv_sb(rd, off, rs), ctx); in bpf_jit_emit_insn()
1735 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1740 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1741 emit(rv_sh(rd, off, rs), ctx); in bpf_jit_emit_insn()
1745 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1750 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1751 emit_sw(rd, off, rs, ctx); in bpf_jit_emit_insn()
1755 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1760 if (is_12b_int(off)) { in bpf_jit_emit_insn()
1761 emit_sd(rd, off, rs, ctx); in bpf_jit_emit_insn()
1765 emit_imm(RV_REG_T1, off, ctx); in bpf_jit_emit_insn()
1771 emit_atomic(rd, rs, off, imm, in bpf_jit_emit_insn()
1775 pr_err("bpf-jit: unknown opcode %02x\n", code); in bpf_jit_emit_insn()
1776 return -EINVAL; in bpf_jit_emit_insn()
1786 bpf_stack_adjust = round_up(ctx->prog->aux->stack_depth, 16); in bpf_jit_build_prologue()
1809 store_offset = stack_adjust - 8; in bpf_jit_build_prologue()
1815 /* First instruction is always setting the tail-call-counter in bpf_jit_build_prologue()
1817 * Force using a 4-byte (non-compressed) instruction. in bpf_jit_build_prologue()
1821 emit_addi(RV_REG_SP, RV_REG_SP, -stack_adjust, ctx); in bpf_jit_build_prologue()
1825 store_offset -= 8; in bpf_jit_build_prologue()
1828 store_offset -= 8; in bpf_jit_build_prologue()
1831 store_offset -= 8; in bpf_jit_build_prologue()
1835 store_offset -= 8; in bpf_jit_build_prologue()
1839 store_offset -= 8; in bpf_jit_build_prologue()
1843 store_offset -= 8; in bpf_jit_build_prologue()
1847 store_offset -= 8; in bpf_jit_build_prologue()
1851 store_offset -= 8; in bpf_jit_build_prologue()
1865 ctx->stack_size = stack_adjust; in bpf_jit_build_prologue()