Lines Matching refs:nfp_prog
22 #define nfp_for_each_insn_walk2(nfp_prog, pos, next) \
23 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
25 &(nfp_prog)->insns != &pos->l && \
26 &(nfp_prog)->insns != &next->l; \
30 #define nfp_for_each_insn_walk3(nfp_prog, pos, next, next2) \
31 for (pos = list_first_entry(&(nfp_prog)->insns, typeof(*pos), l), \
34 &(nfp_prog)->insns != &pos->l && \
35 &(nfp_prog)->insns != &next->l && \
36 &(nfp_prog)->insns != &next2->l; \
42 nfp_meta_has_prev(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
44 return meta->l.prev != &nfp_prog->insns;
47 static void nfp_prog_push(struct nfp_prog *nfp_prog, u64 insn)
49 if (nfp_prog->__prog_alloc_len / sizeof(u64) == nfp_prog->prog_len) {
51 nfp_prog->prog_len);
52 nfp_prog->error = -ENOSPC;
56 nfp_prog->prog[nfp_prog->prog_len] = insn;
57 nfp_prog->prog_len++;
60 static unsigned int nfp_prog_current_offset(struct nfp_prog *nfp_prog)
62 return nfp_prog->prog_len;
66 nfp_prog_confirm_current_offset(struct nfp_prog *nfp_prog, unsigned int off)
72 if (nfp_prog->error)
74 return !WARN_ON_ONCE(nfp_prog_current_offset(nfp_prog) != off);
79 __emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op,
96 nfp_prog_push(nfp_prog, insn);
100 emit_cmd_any(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
108 nfp_prog->error = err;
113 nfp_prog->error = -EFAULT;
118 nfp_prog->error = -EFAULT;
122 __emit_cmd(nfp_prog, op, mode, xfer, reg.areg, reg.breg, size, ctx,
127 emit_cmd(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
130 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, false);
134 emit_cmd_indir(struct nfp_prog *nfp_prog, enum cmd_tgt_map op, u8 mode, u8 xfer,
137 emit_cmd_any(nfp_prog, op, mode, xfer, lreg, rreg, size, ctx, true);
141 __emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, enum br_ev_pip ev_pip,
158 nfp_prog_push(nfp_prog, insn);
162 emit_br_relo(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer,
167 nfp_prog->error = -EFAULT;
171 __emit_br(nfp_prog, mask,
175 nfp_prog->prog[nfp_prog->prog_len - 1] |=
180 emit_br(struct nfp_prog *nfp_prog, enum br_mask mask, u16 addr, u8 defer)
182 emit_br_relo(nfp_prog, mask, addr, defer, RELO_BR_REL);
186 __emit_br_bit(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 addr, u8 defer,
204 nfp_prog_push(nfp_prog, insn);
208 emit_br_bit_relo(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr,
222 nfp_prog->error = err;
226 __emit_br_bit(nfp_prog, reg.areg, reg.breg, addr, defer, set,
229 nfp_prog->prog[nfp_prog->prog_len - 1] |=
234 emit_br_bset(struct nfp_prog *nfp_prog, swreg src, u8 bit, u16 addr, u8 defer)
236 emit_br_bit_relo(nfp_prog, src, bit, addr, defer, true, RELO_BR_REL);
240 __emit_br_alu(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
253 nfp_prog_push(nfp_prog, insn);
256 static void emit_rtn(struct nfp_prog *nfp_prog, swreg base, u8 defer)
263 nfp_prog->error = err;
267 __emit_br_alu(nfp_prog, reg.areg, reg.breg, 0, defer, reg.dst_lmextn,
272 __emit_immed(struct nfp_prog *nfp_prog, u16 areg, u16 breg, u16 imm_hi,
290 nfp_prog_push(nfp_prog, insn);
294 emit_immed(struct nfp_prog *nfp_prog, swreg dst, u16 imm,
301 nfp_prog->error = -EFAULT;
307 nfp_prog->error = err;
312 __emit_immed(nfp_prog,
319 __emit_shf(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
327 nfp_prog->error = -EFAULT;
359 nfp_prog_push(nfp_prog, insn);
363 emit_shf(struct nfp_prog *nfp_prog, swreg dst,
371 nfp_prog->error = err;
375 __emit_shf(nfp_prog, reg.dst, reg.dst_ab, sc, shift,
381 emit_shf_indir(struct nfp_prog *nfp_prog, swreg dst,
386 nfp_prog->error = -EFAULT;
390 emit_shf(nfp_prog, dst, lreg, op, rreg, sc, 0);
394 __emit_alu(struct nfp_prog *nfp_prog, u16 dst, enum alu_dst_ab dst_ab,
411 nfp_prog_push(nfp_prog, insn);
415 emit_alu(struct nfp_prog *nfp_prog, swreg dst,
423 nfp_prog->error = err;
427 __emit_alu(nfp_prog, reg.dst, reg.dst_ab,
433 __emit_mul(struct nfp_prog *nfp_prog, enum alu_dst_ab dst_ab, u16 areg,
450 nfp_prog_push(nfp_prog, insn);
454 emit_mul(struct nfp_prog *nfp_prog, swreg lreg, enum mul_type type,
462 nfp_prog->error = -EINVAL;
478 nfp_prog->error = err;
482 __emit_mul(nfp_prog, reg.dst_ab, areg, type, step, reg.breg, reg.swap,
487 __emit_ld_field(struct nfp_prog *nfp_prog, enum shf_sc sc,
507 nfp_prog_push(nfp_prog, insn);
511 emit_ld_field_any(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
520 nfp_prog->error = err;
524 __emit_ld_field(nfp_prog, sc, reg.areg, bmask, reg.breg, shift,
530 emit_ld_field(struct nfp_prog *nfp_prog, swreg dst, u8 bmask, swreg src,
533 emit_ld_field_any(nfp_prog, dst, bmask, src, sc, shift, false);
537 __emit_lcsr(struct nfp_prog *nfp_prog, u16 areg, u16 breg, bool wr, u16 addr,
550 nfp_prog_push(nfp_prog, insn);
553 static void emit_csr_wr(struct nfp_prog *nfp_prog, swreg src, u16 addr)
570 nfp_prog->error = err;
574 __emit_lcsr(nfp_prog, reg.areg, reg.breg, true, addr,
579 static void __emit_csr_rd(struct nfp_prog *nfp_prog, u16 addr)
581 __emit_lcsr(nfp_prog, 0, 0, false, addr, false, false);
584 static void emit_nop(struct nfp_prog *nfp_prog)
586 __emit_immed(nfp_prog, UR_REG_IMM, UR_REG_IMM, 0, 0, 0, 0, 0, 0, 0);
608 static void wrp_immed(struct nfp_prog *nfp_prog, swreg dst, u32 imm)
614 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, false, shift);
616 emit_immed(nfp_prog, dst, val, IMMED_WIDTH_ALL, true, shift);
618 emit_immed(nfp_prog, dst, imm & 0xffff, IMMED_WIDTH_ALL,
620 emit_immed(nfp_prog, dst, imm >> 16, IMMED_WIDTH_WORD,
626 wrp_zext(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst)
629 wrp_immed(nfp_prog, reg_both(dst + 1), 0);
633 wrp_immed_relo(struct nfp_prog *nfp_prog, swreg dst, u32 imm,
638 nfp_prog->error = -EFAULT;
641 emit_immed(nfp_prog, dst, imm, IMMED_WIDTH_ALL, false, IMMED_SHIFT_0B);
643 nfp_prog->prog[nfp_prog->prog_len - 1] |=
651 static swreg ur_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
656 wrp_immed(nfp_prog, tmp_reg, imm);
664 static swreg re_load_imm_any(struct nfp_prog *nfp_prog, u32 imm, swreg tmp_reg)
669 wrp_immed(nfp_prog, tmp_reg, imm);
673 static void wrp_nops(struct nfp_prog *nfp_prog, unsigned int count)
676 emit_nop(nfp_prog);
679 static void wrp_mov(struct nfp_prog *nfp_prog, swreg dst, swreg src)
681 emit_alu(nfp_prog, dst, reg_none(), ALU_OP_NONE, src);
684 static void wrp_reg_mov(struct nfp_prog *nfp_prog, u16 dst, u16 src)
686 wrp_mov(nfp_prog, reg_both(dst), reg_b(src));
693 wrp_reg_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src, u8 field_len,
699 emit_ld_field_any(nfp_prog, dst, mask, src, sc, offset * 8, true);
706 wrp_reg_or_subpart(struct nfp_prog *nfp_prog, swreg dst, swreg src,
712 emit_ld_field(nfp_prog, dst, mask, src, sc, 32 - offset * 8);
716 addr40_offset(struct nfp_prog *nfp_prog, u8 src_gpr, swreg offset,
725 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(src_gpr), ALU_OP_ADD, offset);
726 emit_alu(nfp_prog, imm_b(nfp_prog), reg_b(src_gpr + 1), ALU_OP_ADD_C,
728 *rega = imm_a(nfp_prog);
729 *regb = imm_b(nfp_prog);
733 static int nfp_cpp_memcpy(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
742 off = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
748 addr40_offset(nfp_prog, meta->insn.src_reg * 2, off, &src_base,
753 wrp_immed(nfp_prog, reg_none(),
757 emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP,
763 wrp_mov(nfp_prog, reg_xfer(i), reg_xfer(i));
765 off = re_load_imm_any(nfp_prog, meta->paired_st->off, imm_b(nfp_prog));
769 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
774 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
779 wrp_immed(nfp_prog, reg_none(),
781 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
786 wrp_immed(nfp_prog, reg_none(),
788 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
795 emit_cmd(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
799 off = re_load_imm_any(nfp_prog, meta->paired_st->off + 32,
800 imm_b(nfp_prog));
801 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 8,
810 wrp_immed(nfp_prog, reg_none(),
812 emit_cmd_indir(nfp_prog, CMD_TGT_WRITE32_SWAP, CMD_MODE_32b, 0,
816 off = re_load_imm_any(nfp_prog, new_off, imm_b(nfp_prog));
817 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b,
841 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
846 wrp_reg_subpart(nfp_prog, reg_both(meta->insn.dst_reg * 2),
850 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
854 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2),
856 wrp_mov(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1),
862 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
868 data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, swreg offset,
880 emit_cmd(nfp_prog, CMD_TGT_READ8, CMD_MODE_32b, 0,
881 pptr_reg(nfp_prog), offset, sz - 1, CMD_CTX_SWAP);
885 emit_shf(nfp_prog, reg_both(dst_gpr), reg_none(), SHF_OP_NONE,
889 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
892 wrp_zext(nfp_prog, meta, dst_gpr);
898 data_ld_host_order(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
911 emit_cmd(nfp_prog, CMD_TGT_READ32_SWAP, mode, 0,
916 emit_ld_field_any(nfp_prog, reg_both(dst_gpr), mask,
920 wrp_mov(nfp_prog, reg_both(dst_gpr + i), reg_xfer(i));
923 wrp_zext(nfp_prog, meta, dst_gpr);
929 data_ld_host_order_addr32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
932 return data_ld_host_order(nfp_prog, meta, dst_gpr, reg_a(src_gpr),
937 data_ld_host_order_addr40(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
942 addr40_offset(nfp_prog, src_gpr, offset, ®a, ®b);
944 return data_ld_host_order(nfp_prog, meta, dst_gpr, rega, regb,
949 construct_data_ind_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
955 tmp_reg = ur_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
956 emit_alu(nfp_prog, imm_both(nfp_prog), reg_a(src), ALU_OP_ADD, tmp_reg);
959 emit_alu(nfp_prog, imm_a(nfp_prog),
960 imm_a(nfp_prog), ALU_OP_ADD, reg_imm(size));
961 emit_alu(nfp_prog, reg_none(),
962 plen_reg(nfp_prog), ALU_OP_SUB, imm_a(nfp_prog));
963 emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
966 return data_ld(nfp_prog, meta, imm_b(nfp_prog), 0, size);
970 construct_data_ld(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
976 tmp_reg = ur_load_imm_any(nfp_prog, offset + size, imm_a(nfp_prog));
977 emit_alu(nfp_prog, reg_none(), plen_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
978 emit_br_relo(nfp_prog, BR_BLO, BR_OFF_RELO, 0, RELO_BR_GO_ABORT);
981 tmp_reg = re_load_imm_any(nfp_prog, offset, imm_b(nfp_prog));
982 return data_ld(nfp_prog, meta, tmp_reg, 0, size);
986 data_stx_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset,
992 wrp_mov(nfp_prog, reg_xfer(i), reg_a(src_gpr + i));
994 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
1001 data_st_host_order(struct nfp_prog *nfp_prog, u8 dst_gpr, swreg offset,
1004 wrp_immed(nfp_prog, reg_xfer(0), imm);
1006 wrp_immed(nfp_prog, reg_xfer(1), imm >> 32);
1008 emit_cmd(nfp_prog, CMD_TGT_WRITE8_SWAP, CMD_MODE_32b, 0,
1015 (*lmem_step)(struct nfp_prog *nfp_prog, u8 gpr, u8 gpr_byte, s32 off,
1020 wrp_lmem_load(struct nfp_prog *nfp_prog, u8 dst, u8 dst_byte, s32 off,
1038 wrp_mov(nfp_prog, reg_both(dst),
1070 reg = imm_a(nfp_prog);
1077 wrp_mov(nfp_prog, reg, reg_lm(0, idx));
1080 emit_ld_field_any(nfp_prog, reg_both(dst), mask, reg, sc, shf, new_gpr);
1083 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3));
1089 wrp_lmem_store(struct nfp_prog *nfp_prog, u8 src, u8 src_byte, s32 off,
1107 wrp_mov(nfp_prog,
1140 reg = imm_a(nfp_prog);
1145 wrp_mov(nfp_prog, reg, reg_lm(0, idx));
1148 emit_ld_field(nfp_prog, reg, mask, reg_b(src), sc, shf);
1152 wrp_mov(nfp_prog, reg_lm(0, idx), reg);
1154 wrp_mov(nfp_prog, reg_none(), reg_lm_inc(3));
1161 mem_op_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1165 s32 off = nfp_prog->stack_frame_depth + meta->insn.off + ptr_off;
1180 stack_off_reg = ur_load_imm_any(nfp_prog, meta->insn.off,
1181 stack_imm(nfp_prog));
1183 emit_alu(nfp_prog, imm_b(nfp_prog),
1197 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 32),
1198 stack_imm(nfp_prog));
1199 emit_alu(nfp_prog, imm_b(nfp_prog),
1200 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg);
1204 stack_off_reg = ur_load_imm_any(nfp_prog, round_down(off, 4),
1205 stack_imm(nfp_prog));
1207 emit_alu(nfp_prog, imm_b(nfp_prog),
1208 stack_reg(nfp_prog), ALU_OP_ADD, stack_off_reg);
1218 emit_csr_wr(nfp_prog, imm_b(nfp_prog), NFP_CSR_ACT_LM_ADDR3);
1224 wrp_nops(nfp_prog, nop_cnt);
1228 wrp_zext(nfp_prog, meta, gpr);
1243 ret = step(nfp_prog, gpr, gpr_byte, off, slice_size,
1265 wrp_alu_imm(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u32 imm)
1271 wrp_immed(nfp_prog, reg_both(dst), 0);
1277 wrp_immed(nfp_prog, reg_both(dst), ~0U);
1283 emit_alu(nfp_prog, reg_both(dst), reg_none(),
1289 tmp_reg = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
1290 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, tmp_reg);
1294 wrp_alu64_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1305 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, alu_op, imm & ~0U);
1306 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, alu_op, imm >> 32);
1312 wrp_alu64_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1317 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
1318 emit_alu(nfp_prog, reg_both(dst + 1),
1325 wrp_alu32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1331 wrp_alu_imm(nfp_prog, dst, alu_op, insn->imm);
1332 wrp_zext(nfp_prog, meta, dst);
1338 wrp_alu32_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1343 emit_alu(nfp_prog, reg_both(dst), reg_a(dst), alu_op, reg_b(src));
1344 wrp_zext(nfp_prog, meta, dst);
1350 wrp_test_reg_one(struct nfp_prog *nfp_prog, u8 dst, enum alu_op alu_op, u8 src,
1353 emit_alu(nfp_prog, reg_none(), reg_a(dst), alu_op, reg_b(src));
1354 emit_br(nfp_prog, br_mask, off, 0);
1358 wrp_test_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1363 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2, alu_op,
1366 wrp_test_reg_one(nfp_prog, insn->dst_reg * 2 + 1, alu_op,
1400 static int cmp_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1416 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
1418 emit_alu(nfp_prog, reg_none(), reg_a(reg), alu_op, tmp_reg);
1420 emit_alu(nfp_prog, reg_none(), tmp_reg, alu_op, reg_a(reg));
1423 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
1425 emit_alu(nfp_prog, reg_none(),
1428 emit_alu(nfp_prog, reg_none(),
1432 emit_br(nfp_prog, code->br_mask, insn->off, 0);
1437 static int cmp_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1456 emit_alu(nfp_prog, reg_none(), reg_a(areg), ALU_OP_SUB, reg_b(breg));
1458 emit_alu(nfp_prog, reg_none(),
1460 emit_br(nfp_prog, code->br_mask, insn->off, 0);
1465 static void wrp_end32(struct nfp_prog *nfp_prog, swreg reg_in, u8 gpr_out)
1467 emit_ld_field(nfp_prog, reg_both(gpr_out), 0xf, reg_in,
1469 emit_ld_field(nfp_prog, reg_both(gpr_out), 0x5, reg_a(gpr_out),
1474 wrp_mul_u32(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg,
1477 emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg);
1478 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_1, rreg);
1479 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_2, rreg);
1480 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_3, rreg);
1481 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_32x32, MUL_STEP_4, rreg);
1482 emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_32x32, MUL_LAST, reg_none());
1484 emit_mul(nfp_prog, dst_hi, MUL_TYPE_STEP_32x32, MUL_LAST_2,
1487 wrp_immed(nfp_prog, dst_hi, 0);
1491 wrp_mul_u16(struct nfp_prog *nfp_prog, swreg dst_hi, swreg dst_lo, swreg lreg,
1494 emit_mul(nfp_prog, lreg, MUL_TYPE_START, MUL_STEP_NONE, rreg);
1495 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_1, rreg);
1496 emit_mul(nfp_prog, lreg, MUL_TYPE_STEP_16x16, MUL_STEP_2, rreg);
1497 emit_mul(nfp_prog, dst_lo, MUL_TYPE_STEP_16x16, MUL_LAST, reg_none());
1501 wrp_mul(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
1520 multiplier = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
1524 wrp_mul_u32(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier,
1527 wrp_mul_u16(nfp_prog, dst_hi, dst_lo, multiplicand, multiplier);
1532 static int wrp_div_imm(struct nfp_prog *nfp_prog, u8 dst, u64 imm)
1540 wrp_immed(nfp_prog, dst_both, 0);
1556 swreg tmp_b = ur_load_imm_any(nfp_prog, imm, imm_b(nfp_prog));
1558 emit_alu(nfp_prog, reg_none(), dst_a, ALU_OP_SUB, tmp_b);
1559 wrp_immed(nfp_prog, imm_a(nfp_prog), 0);
1560 emit_alu(nfp_prog, dst_both, imm_a(nfp_prog), ALU_OP_ADD_C,
1573 magic = ur_load_imm_any(nfp_prog, rvalue.m, imm_b(nfp_prog));
1575 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
1578 wrp_mul_u32(nfp_prog, imm_both(nfp_prog), reg_none(), dst_a,
1580 emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_SUB,
1581 imm_b(nfp_prog));
1582 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
1584 emit_alu(nfp_prog, dst_both, dst_a, ALU_OP_ADD,
1585 imm_b(nfp_prog));
1586 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE, dst_b,
1590 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE,
1592 wrp_mul_u32(nfp_prog, dst_both, reg_none(), dst_a, magic, true);
1593 emit_shf(nfp_prog, dst_both, reg_none(), SHF_OP_NONE,
1600 static int adjust_head(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1602 swreg tmp = imm_a(nfp_prog), tmp_len = imm_b(nfp_prog);
1606 adjust_head = &nfp_prog->bpf->adjust_head;
1609 if (nfp_prog->adjust_head_location != UINT_MAX) {
1610 if (WARN_ON_ONCE(nfp_prog->adjust_head_location != meta->n))
1613 emit_alu(nfp_prog, pptr_reg(nfp_prog),
1614 reg_a(2 * 2), ALU_OP_ADD, pptr_reg(nfp_prog));
1615 emit_alu(nfp_prog, plen_reg(nfp_prog),
1616 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
1617 emit_alu(nfp_prog, pv_len(nfp_prog),
1618 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
1620 wrp_immed(nfp_prog, reg_both(0), 0);
1621 wrp_immed(nfp_prog, reg_both(1), 0);
1630 ret_einval = nfp_prog_current_offset(nfp_prog) + 14;
1634 emit_alu(nfp_prog, tmp,
1635 reg_a(2 * 2), ALU_OP_ADD_2B, pptr_reg(nfp_prog));
1638 emit_alu(nfp_prog, reg_none(),
1640 emit_br(nfp_prog, BR_BLO, ret_einval, 0);
1641 emit_alu(nfp_prog, reg_none(),
1643 emit_br(nfp_prog, BR_BLO, ret_einval, 0);
1646 emit_alu(nfp_prog, tmp_len,
1647 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
1648 emit_alu(nfp_prog, reg_none(),
1650 emit_br(nfp_prog, BR_BMI, ret_einval, 0);
1653 wrp_immed(nfp_prog, reg_both(0), 0);
1654 wrp_immed(nfp_prog, reg_both(1), 0);
1657 emit_ld_field(nfp_prog, pptr_reg(nfp_prog), 0x3, tmp, SHF_SC_NONE, 0);
1660 emit_br(nfp_prog, BR_UNC, end, 2);
1662 emit_alu(nfp_prog, plen_reg(nfp_prog),
1663 plen_reg(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
1664 emit_alu(nfp_prog, pv_len(nfp_prog),
1665 pv_len(nfp_prog), ALU_OP_SUB, reg_a(2 * 2));
1668 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval))
1671 wrp_immed(nfp_prog, reg_both(0), -22);
1672 wrp_immed(nfp_prog, reg_both(1), ~0);
1674 if (!nfp_prog_confirm_current_offset(nfp_prog, end))
1680 static int adjust_tail(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1685 BUILD_BUG_ON(plen_reg(nfp_prog) != reg_b(STATIC_REG_PKT_LEN));
1687 plen = imm_a(nfp_prog);
1690 ret_einval = nfp_prog_current_offset(nfp_prog) + 9;
1691 end = nfp_prog_current_offset(nfp_prog) + 11;
1694 emit_alu(nfp_prog, plen, plen_reg(nfp_prog), ALU_OP_ADD, delta);
1698 emit_br(nfp_prog, BR_BCC, ret_einval, 0);
1701 emit_alu(nfp_prog, reg_none(), plen, ALU_OP_SUB, reg_imm(ETH_HLEN));
1702 emit_br(nfp_prog, BR_BMI, ret_einval, 0);
1704 emit_alu(nfp_prog, plen_reg(nfp_prog),
1705 plen_reg(nfp_prog), ALU_OP_ADD, delta);
1706 emit_alu(nfp_prog, pv_len(nfp_prog),
1707 pv_len(nfp_prog), ALU_OP_ADD, delta);
1709 emit_br(nfp_prog, BR_UNC, end, 2);
1710 wrp_immed(nfp_prog, reg_both(0), 0);
1711 wrp_immed(nfp_prog, reg_both(1), 0);
1713 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_einval))
1716 wrp_immed(nfp_prog, reg_both(0), -22);
1717 wrp_immed(nfp_prog, reg_both(1), ~0);
1719 if (!nfp_prog_confirm_current_offset(nfp_prog, end))
1726 map_call_stack_common(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1733 lm_off = nfp_prog->stack_frame_depth;
1739 emit_csr_wr(nfp_prog, reg_b(2 * 2), NFP_CSR_ACT_LM_ADDR0);
1741 emit_csr_wr(nfp_prog, reg_b(3 * 2), NFP_CSR_ACT_LM_ADDR2);
1743 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
1745 ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;
1748 wrp_mov(nfp_prog, reg_a(0), reg_a(2));
1751 wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL);
1753 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
1760 emit_csr_wr(nfp_prog, stack_reg(nfp_prog), NFP_CSR_ACT_LM_ADDR0);
1761 wrp_nops(nfp_prog, 3);
1767 nfp_get_prandom_u32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1769 __emit_csr_rd(nfp_prog, NFP_CSR_PSEUDO_RND_NUM);
1771 emit_immed(nfp_prog, reg_both(0), 0,
1773 emit_immed(nfp_prog, reg_both(1), 0,
1779 nfp_perf_event_output(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1784 ptr_type = ur_load_imm_any(nfp_prog, meta->arg1.type, imm_a(nfp_prog));
1786 ret_tgt = nfp_prog_current_offset(nfp_prog) + 3;
1788 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO + meta->func_id,
1792 wrp_mov(nfp_prog, reg_a(1), ptr_type);
1795 wrp_immed_relo(nfp_prog, reg_b(0), ret_tgt, RELO_IMMED_REL);
1797 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
1804 nfp_queue_select(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1808 jmp_tgt = nfp_prog_current_offset(nfp_prog) + 5;
1811 emit_alu(nfp_prog, reg_none(), reg_a(meta->insn.src_reg * 2),
1813 emit_br(nfp_prog, BR_BEQ, jmp_tgt, 2);
1816 emit_shf(nfp_prog, pv_qsel_set(nfp_prog),
1817 pv_qsel_set(nfp_prog), SHF_OP_OR, reg_imm(1),
1819 emit_ld_field(nfp_prog,
1820 pv_qsel_val(nfp_prog), 0x1, reg_b(meta->insn.src_reg * 2),
1825 emit_ld_field(nfp_prog,
1826 pv_qsel_val(nfp_prog), 0x1, reg_imm(NFP_NET_RXR_MAX),
1829 if (!nfp_prog_confirm_current_offset(nfp_prog, jmp_tgt))
1836 static int mov_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1845 stack_depth_reg = ur_load_imm_any(nfp_prog,
1846 nfp_prog->stack_frame_depth,
1847 stack_imm(nfp_prog));
1848 emit_alu(nfp_prog, reg_both(dst), stack_reg(nfp_prog),
1850 wrp_immed(nfp_prog, reg_both(dst + 1), 0);
1852 wrp_reg_mov(nfp_prog, dst, src);
1853 wrp_reg_mov(nfp_prog, dst + 1, src + 1);
1859 static int mov_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1863 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2), imm & ~0U);
1864 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), imm >> 32);
1869 static int xor_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1871 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_XOR);
1874 static int xor_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1876 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_XOR, !meta->insn.imm);
1879 static int and_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1881 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_AND);
1884 static int and_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1886 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_AND, !~meta->insn.imm);
1889 static int or_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1891 return wrp_alu64_reg(nfp_prog, meta, ALU_OP_OR);
1894 static int or_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1896 return wrp_alu64_imm(nfp_prog, meta, ALU_OP_OR, !meta->insn.imm);
1899 static int add_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1903 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
1906 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
1913 static int add_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1918 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_ADD, imm & ~0U);
1919 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_ADD_C, imm >> 32);
1924 static int sub_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1928 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2),
1931 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1),
1938 static int sub_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1943 wrp_alu_imm(nfp_prog, insn->dst_reg * 2, ALU_OP_SUB, imm & ~0U);
1944 wrp_alu_imm(nfp_prog, insn->dst_reg * 2 + 1, ALU_OP_SUB_C, imm >> 32);
1949 static int mul_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1951 return wrp_mul(nfp_prog, meta, true, true);
1954 static int mul_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1956 return wrp_mul(nfp_prog, meta, true, false);
1959 static int div_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1963 return wrp_div_imm(nfp_prog, insn->dst_reg * 2, insn->imm);
1966 static int div_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1971 return wrp_div_imm(nfp_prog, meta->insn.dst_reg * 2, meta->umin_src);
1974 static int neg_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
1978 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2), reg_imm(0),
1980 emit_alu(nfp_prog, reg_both(insn->dst_reg * 2 + 1), reg_imm(0),
1996 static int __shl_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
2002 emit_shf(nfp_prog, reg_both(dst + 1), reg_a(dst + 1),
2005 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
2008 wrp_reg_mov(nfp_prog, dst + 1, dst);
2009 wrp_immed(nfp_prog, reg_both(dst), 0);
2011 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
2013 wrp_immed(nfp_prog, reg_both(dst), 0);
2019 static int shl_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2024 return __shl_imm64(nfp_prog, dst, insn->imm);
2027 static void shl_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2029 emit_alu(nfp_prog, imm_both(nfp_prog), reg_imm(32), ALU_OP_SUB,
2031 emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR, reg_imm(0));
2032 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_a(dst + 1), SHF_OP_NONE,
2037 static void shl_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2039 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
2040 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
2044 static void shl_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2046 shl_reg64_lt32_high(nfp_prog, dst, src);
2047 shl_reg64_lt32_low(nfp_prog, dst, src);
2050 static void shl_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2052 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
2053 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
2055 wrp_immed(nfp_prog, reg_both(dst), 0);
2058 static int shl_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2068 return __shl_imm64(nfp_prog, dst, umin);
2072 shl_reg64_lt32(nfp_prog, dst, src);
2074 shl_reg64_ge32(nfp_prog, dst, src);
2081 label_ge32 = nfp_prog_current_offset(nfp_prog) + 7;
2082 emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
2084 shl_reg64_lt32_high(nfp_prog, dst, src);
2085 label_end = nfp_prog_current_offset(nfp_prog) + 6;
2086 emit_br(nfp_prog, BR_UNC, label_end, 2);
2088 shl_reg64_lt32_low(nfp_prog, dst, src);
2090 if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
2092 shl_reg64_ge32(nfp_prog, dst, src);
2094 if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
2111 static int __shr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
2117 emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
2119 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
2122 wrp_reg_mov(nfp_prog, dst, dst + 1);
2123 wrp_immed(nfp_prog, reg_both(dst + 1), 0);
2125 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
2127 wrp_immed(nfp_prog, reg_both(dst + 1), 0);
2133 static int shr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2138 return __shr_imm64(nfp_prog, dst, insn->imm);
2142 static void shr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2144 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
2145 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_NONE,
2149 static void shr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2151 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
2152 emit_shf_indir(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
2156 static void shr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2158 shr_reg64_lt32_low(nfp_prog, dst, src);
2159 shr_reg64_lt32_high(nfp_prog, dst, src);
2162 static void shr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2164 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
2165 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
2167 wrp_immed(nfp_prog, reg_both(dst + 1), 0);
2170 static int shr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2180 return __shr_imm64(nfp_prog, dst, umin);
2184 shr_reg64_lt32(nfp_prog, dst, src);
2186 shr_reg64_ge32(nfp_prog, dst, src);
2193 label_ge32 = nfp_prog_current_offset(nfp_prog) + 6;
2194 emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
2195 shr_reg64_lt32_low(nfp_prog, dst, src);
2196 label_end = nfp_prog_current_offset(nfp_prog) + 6;
2197 emit_br(nfp_prog, BR_UNC, label_end, 2);
2199 shr_reg64_lt32_high(nfp_prog, dst, src);
2201 if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
2203 shr_reg64_ge32(nfp_prog, dst, src);
2205 if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
2215 static int __ashr_imm64(struct nfp_prog *nfp_prog, u8 dst, u8 shift_amt)
2221 emit_shf(nfp_prog, reg_both(dst), reg_a(dst + 1), SHF_OP_NONE,
2224 emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR,
2226 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
2230 wrp_reg_mov(nfp_prog, dst, dst + 1);
2231 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
2234 emit_alu(nfp_prog, reg_none(), reg_a(dst + 1), ALU_OP_OR,
2236 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
2238 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
2245 static int ashr_imm64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2250 return __ashr_imm64(nfp_prog, dst, insn->imm);
2253 static void ashr_reg64_lt32_high(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2258 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1));
2259 emit_shf_indir(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
2263 static void ashr_reg64_lt32_low(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2268 return shr_reg64_lt32_low(nfp_prog, dst, src);
2271 static void ashr_reg64_lt32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2273 ashr_reg64_lt32_low(nfp_prog, dst, src);
2274 ashr_reg64_lt32_high(nfp_prog, dst, src);
2277 static void ashr_reg64_ge32(struct nfp_prog *nfp_prog, u8 dst, u8 src)
2279 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst + 1));
2280 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
2282 emit_shf(nfp_prog, reg_both(dst + 1), reg_none(), SHF_OP_ASHR,
2287 static int ashr_reg64(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2297 return __ashr_imm64(nfp_prog, dst, umin);
2301 ashr_reg64_lt32(nfp_prog, dst, src);
2303 ashr_reg64_ge32(nfp_prog, dst, src);
2307 label_ge32 = nfp_prog_current_offset(nfp_prog) + 6;
2308 emit_br_bset(nfp_prog, reg_a(src), 5, label_ge32, 0);
2309 ashr_reg64_lt32_low(nfp_prog, dst, src);
2310 label_end = nfp_prog_current_offset(nfp_prog) + 6;
2311 emit_br(nfp_prog, BR_UNC, label_end, 2);
2313 ashr_reg64_lt32_high(nfp_prog, dst, src);
2315 if (!nfp_prog_confirm_current_offset(nfp_prog, label_ge32))
2317 ashr_reg64_ge32(nfp_prog, dst, src);
2319 if (!nfp_prog_confirm_current_offset(nfp_prog, label_end))
2326 static int mov_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2330 wrp_reg_mov(nfp_prog, insn->dst_reg * 2, insn->src_reg * 2);
2331 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
2336 static int mov_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2340 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2), insn->imm);
2341 wrp_immed(nfp_prog, reg_both(insn->dst_reg * 2 + 1), 0);
2346 static int xor_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2348 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_XOR);
2351 static int xor_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2353 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_XOR);
2356 static int and_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2358 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_AND);
2361 static int and_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2363 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_AND);
2366 static int or_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2368 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_OR);
2371 static int or_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2373 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_OR);
2376 static int add_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2378 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_ADD);
2381 static int add_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2383 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_ADD);
2386 static int sub_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2388 return wrp_alu32_reg(nfp_prog, meta, ALU_OP_SUB);
2391 static int sub_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2393 return wrp_alu32_imm(nfp_prog, meta, ALU_OP_SUB);
2396 static int mul_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2398 return wrp_mul(nfp_prog, meta, false, true);
2401 static int mul_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2403 return wrp_mul(nfp_prog, meta, false, false);
2406 static int div_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2408 return div_reg64(nfp_prog, meta);
2411 static int div_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2413 return div_imm64(nfp_prog, meta);
2416 static int neg_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2420 emit_alu(nfp_prog, reg_both(dst), reg_imm(0), ALU_OP_SUB, reg_b(dst));
2421 wrp_zext(nfp_prog, meta, dst);
2427 __ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
2432 emit_alu(nfp_prog, reg_none(), reg_a(dst), ALU_OP_OR,
2434 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
2437 wrp_zext(nfp_prog, meta, dst);
2442 static int ashr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2452 return __ashr_imm(nfp_prog, meta, dst, umin);
2458 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_b(dst));
2459 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_ASHR,
2461 wrp_zext(nfp_prog, meta, dst);
2466 static int ashr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2471 return __ashr_imm(nfp_prog, meta, dst, insn->imm);
2475 __shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
2479 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
2481 wrp_zext(nfp_prog, meta, dst);
2485 static int shr_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2490 return __shr_imm(nfp_prog, meta, dst, insn->imm);
2493 static int shr_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2503 return __shr_imm(nfp_prog, meta, dst, umin);
2506 emit_alu(nfp_prog, reg_none(), reg_a(src), ALU_OP_OR, reg_imm(0));
2507 emit_shf_indir(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
2509 wrp_zext(nfp_prog, meta, dst);
2514 __shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, u8 dst,
2518 emit_shf(nfp_prog, reg_both(dst), reg_none(), SHF_OP_NONE,
2520 wrp_zext(nfp_prog, meta, dst);
2524 static int shl_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2529 return __shl_imm(nfp_prog, meta, dst, insn->imm);
2532 static int shl_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2542 return __shl_imm(nfp_prog, meta, dst, umin);
2545 shl_reg64_lt32_low(nfp_prog, dst, src);
2546 wrp_zext(nfp_prog, meta, dst);
2550 static int end_reg32(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2557 emit_ld_field(nfp_prog, reg_both(gpr), 0x9, reg_b(gpr),
2559 emit_ld_field(nfp_prog, reg_both(gpr), 0xe, reg_a(gpr),
2562 wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
2565 wrp_end32(nfp_prog, reg_a(gpr), gpr);
2566 wrp_immed(nfp_prog, reg_both(gpr + 1), 0);
2569 wrp_mov(nfp_prog, imm_a(nfp_prog), reg_b(gpr + 1));
2571 wrp_end32(nfp_prog, reg_a(gpr), gpr + 1);
2572 wrp_end32(nfp_prog, imm_a(nfp_prog), gpr);
2579 static int imm_ld8_part2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2589 wrp_immed(nfp_prog, reg_both(dst), imm_lo);
2593 wrp_mov(nfp_prog, reg_both(dst + 1), reg_a(dst));
2595 wrp_immed(nfp_prog, reg_both(dst + 1), imm_hi);
2600 static int imm_ld8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2606 static int data_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2608 return construct_data_ld(nfp_prog, meta, meta->insn.imm, 1);
2611 static int data_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2613 return construct_data_ld(nfp_prog, meta, meta->insn.imm, 2);
2616 static int data_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2618 return construct_data_ld(nfp_prog, meta, meta->insn.imm, 4);
2621 static int data_ind_ld1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2623 return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
2627 static int data_ind_ld2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2629 return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
2633 static int data_ind_ld4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2635 return construct_data_ind_ld(nfp_prog, meta, meta->insn.imm,
2640 mem_ldx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2643 return mem_op_stack(nfp_prog, meta, size, ptr_off,
2648 static int mem_ldx_skb(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2657 wrp_mov(nfp_prog, dst, plen_reg(nfp_prog));
2662 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
2667 emit_alu(nfp_prog, dst,
2668 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
2674 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
2679 static int mem_ldx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2688 wrp_mov(nfp_prog, dst, pptr_reg(nfp_prog));
2693 emit_alu(nfp_prog, dst,
2694 plen_reg(nfp_prog), ALU_OP_ADD, pptr_reg(nfp_prog));
2700 wrp_immed(nfp_prog, reg_both(meta->insn.dst_reg * 2 + 1), 0);
2706 mem_ldx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2711 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2713 return data_ld_host_order_addr32(nfp_prog, meta, meta->insn.src_reg * 2,
2718 mem_ldx_emem(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2723 tmp_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2725 return data_ld_host_order_addr40(nfp_prog, meta, meta->insn.src_reg * 2,
2730 mem_ldx_data_init_pktcache(struct nfp_prog *nfp_prog,
2739 off = re_load_imm_any(nfp_prog, range_start, imm_b(nfp_prog));
2747 wrp_immed(nfp_prog, reg_none(),
2751 emit_cmd_any(nfp_prog, CMD_TGT_READ32_SWAP, CMD_MODE_32b, 0, src_base,
2756 mem_ldx_data_from_pktcache_unaligned(struct nfp_prog *nfp_prog,
2783 wrp_reg_subpart(nfp_prog, dst_lo, src_lo, len_lo, off);
2786 wrp_zext(nfp_prog, meta, dst_gpr);
2793 wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid, len_mid, len_lo);
2794 wrp_zext(nfp_prog, meta, dst_gpr);
2798 wrp_reg_or_subpart(nfp_prog, dst_lo, src_mid,
2800 wrp_reg_subpart(nfp_prog, dst_hi, src_mid, len_lo,
2802 wrp_reg_or_subpart(nfp_prog, dst_hi, src_hi, REG_WIDTH - len_lo,
2810 mem_ldx_data_from_pktcache_aligned(struct nfp_prog *nfp_prog,
2824 wrp_reg_subpart(nfp_prog, dst_lo, src_lo, size, 0);
2825 wrp_zext(nfp_prog, meta, dst_gpr);
2827 wrp_mov(nfp_prog, dst_lo, src_lo);
2828 wrp_zext(nfp_prog, meta, dst_gpr);
2832 wrp_mov(nfp_prog, dst_lo, src_lo);
2833 wrp_mov(nfp_prog, dst_hi, src_hi);
2840 mem_ldx_data_from_pktcache(struct nfp_prog *nfp_prog,
2846 return mem_ldx_data_from_pktcache_aligned(nfp_prog, meta, size);
2848 return mem_ldx_data_from_pktcache_unaligned(nfp_prog, meta, size);
2852 mem_ldx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2856 return nfp_cpp_memcpy(nfp_prog, meta);
2859 if (nfp_prog->type == BPF_PROG_TYPE_XDP)
2860 return mem_ldx_xdp(nfp_prog, meta, size);
2862 return mem_ldx_skb(nfp_prog, meta, size);
2868 mem_ldx_data_init_pktcache(nfp_prog, meta);
2870 return mem_ldx_data_from_pktcache(nfp_prog, meta, size);
2872 return mem_ldx_data(nfp_prog, meta, size);
2877 return mem_ldx_stack(nfp_prog, meta, size,
2881 return mem_ldx_emem(nfp_prog, meta, size);
2886 static int mem_ldx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2888 return mem_ldx(nfp_prog, meta, 1);
2891 static int mem_ldx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2893 return mem_ldx(nfp_prog, meta, 2);
2896 static int mem_ldx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2898 return mem_ldx(nfp_prog, meta, 4);
2901 static int mem_ldx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2903 return mem_ldx(nfp_prog, meta, 8);
2907 mem_st_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2913 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2915 return data_st_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
2919 static int mem_st(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2923 return mem_st_data(nfp_prog, meta, size);
2928 static int mem_st1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2930 return mem_st(nfp_prog, meta, 1);
2933 static int mem_st2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2935 return mem_st(nfp_prog, meta, 2);
2938 static int mem_st4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2940 return mem_st(nfp_prog, meta, 4);
2943 static int mem_st8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2945 return mem_st(nfp_prog, meta, 8);
2949 mem_stx_data(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2954 off_reg = re_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
2956 return data_stx_host_order(nfp_prog, meta->insn.dst_reg * 2, off_reg,
2961 mem_stx_stack(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2964 return mem_op_stack(nfp_prog, meta, size, ptr_off,
2969 static int mem_stx_xdp(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2973 return nfp_queue_select(nfp_prog, meta);
2981 mem_stx(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
2985 return mem_stx_data(nfp_prog, meta, size);
2988 return mem_stx_stack(nfp_prog, meta, size,
2994 static int mem_stx1(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
2996 return mem_stx(nfp_prog, meta, 1);
2999 static int mem_stx2(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3001 return mem_stx(nfp_prog, meta, 2);
3004 static int mem_stx4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3007 if (nfp_prog->type == BPF_PROG_TYPE_XDP)
3008 return mem_stx_xdp(nfp_prog, meta);
3009 return mem_stx(nfp_prog, meta, 4);
3012 static int mem_stx8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3014 return mem_stx(nfp_prog, meta, 8);
3018 mem_xadd(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta, bool is64)
3025 off = ur_load_imm_any(nfp_prog, meta->insn.off, imm_b(nfp_prog));
3032 out = nfp_prog_current_offset(nfp_prog);
3033 full_add = nfp_prog_current_offset(nfp_prog);
3052 swreg max_imm = imm_a(nfp_prog);
3054 wrp_immed(nfp_prog, max_imm, 0xffff);
3055 emit_alu(nfp_prog, reg_none(),
3057 emit_alu(nfp_prog, reg_none(),
3059 emit_br(nfp_prog, BR_BLO, full_add, meta->insn.off ? 2 : 0);
3068 emit_alu(nfp_prog, imma_a(nfp_prog),
3070 emit_alu(nfp_prog, imma_b(nfp_prog),
3072 addra = imma_a(nfp_prog);
3073 addrb = imma_b(nfp_prog);
3078 swreg prev_alu = imm_a(nfp_prog);
3080 wrp_immed(nfp_prog, prev_alu,
3084 wrp_reg_or_subpart(nfp_prog, prev_alu, reg_b(src_gpr), 2, 2);
3085 emit_cmd_indir(nfp_prog, CMD_TGT_ADD_IMM, CMD_MODE_40b_BA, 0,
3089 emit_br(nfp_prog, BR_UNC, out, 0);
3092 if (!nfp_prog_confirm_current_offset(nfp_prog, full_add))
3097 emit_cmd(nfp_prog, CMD_TGT_ADD, CMD_MODE_40b_BA, 0,
3101 wrp_mov(nfp_prog, reg_xfer(0), reg_a(src_gpr));
3103 wrp_mov(nfp_prog, reg_xfer(1), reg_a(src_gpr + 1));
3106 if (!nfp_prog_confirm_current_offset(nfp_prog, out))
3112 static int mem_atomic4(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3117 return mem_xadd(nfp_prog, meta, false);
3120 static int mem_atomic8(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3125 return mem_xadd(nfp_prog, meta, true);
3128 static int jump(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3130 emit_br(nfp_prog, BR_UNC, meta->insn.off, 0);
3135 static int jeq_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3145 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
3146 emit_alu(nfp_prog, imm_a(nfp_prog),
3148 or1 = imm_a(nfp_prog);
3152 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
3153 emit_alu(nfp_prog, imm_b(nfp_prog),
3155 or2 = imm_b(nfp_prog);
3158 emit_alu(nfp_prog, reg_none(), or1, ALU_OP_OR, or2);
3159 emit_br(nfp_prog, BR_BEQ, insn->off, 0);
3164 static int jeq32_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3169 tmp_reg = ur_load_imm_any(nfp_prog, insn->imm, imm_b(nfp_prog));
3170 emit_alu(nfp_prog, reg_none(),
3172 emit_br(nfp_prog, BR_BEQ, insn->off, 0);
3177 static int jset_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3184 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
3185 emit_alu(nfp_prog, imm_b(nfp_prog),
3191 emit_alu(nfp_prog, reg_none(),
3192 reg_a(dst_gpr + 1), ALU_OP_OR, imm_b(nfp_prog));
3194 emit_br(nfp_prog, BR_BNE, insn->off, 0);
3199 static int jne_imm(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3208 emit_alu(nfp_prog, reg_none(), reg_none(), ALU_OP_NONE,
3211 emit_alu(nfp_prog, reg_none(), reg_a(insn->dst_reg * 2),
3213 emit_br(nfp_prog, BR_BNE, insn->off, 0);
3217 tmp_reg = ur_load_imm_any(nfp_prog, imm & ~0U, imm_b(nfp_prog));
3218 emit_alu(nfp_prog, reg_none(),
3220 emit_br(nfp_prog, BR_BNE, insn->off, 0);
3225 tmp_reg = ur_load_imm_any(nfp_prog, imm >> 32, imm_b(nfp_prog));
3226 emit_alu(nfp_prog, reg_none(),
3228 emit_br(nfp_prog, BR_BNE, insn->off, 0);
3233 static int jeq_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3237 emit_alu(nfp_prog, imm_a(nfp_prog), reg_a(insn->dst_reg * 2),
3240 emit_alu(nfp_prog, imm_b(nfp_prog),
3243 emit_alu(nfp_prog, reg_none(), imm_a(nfp_prog), ALU_OP_OR,
3244 imm_b(nfp_prog));
3246 emit_br(nfp_prog, BR_BEQ, insn->off, 0);
3251 static int jset_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3253 return wrp_test_reg(nfp_prog, meta, ALU_OP_AND, BR_BNE);
3256 static int jne_reg(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3258 return wrp_test_reg(nfp_prog, meta, ALU_OP_XOR, BR_BNE);
3262 bpf_to_bpf_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3267 stack_depth = round_up(nfp_prog->stack_frame_depth, STACK_FRAME_ALIGN);
3272 tmp_reg = ur_load_imm_any(nfp_prog, stack_depth,
3273 stack_imm(nfp_prog));
3274 emit_alu(nfp_prog, stack_reg(nfp_prog),
3275 stack_reg(nfp_prog), ALU_OP_ADD, tmp_reg);
3276 emit_csr_wr(nfp_prog, stack_reg(nfp_prog),
3315 if (nfp_prog->subprog[meta->jmp_dst->subprog_idx].needs_reg_push) {
3316 ret_tgt = nfp_prog_current_offset(nfp_prog) + 3;
3317 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2,
3319 offset_br = nfp_prog_current_offset(nfp_prog);
3320 wrp_immed_relo(nfp_prog, imm_b(nfp_prog), 0, RELO_IMMED_REL);
3322 ret_tgt = nfp_prog_current_offset(nfp_prog) + 2;
3323 emit_br(nfp_prog, BR_UNC, meta->insn.imm, 1);
3324 offset_br = nfp_prog_current_offset(nfp_prog);
3326 wrp_immed_relo(nfp_prog, ret_reg(nfp_prog), ret_tgt, RELO_IMMED_REL);
3328 if (!nfp_prog_confirm_current_offset(nfp_prog, ret_tgt))
3332 tmp_reg = ur_load_imm_any(nfp_prog, stack_depth,
3333 stack_imm(nfp_prog));
3334 emit_alu(nfp_prog, stack_reg(nfp_prog),
3335 stack_reg(nfp_prog), ALU_OP_SUB, tmp_reg);
3336 emit_csr_wr(nfp_prog, stack_reg(nfp_prog),
3338 wrp_nops(nfp_prog, 3);
3341 meta->num_insns_after_br = nfp_prog_current_offset(nfp_prog);
3347 static int helper_call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3351 return adjust_head(nfp_prog, meta);
3353 return adjust_tail(nfp_prog, meta);
3357 return map_call_stack_common(nfp_prog, meta);
3359 return nfp_get_prandom_u32(nfp_prog, meta);
3361 return nfp_perf_event_output(nfp_prog, meta);
3368 static int call(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3371 return bpf_to_bpf_call(nfp_prog, meta);
3373 return helper_call(nfp_prog, meta);
3381 static int goto_out(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3383 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 0, RELO_BR_GO_OUT);
3389 nfp_subprog_epilogue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3391 if (nfp_prog->subprog[meta->subprog_idx].needs_reg_push) {
3397 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 1,
3400 wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0));
3403 wrp_mov(nfp_prog, ret_reg(nfp_prog), reg_lm(0, 0));
3407 emit_rtn(nfp_prog, ret_reg(nfp_prog), 0);
3413 static int jmp_exit(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3416 return goto_out(nfp_prog, meta);
3418 return nfp_subprog_epilogue(nfp_prog, meta);
3541 nfp_fixup_immed_relo(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta,
3544 if (immed_get_value(nfp_prog->prog[br_idx + 1])) {
3549 immed_set_value(&nfp_prog->prog[br_idx + 1], jmp_dst->off);
3554 static int nfp_fixup_branches(struct nfp_prog *nfp_prog)
3560 list_for_each_entry(meta, &nfp_prog->insns, l) {
3571 if (list_is_last(&meta->l, &nfp_prog->insns))
3572 br_idx = nfp_prog->last_bpf_off;
3584 if (!nfp_is_br(nfp_prog->prog[br_idx])) {
3586 br_idx, meta->insn.code, nfp_prog->prog[br_idx]);
3594 if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
3611 nfp_prog->subprog[jmp_dst->subprog_idx].needs_reg_push) {
3612 err = nfp_fixup_immed_relo(nfp_prog, meta,
3618 if (FIELD_GET(OP_RELO_TYPE, nfp_prog->prog[br_idx]) !=
3623 if (!nfp_is_br(nfp_prog->prog[idx]))
3625 br_set_offset(&nfp_prog->prog[idx], jmp_dst->off);
3632 static void nfp_intro(struct nfp_prog *nfp_prog)
3634 wrp_immed(nfp_prog, plen_reg(nfp_prog), GENMASK(13, 0));
3635 emit_alu(nfp_prog, plen_reg(nfp_prog),
3636 plen_reg(nfp_prog), ALU_OP_AND, pv_len(nfp_prog));
3640 nfp_subprog_prologue(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3643 wrp_mov(nfp_prog, reg_lm(0, 0), ret_reg(nfp_prog));
3647 nfp_start_subprog(struct nfp_prog *nfp_prog, struct nfp_insn_meta *meta)
3649 unsigned int depth = nfp_prog->subprog[meta->subprog_idx].stack_depth;
3651 nfp_prog->stack_frame_depth = round_up(depth, 4);
3652 nfp_subprog_prologue(nfp_prog, meta);
3660 static void nfp_outro_tc_da(struct nfp_prog *nfp_prog)
3675 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
3677 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
3679 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
3680 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x11), SHF_SC_L_SHF, 16);
3683 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
3686 emit_alu(nfp_prog, reg_none(), reg_imm(7), ALU_OP_SUB, reg_b(0));
3687 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
3688 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
3690 wrp_immed(nfp_prog, reg_b(2), 0x41221211);
3691 wrp_immed(nfp_prog, reg_b(3), 0x41001211);
3693 emit_shf(nfp_prog, reg_a(1),
3696 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
3697 emit_shf(nfp_prog, reg_a(2),
3700 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
3701 emit_shf(nfp_prog, reg_b(2),
3704 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
3706 emit_shf(nfp_prog, reg_b(2),
3708 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
3711 static void nfp_outro_xdp(struct nfp_prog *nfp_prog)
3721 nfp_prog->tgt_abort = nfp_prog_current_offset(nfp_prog);
3723 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
3725 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
3726 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_imm(0x82), SHF_SC_L_SHF, 16);
3729 nfp_prog->tgt_out = nfp_prog_current_offset(nfp_prog);
3732 emit_alu(nfp_prog, reg_none(), reg_imm(3), ALU_OP_SUB, reg_b(0));
3733 emit_br(nfp_prog, BR_BLO, nfp_prog->tgt_abort, 0);
3735 wrp_immed(nfp_prog, reg_b(2), 0x44112282);
3737 emit_shf(nfp_prog, reg_a(1),
3740 emit_alu(nfp_prog, reg_none(), reg_a(1), ALU_OP_OR, reg_imm(0));
3741 emit_shf(nfp_prog, reg_b(2),
3744 emit_br_relo(nfp_prog, BR_UNC, BR_OFF_RELO, 2, RELO_BR_NEXT_PKT);
3746 wrp_mov(nfp_prog, reg_a(0), NFP_BPF_ABI_FLAGS);
3747 emit_ld_field(nfp_prog, reg_a(0), 0xc, reg_b(2), SHF_SC_L_SHF, 16);
3750 static bool nfp_prog_needs_callee_reg_save(struct nfp_prog *nfp_prog)
3754 for (idx = 1; idx < nfp_prog->subprog_cnt; idx++)
3755 if (nfp_prog->subprog[idx].needs_reg_push)
3761 static void nfp_push_callee_registers(struct nfp_prog *nfp_prog)
3768 nfp_prog->tgt_call_push_regs = nfp_prog_current_offset(nfp_prog);
3776 wrp_mov(nfp_prog, reg_lm(0, 1 + idx), reg_b(adj));
3780 emit_rtn(nfp_prog, imm_b(nfp_prog), 3);
3782 wrp_mov(nfp_prog, reg_lm(0, 1 + idx + 1), reg_b(adj + 1));
3786 static void nfp_pop_callee_registers(struct nfp_prog *nfp_prog)
3793 nfp_prog->tgt_call_pop_regs = nfp_prog_current_offset(nfp_prog);
3801 wrp_mov(nfp_prog, reg_both(adj), reg_lm(0, 1 + idx));
3805 emit_rtn(nfp_prog, ret_reg(nfp_prog), 3);
3807 wrp_mov(nfp_prog, reg_both(adj + 1), reg_lm(0, 1 + idx + 1));
3811 static void nfp_outro(struct nfp_prog *nfp_prog)
3813 switch (nfp_prog->type) {
3815 nfp_outro_tc_da(nfp_prog);
3818 nfp_outro_xdp(nfp_prog);
3824 if (!nfp_prog_needs_callee_reg_save(nfp_prog))
3827 nfp_push_callee_registers(nfp_prog);
3828 nfp_pop_callee_registers(nfp_prog);
3831 static int nfp_translate(struct nfp_prog *nfp_prog)
3837 depth = nfp_prog->subprog[0].stack_depth;
3838 nfp_prog->stack_frame_depth = round_up(depth, 4);
3840 nfp_intro(nfp_prog);
3841 if (nfp_prog->error)
3842 return nfp_prog->error;
3844 list_for_each_entry(meta, &nfp_prog->insns, l) {
3847 meta->off = nfp_prog_current_offset(nfp_prog);
3850 nfp_start_subprog(nfp_prog, meta);
3851 if (nfp_prog->error)
3852 return nfp_prog->error;
3856 nfp_prog->n_translated++;
3860 if (nfp_meta_has_prev(nfp_prog, meta) &&
3865 err = cb(nfp_prog, meta);
3868 if (nfp_prog->error)
3869 return nfp_prog->error;
3871 nfp_prog->n_translated++;
3874 nfp_prog->last_bpf_off = nfp_prog_current_offset(nfp_prog) - 1;
3876 nfp_outro(nfp_prog);
3877 if (nfp_prog->error)
3878 return nfp_prog->error;
3880 wrp_nops(nfp_prog, NFP_USTORE_PREFETCH_WINDOW);
3881 if (nfp_prog->error)
3882 return nfp_prog->error;
3884 return nfp_fixup_branches(nfp_prog);
3888 static void nfp_bpf_opt_reg_init(struct nfp_prog *nfp_prog)
3892 list_for_each_entry(meta, &nfp_prog->insns, l) {
3914 static void nfp_bpf_opt_neg_add_sub(struct nfp_prog *nfp_prog)
3918 list_for_each_entry(meta, &nfp_prog->insns, l) {
3958 static void nfp_bpf_opt_ld_mask(struct nfp_prog *nfp_prog)
3967 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
3997 static void nfp_bpf_opt_ld_shift(struct nfp_prog *nfp_prog)
4001 nfp_for_each_insn_walk3(nfp_prog, meta1, meta2, meta3) {
4186 static void nfp_bpf_opt_ldst_gather(struct nfp_prog *nfp_prog)
4195 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
4276 static void nfp_bpf_opt_pkt_cache(struct nfp_prog *nfp_prog)
4285 list_for_each_entry(meta, &nfp_prog->insns, l) {
4374 list_for_each_entry(meta, &nfp_prog->insns, l) {
4390 static int nfp_bpf_optimize(struct nfp_prog *nfp_prog)
4392 nfp_bpf_opt_reg_init(nfp_prog);
4394 nfp_bpf_opt_neg_add_sub(nfp_prog);
4395 nfp_bpf_opt_ld_mask(nfp_prog);
4396 nfp_bpf_opt_ld_shift(nfp_prog);
4397 nfp_bpf_opt_ldst_gather(nfp_prog);
4398 nfp_bpf_opt_pkt_cache(nfp_prog);
4403 static int nfp_bpf_replace_map_ptrs(struct nfp_prog *nfp_prog)
4410 nfp_for_each_insn_walk2(nfp_prog, meta1, meta2) {
4453 static void nfp_bpf_prog_trim(struct nfp_prog *nfp_prog)
4457 prog = kvmalloc_array(nfp_prog->prog_len, sizeof(u64), GFP_KERNEL);
4461 nfp_prog->__prog_alloc_len = nfp_prog->prog_len * sizeof(u64);
4462 memcpy(prog, nfp_prog->prog, nfp_prog->__prog_alloc_len);
4463 kvfree(nfp_prog->prog);
4464 nfp_prog->prog = prog;
4467 int nfp_bpf_jit(struct nfp_prog *nfp_prog)
4471 ret = nfp_bpf_replace_map_ptrs(nfp_prog);
4475 ret = nfp_bpf_optimize(nfp_prog);
4479 ret = nfp_translate(nfp_prog);
4482 ret, nfp_prog->n_translated);
4486 nfp_bpf_prog_trim(nfp_prog);
4491 void nfp_bpf_jit_prepare(struct nfp_prog *nfp_prog)
4496 list_for_each_entry(meta, &nfp_prog->insns, l) {
4519 dst_meta = nfp_bpf_goto_meta(nfp_prog, meta, dst_idx);
4534 void *nfp_bpf_relo_for_vnic(struct nfp_prog *nfp_prog, struct nfp_bpf_vnic *bv)
4540 prog = kmemdup_array(nfp_prog->prog, nfp_prog->prog_len, sizeof(u64),
4545 for (i = 0; i < nfp_prog->prog_len; i++) {
4559 nfp_prog->tgt_out + bv->start_off);
4563 nfp_prog->tgt_abort + bv->start_off);
4566 if (!nfp_prog->tgt_call_push_regs) {
4571 off = nfp_prog->tgt_call_push_regs + bv->start_off;
4575 if (!nfp_prog->tgt_call_pop_regs) {
4580 off = nfp_prog->tgt_call_pop_regs + bv->start_off;
4591 val = nfp_prog->bpf->helpers.map_lookup;
4594 val = nfp_prog->bpf->helpers.map_update;
4597 val = nfp_prog->bpf->helpers.map_delete;
4600 val = nfp_prog->bpf->helpers.perf_event_output;
4618 err = nfp_bpf_ustore_calc(prog, nfp_prog->prog_len);