Lines Matching +full:0 +full:x40
64 do { prog = emit_code(prog, bytes, len); cnt += len; } while (0)
73 do { EMIT1(b1); EMIT(off, 4); } while (0)
75 do { EMIT2(b1, b2); EMIT(off, 4); } while (0)
77 do { EMIT3(b1, b2, b3); EMIT(off, 4); } while (0)
79 do { EMIT4(b1, b2, b3, b4); EMIT(off, 4); } while (0)
94 #define TCALL_CNT (MAX_BPF_JIT_REG + 0) /* Tail Call Count */
96 #define IA32_EAX (0x0)
97 #define IA32_EBX (0x3)
98 #define IA32_ECX (0x1)
99 #define IA32_EDX (0x2)
100 #define IA32_ESI (0x6)
101 #define IA32_EDI (0x7)
102 #define IA32_EBP (0x5)
103 #define IA32_ESP (0x4)
107 * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
109 #define IA32_JB 0x72
110 #define IA32_JAE 0x73
111 #define IA32_JE 0x74
112 #define IA32_JNE 0x75
113 #define IA32_JBE 0x76
114 #define IA32_JA 0x77
115 #define IA32_JL 0x7C
116 #define IA32_JGE 0x7D
117 #define IA32_JLE 0x7E
118 #define IA32_JG 0x7F
120 #define COND_JMP_OPCODE_INVALID (0xFF)
139 [BPF_REG_0] = {STACK_OFFSET(0), STACK_OFFSET(4)},
166 #define dst_lo dst[0]
168 #define src_lo src[0]
202 memset(area, 0xcc, size); in jit_fill_hole()
209 int cnt = 0; in emit_ia32_mov_i()
212 if (val == 0) { in emit_ia32_mov_i()
214 EMIT2(0x33, add_2reg(0xC0, IA32_EAX, IA32_EAX)); in emit_ia32_mov_i()
216 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mov_i()
219 EMIT3_off32(0xC7, add_1reg(0x40, IA32_EBP), in emit_ia32_mov_i()
223 if (val == 0) in emit_ia32_mov_i()
224 EMIT2(0x33, add_2reg(0xC0, dst, dst)); in emit_ia32_mov_i()
226 EMIT2_off32(0xC7, add_1reg(0xC0, dst), in emit_ia32_mov_i()
237 int cnt = 0; in emit_ia32_mov_r()
242 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src)); in emit_ia32_mov_r()
245 EMIT3(0x89, add_2reg(0x40, IA32_EBP, sreg), STACK_VAR(dst)); in emit_ia32_mov_r()
248 EMIT2(0x89, add_2reg(0xC0, dst, sreg)); in emit_ia32_mov_r()
265 emit_ia32_mov_i(dst_hi, 0, dstk, pprog); in emit_ia32_mov_r64()
272 u32 hi = 0; in emit_ia32_mov_i64()
275 hi = (u32)~0; in emit_ia32_mov_i64()
288 int cnt = 0; in emit_ia32_mul_r()
293 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src)); in emit_ia32_mul_r()
297 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst)); in emit_ia32_mul_r()
300 EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_mul_r()
303 EMIT2(0xF7, add_1reg(0xE0, sreg)); in emit_ia32_mul_r()
307 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r()
311 EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_mul_r()
321 int cnt = 0; in emit_ia32_to_le_r64()
326 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_to_le_r64()
328 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_to_le_r64()
337 EMIT2(0x0F, 0xB7); in emit_ia32_to_le_r64()
338 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_to_le_r64()
341 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_le_r64()
346 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_le_r64()
355 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_to_le_r64()
358 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_to_le_r64()
369 int cnt = 0; in emit_ia32_to_be_r64()
374 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_to_be_r64()
376 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_to_be_r64()
382 EMIT1(0x66); in emit_ia32_to_be_r64()
383 EMIT3(0xC1, add_1reg(0xC8, dreg_lo), 8); in emit_ia32_to_be_r64()
385 EMIT2(0x0F, 0xB7); in emit_ia32_to_be_r64()
386 EMIT1(add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_to_be_r64()
390 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_be_r64()
394 EMIT1(0x0F); in emit_ia32_to_be_r64()
395 EMIT1(add_1reg(0xC8, dreg_lo)); in emit_ia32_to_be_r64()
399 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_to_be_r64()
403 EMIT1(0x0F); in emit_ia32_to_be_r64()
404 EMIT1(add_1reg(0xC8, dreg_lo)); in emit_ia32_to_be_r64()
407 EMIT1(0x0F); in emit_ia32_to_be_r64()
408 EMIT1(add_1reg(0xC8, dreg_hi)); in emit_ia32_to_be_r64()
411 EMIT2(0x89, add_2reg(0xC0, IA32_ECX, dreg_hi)); in emit_ia32_to_be_r64()
413 EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_to_be_r64()
415 EMIT2(0x89, add_2reg(0xC0, dreg_lo, IA32_ECX)); in emit_ia32_to_be_r64()
421 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_to_be_r64()
424 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_to_be_r64()
438 int cnt = 0; in emit_ia32_div_mod_r()
442 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_div_mod_r()
446 EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX)); in emit_ia32_div_mod_r()
450 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_div_mod_r()
454 EMIT2(0x8B, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_div_mod_r()
457 EMIT2(0x31, add_2reg(0xC0, IA32_EDX, IA32_EDX)); in emit_ia32_div_mod_r()
459 EMIT2(0xF7, add_1reg(0xF0, IA32_ECX)); in emit_ia32_div_mod_r()
463 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_div_mod_r()
466 EMIT2(0x89, add_2reg(0xC0, dst, IA32_EDX)); in emit_ia32_div_mod_r()
469 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_div_mod_r()
472 EMIT2(0x89, add_2reg(0xC0, dst, IA32_EAX)); in emit_ia32_div_mod_r()
485 int cnt = 0; in emit_ia32_shift_r()
491 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst)); in emit_ia32_shift_r()
495 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src)); in emit_ia32_shift_r()
498 EMIT2(0x8B, add_2reg(0xC0, src, IA32_ECX)); in emit_ia32_shift_r()
502 b2 = 0xE0; break; in emit_ia32_shift_r()
504 b2 = 0xE8; break; in emit_ia32_shift_r()
506 b2 = 0xF8; break; in emit_ia32_shift_r()
510 EMIT2(0xD3, add_1reg(b2, dreg)); in emit_ia32_shift_r()
514 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), STACK_VAR(dst)); in emit_ia32_shift_r()
527 int cnt = 0; in emit_ia32_alu_r()
533 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(src)); in emit_ia32_alu_r()
537 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(dst)); in emit_ia32_alu_r()
543 EMIT2(0x11, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
545 EMIT2(0x01, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
550 EMIT2(0x19, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
552 EMIT2(0x29, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
556 EMIT2(0x09, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
560 EMIT2(0x21, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
564 EMIT2(0x31, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_r()
570 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), in emit_ia32_alu_r()
588 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in emit_ia32_alu_r64()
601 int cnt = 0; in emit_ia32_alu_i()
607 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(dst)); in emit_ia32_alu_i()
611 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EDX), val); in emit_ia32_alu_i()
618 EMIT3(0x83, add_1reg(0xD0, dreg), val); in emit_ia32_alu_i()
620 EMIT2(0x11, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
623 EMIT3(0x83, add_1reg(0xC0, dreg), val); in emit_ia32_alu_i()
625 EMIT2(0x01, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
632 EMIT3(0x83, add_1reg(0xD8, dreg), val); in emit_ia32_alu_i()
634 EMIT2(0x19, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
637 EMIT3(0x83, add_1reg(0xE8, dreg), val); in emit_ia32_alu_i()
639 EMIT2(0x29, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
645 EMIT3(0x83, add_1reg(0xC8, dreg), val); in emit_ia32_alu_i()
647 EMIT2(0x09, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
652 EMIT3(0x83, add_1reg(0xE0, dreg), val); in emit_ia32_alu_i()
654 EMIT2(0x21, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
659 EMIT3(0x83, add_1reg(0xF0, dreg), val); in emit_ia32_alu_i()
661 EMIT2(0x31, add_2reg(0xC0, dreg, sreg)); in emit_ia32_alu_i()
664 EMIT2(0xF7, add_1reg(0xD8, dreg)); in emit_ia32_alu_i()
670 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg), in emit_ia32_alu_i()
682 u32 hi = 0; in emit_ia32_alu_i64()
685 hi = (u32)~0; in emit_ia32_alu_i64()
691 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in emit_ia32_alu_i64()
700 int cnt = 0; in emit_ia32_neg64()
705 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_neg64()
707 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_neg64()
712 EMIT2(0xF7, add_1reg(0xD8, dreg_lo)); in emit_ia32_neg64()
713 /* adc dreg_hi,0x0 */ in emit_ia32_neg64()
714 EMIT3(0x83, add_1reg(0xD0, dreg_hi), 0x00); in emit_ia32_neg64()
716 EMIT2(0xF7, add_1reg(0xD8, dreg_hi)); in emit_ia32_neg64()
720 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_neg64()
723 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_neg64()
734 int cnt = 0; in emit_ia32_lsh_r64()
739 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_lsh_r64()
741 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_lsh_r64()
747 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_lsh_r64()
751 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX)); in emit_ia32_lsh_r64()
754 EMIT3(0x0F, 0xA5, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_lsh_r64()
756 EMIT2(0xD3, add_1reg(0xE0, dreg_lo)); in emit_ia32_lsh_r64()
761 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32); in emit_ia32_lsh_r64()
766 EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_lsh_r64()
768 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_lsh_r64()
772 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_lsh_r64()
775 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_lsh_r64()
787 int cnt = 0; in emit_ia32_arsh_r64()
792 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_arsh_r64()
794 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_arsh_r64()
800 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_arsh_r64()
804 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX)); in emit_ia32_arsh_r64()
807 EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_r64()
809 EMIT2(0xD3, add_1reg(0xF8, dreg_hi)); in emit_ia32_arsh_r64()
814 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32); in emit_ia32_arsh_r64()
819 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_r64()
821 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31); in emit_ia32_arsh_r64()
825 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_arsh_r64()
828 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_arsh_r64()
840 int cnt = 0; in emit_ia32_rsh_r64()
845 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_rsh_r64()
847 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_rsh_r64()
853 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_rsh_r64()
857 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_ECX)); in emit_ia32_rsh_r64()
860 EMIT3(0x0F, 0xAD, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_rsh_r64()
862 EMIT2(0xD3, add_1reg(0xE8, dreg_hi)); in emit_ia32_rsh_r64()
867 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), 32); in emit_ia32_rsh_r64()
872 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_rsh_r64()
874 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_rsh_r64()
878 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_rsh_r64()
881 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_rsh_r64()
893 int cnt = 0; in emit_ia32_lsh_i64()
898 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_lsh_i64()
900 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_lsh_i64()
906 EMIT4(0x0F, 0xA4, add_2reg(0xC0, dreg_hi, dreg_lo), val); in emit_ia32_lsh_i64()
908 EMIT3(0xC1, add_1reg(0xE0, dreg_lo), val); in emit_ia32_lsh_i64()
913 EMIT3(0xC1, add_1reg(0xE0, dreg_lo), value); in emit_ia32_lsh_i64()
915 EMIT2(0x89, add_2reg(0xC0, dreg_hi, dreg_lo)); in emit_ia32_lsh_i64()
917 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_lsh_i64()
920 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_lsh_i64()
922 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_lsh_i64()
927 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_lsh_i64()
930 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_lsh_i64()
941 int cnt = 0; in emit_ia32_rsh_i64()
946 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_rsh_i64()
948 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_rsh_i64()
955 EMIT4(0x0F, 0xAC, add_2reg(0xC0, dreg_lo, dreg_hi), val); in emit_ia32_rsh_i64()
957 EMIT3(0xC1, add_1reg(0xE8, dreg_hi), val); in emit_ia32_rsh_i64()
962 EMIT3(0xC1, add_1reg(0xE8, dreg_hi), value); in emit_ia32_rsh_i64()
964 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_rsh_i64()
966 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_rsh_i64()
969 EMIT2(0x33, add_2reg(0xC0, dreg_lo, dreg_lo)); in emit_ia32_rsh_i64()
971 EMIT2(0x33, add_2reg(0xC0, dreg_hi, dreg_hi)); in emit_ia32_rsh_i64()
976 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_rsh_i64()
979 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_rsh_i64()
990 int cnt = 0; in emit_ia32_arsh_i64()
995 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_arsh_i64()
997 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in emit_ia32_arsh_i64()
1003 EMIT4(0x0F, 0xAC, add_2reg(0xC0, dreg_lo, dreg_hi), val); in emit_ia32_arsh_i64()
1005 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), val); in emit_ia32_arsh_i64()
1010 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), value); in emit_ia32_arsh_i64()
1012 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_i64()
1015 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31); in emit_ia32_arsh_i64()
1018 EMIT3(0xC1, add_1reg(0xF8, dreg_hi), 31); in emit_ia32_arsh_i64()
1020 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dreg_hi)); in emit_ia32_arsh_i64()
1025 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_lo), in emit_ia32_arsh_i64()
1028 EMIT3(0x89, add_2reg(0x40, IA32_EBP, dreg_hi), in emit_ia32_arsh_i64()
1038 int cnt = 0; in emit_ia32_mul_r64()
1042 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1046 EMIT2(0x8B, add_2reg(0xC0, dst_hi, IA32_EAX)); in emit_ia32_mul_r64()
1050 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo)); in emit_ia32_mul_r64()
1053 EMIT2(0xF7, add_1reg(0xE0, src_lo)); in emit_ia32_mul_r64()
1056 EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_r64()
1060 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1064 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_r64()
1068 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_hi)); in emit_ia32_mul_r64()
1071 EMIT2(0xF7, add_1reg(0xE0, src_hi)); in emit_ia32_mul_r64()
1074 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_r64()
1078 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1082 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_r64()
1086 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(src_lo)); in emit_ia32_mul_r64()
1089 EMIT2(0xF7, add_1reg(0xE0, src_lo)); in emit_ia32_mul_r64()
1092 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX)); in emit_ia32_mul_r64()
1096 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_r64()
1099 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_mul_r64()
1103 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_r64()
1105 EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX)); in emit_ia32_mul_r64()
1115 int cnt = 0; in emit_ia32_mul_i64()
1118 hi = val & (1<<31) ? (u32)~0 : 0; in emit_ia32_mul_i64()
1120 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val); in emit_ia32_mul_i64()
1123 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_hi)); in emit_ia32_mul_i64()
1126 EMIT2(0xF7, add_1reg(0xE0, dst_hi)); in emit_ia32_mul_i64()
1129 EMIT2(0x89, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_i64()
1132 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), hi); in emit_ia32_mul_i64()
1135 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo)); in emit_ia32_mul_i64()
1138 EMIT2(0xF7, add_1reg(0xE0, dst_lo)); in emit_ia32_mul_i64()
1140 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EAX)); in emit_ia32_mul_i64()
1143 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EAX), val); in emit_ia32_mul_i64()
1146 EMIT3(0xF7, add_1reg(0x60, IA32_EBP), STACK_VAR(dst_lo)); in emit_ia32_mul_i64()
1149 EMIT2(0xF7, add_1reg(0xE0, dst_lo)); in emit_ia32_mul_i64()
1152 EMIT2(0x01, add_2reg(0xC0, IA32_ECX, IA32_EDX)); in emit_ia32_mul_i64()
1156 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in emit_ia32_mul_i64()
1159 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), in emit_ia32_mul_i64()
1163 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EAX)); in emit_ia32_mul_i64()
1165 EMIT2(0x89, add_2reg(0xC0, dst_hi, IA32_ECX)); in emit_ia32_mul_i64()
1182 return 0; in bpf_size_to_x86_bytes()
1202 int cnt = 0; in emit_prologue()
1204 const u8 fplo = bpf2ia32[BPF_REG_FP][0]; in emit_prologue()
1209 EMIT1(0x55); in emit_prologue()
1211 EMIT2(0x89, 0xE5); in emit_prologue()
1213 EMIT1(0x57); in emit_prologue()
1215 EMIT1(0x56); in emit_prologue()
1217 EMIT1(0x53); in emit_prologue()
1220 EMIT2_off32(0x81, 0xEC, STACK_SIZE); in emit_prologue()
1222 EMIT3(0x83, add_1reg(0xE8, IA32_EBP), SCRATCH_SIZE + 12); in emit_prologue()
1224 EMIT2(0x31, add_2reg(0xC0, IA32_EBX, IA32_EBX)); in emit_prologue()
1227 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBP), STACK_VAR(fplo)); in emit_prologue()
1228 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(fphi)); in emit_prologue()
1232 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0])); in emit_prologue()
1233 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(r1[1])); in emit_prologue()
1236 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[0])); in emit_prologue()
1237 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1])); in emit_prologue()
1248 int cnt = 0; in emit_epilogue()
1251 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r0[0])); in emit_epilogue()
1253 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r0[1])); in emit_epilogue()
1256 EMIT3(0x83, add_1reg(0xC0, IA32_EBP), SCRATCH_SIZE + 12); in emit_epilogue()
1259 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), -12); in emit_epilogue()
1261 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ESI), -8); in emit_epilogue()
1263 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDI), -4); in emit_epilogue()
1265 EMIT1(0xC9); /* leave */ in emit_epilogue()
1266 EMIT1(0xC3); /* ret */ in emit_epilogue()
1286 int cnt = 0; in emit_bpf_tail_call()
1299 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r2[0])); in emit_bpf_tail_call()
1301 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), STACK_VAR(r3[0])); in emit_bpf_tail_call()
1304 EMIT3(0x39, add_2reg(0x40, IA32_EAX, IA32_EDX), in emit_bpf_tail_call()
1315 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0])); in emit_bpf_tail_call()
1316 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1])); in emit_bpf_tail_call()
1319 EMIT3(0x83, add_1reg(0xF8, IA32_EBX), hi); in emit_bpf_tail_call()
1322 EMIT3(0x83, add_1reg(0xF8, IA32_ECX), lo); in emit_bpf_tail_call()
1327 /* add eax,0x1 */ in emit_bpf_tail_call()
1328 EMIT3(0x83, add_1reg(0xC0, IA32_ECX), 0x01); in emit_bpf_tail_call()
1329 /* adc ebx,0x0 */ in emit_bpf_tail_call()
1330 EMIT3(0x83, add_1reg(0xD0, IA32_EBX), 0x00); in emit_bpf_tail_call()
1333 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(tcc[0])); in emit_bpf_tail_call()
1335 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EBX), STACK_VAR(tcc[1])); in emit_bpf_tail_call()
1339 EMIT3_off32(0x8B, 0x94, 0x90, offsetof(struct bpf_array, ptrs)); in emit_bpf_tail_call()
1346 EMIT2(0x85, add_2reg(0xC0, IA32_EDX, IA32_EDX)); in emit_bpf_tail_call()
1352 EMIT3(0x8B, add_2reg(0x40, IA32_EDX, IA32_EDX), in emit_bpf_tail_call()
1355 EMIT3(0x83, add_1reg(0xC0, IA32_EDX), PROLOGUE_SIZE); in emit_bpf_tail_call()
1358 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), STACK_VAR(r1[0])); in emit_bpf_tail_call()
1378 int cnt = 0; in emit_push_r64()
1381 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_hi)); in emit_push_r64()
1383 EMIT1(0x51); in emit_push_r64()
1386 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), STACK_VAR(src_lo)); in emit_push_r64()
1388 EMIT1(0x51); in emit_push_r64()
1469 int i, cnt = 0; in do_jit()
1470 int proglen = 0; in do_jit()
1475 for (i = 0; i < insn_cnt; i++, insn++) { in do_jit()
1500 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1563 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), in do_jit()
1570 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1583 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), in do_jit()
1591 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1606 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), in do_jit()
1614 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1628 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); in do_jit()
1632 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1667 dst_lo, 0, dstk, &prog); in do_jit()
1669 emit_ia32_mov_i(dst_hi, 0, dstk, &prog); in do_jit()
1715 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1719 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in do_jit()
1723 EMIT(0xC6, 1); break; in do_jit()
1725 EMIT2(0x66, 0xC7); break; in do_jit()
1728 EMIT(0xC7, 1); break; in do_jit()
1732 EMIT2(add_1reg(0x40, IA32_EAX), insn->off); in do_jit()
1734 EMIT1_off32(add_1reg(0x80, IA32_EAX), in do_jit()
1741 hi = imm32 & (1<<31) ? (u32)~0 : 0; in do_jit()
1742 EMIT2_off32(0xC7, add_1reg(0x80, IA32_EAX), in do_jit()
1755 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1759 EMIT2(0x8B, add_2reg(0xC0, dst_lo, IA32_EAX)); in do_jit()
1763 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
1767 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EDX)); in do_jit()
1771 EMIT(0x88, 1); break; in do_jit()
1773 EMIT2(0x66, 0x89); break; in do_jit()
1776 EMIT(0x89, 1); break; in do_jit()
1780 EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX), in do_jit()
1783 EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX), in do_jit()
1789 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, in do_jit()
1794 EMIT2(0x8B, add_2reg(0xC0, src_hi, in do_jit()
1796 EMIT1(0x89); in do_jit()
1798 EMIT2(add_2reg(0x40, IA32_EAX, in do_jit()
1802 EMIT1(add_2reg(0x80, IA32_EAX, in do_jit()
1816 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1820 EMIT2(0x8B, add_2reg(0xC0, src_lo, IA32_EAX)); in do_jit()
1824 EMIT2(0x0F, 0xB6); break; in do_jit()
1826 EMIT2(0x0F, 0xB7); break; in do_jit()
1829 EMIT(0x8B, 1); break; in do_jit()
1833 EMIT2(add_2reg(0x40, IA32_EAX, IA32_EDX), in do_jit()
1836 EMIT1_off32(add_2reg(0x80, IA32_EAX, IA32_EDX), in do_jit()
1841 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
1845 EMIT2(0x89, add_2reg(0xC0, dst_lo, IA32_EDX)); in do_jit()
1853 EMIT3(0xC7, add_1reg(0x40, IA32_EBP), in do_jit()
1855 EMIT(0x0, 4); in do_jit()
1858 EMIT2(0x33, in do_jit()
1859 add_2reg(0xC0, dst_hi, dst_hi)); in do_jit()
1863 EMIT2_off32(0x8B, in do_jit()
1864 add_2reg(0x80, IA32_EAX, IA32_EDX), in do_jit()
1867 EMIT3(0x89, in do_jit()
1868 add_2reg(0x40, IA32_EBP, in do_jit()
1872 EMIT2(0x89, in do_jit()
1873 add_2reg(0xC0, dst_hi, IA32_EDX)); in do_jit()
1901 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1902 STACK_VAR(r1[0])); in do_jit()
1904 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
1912 EMIT1_off32(0xE8, jmp_offset + 9); in do_jit()
1915 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1916 STACK_VAR(r0[0])); in do_jit()
1918 EMIT3(0x89, add_2reg(0x40, IA32_EBP, IA32_EDX), in do_jit()
1922 EMIT3(0x83, add_1reg(0xC0, IA32_ESP), 32); in do_jit()
1953 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1956 EMIT3(0x8B, in do_jit()
1957 add_2reg(0x40, IA32_EBP, in do_jit()
1963 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in do_jit()
1966 EMIT3(0x8B, in do_jit()
1967 add_2reg(0x40, IA32_EBP, in do_jit()
1974 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
1978 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
1991 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
1993 EMIT3(0x8B, in do_jit()
1994 add_2reg(0x40, IA32_EBP, in do_jit()
2000 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in do_jit()
2002 EMIT3(0x8B, in do_jit()
2003 add_2reg(0x40, IA32_EBP, in do_jit()
2009 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2012 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2024 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2027 EMIT3(0x8B, in do_jit()
2028 add_2reg(0x40, IA32_EBP, in do_jit()
2033 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo)); in do_jit()
2036 EMIT2(0x89, in do_jit()
2037 add_2reg(0xC0, dreg_hi, dst_hi)); in do_jit()
2041 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_ECX), in do_jit()
2044 EMIT3(0x8B, in do_jit()
2045 add_2reg(0x40, IA32_EBP, in do_jit()
2050 EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo)); in do_jit()
2053 EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi)); in do_jit()
2055 EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi)); in do_jit()
2069 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2072 EMIT3(0x8B, in do_jit()
2073 add_2reg(0x40, IA32_EBP, in do_jit()
2078 EMIT2(0x89, add_2reg(0xC0, dreg_lo, dst_lo)); in do_jit()
2081 EMIT2(0x89, in do_jit()
2082 add_2reg(0xC0, dreg_hi, dst_hi)); in do_jit()
2086 EMIT2_off32(0xC7, add_1reg(0xC0, sreg_lo), imm32); in do_jit()
2089 EMIT2(0x23, add_2reg(0xC0, sreg_lo, dreg_lo)); in do_jit()
2091 hi = imm32 & (1 << 31) ? (u32)~0 : 0; in do_jit()
2093 EMIT2_off32(0xC7, add_1reg(0xC0, sreg_hi), hi); in do_jit()
2095 EMIT2(0x23, add_2reg(0xC0, sreg_hi, dreg_hi)); in do_jit()
2097 EMIT2(0x09, add_2reg(0xC0, dreg_lo, dreg_hi)); in do_jit()
2125 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2128 EMIT3(0x8B, in do_jit()
2129 add_2reg(0x40, IA32_EBP, in do_jit()
2135 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); in do_jit()
2137 hi = imm32 & (1 << 31) ? (u32)~0 : 0; in do_jit()
2139 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi); in do_jit()
2141 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2145 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2154 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2172 EMIT3(0x8B, add_2reg(0x40, IA32_EBP, IA32_EAX), in do_jit()
2174 EMIT3(0x8B, in do_jit()
2175 add_2reg(0x40, IA32_EBP, in do_jit()
2181 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_ECX), imm32); in do_jit()
2182 hi = imm32 & (1 << 31) ? (u32)~0 : 0; in do_jit()
2184 EMIT2_off32(0xC7, add_1reg(0xC0, IA32_EBX), hi); in do_jit()
2186 EMIT2(0x39, add_2reg(0xC0, dreg_hi, sreg_hi)); in do_jit()
2189 EMIT2(0x39, add_2reg(0xC0, dreg_lo, sreg_lo)); in do_jit()
2201 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2206 EMIT2(0xEB, 6); in do_jit()
2214 EMIT2_off32(0x0F, jmp_cond + 0x10, jmp_offset); in do_jit()
2238 EMIT2(0xEB, jmp_offset); in do_jit()
2240 EMIT1_off32(0xE9, jmp_offset); in do_jit()
2303 int proglen, oldproglen = 0; in bpf_int_jit_compile()
2336 for (proglen = 0, i = 0; i < prog->len; i++) { in bpf_int_jit_compile()
2348 for (pass = 0; pass < 20 || image; pass++) { in bpf_int_jit_compile()
2350 if (proglen <= 0) { in bpf_int_jit_compile()