1 /* bpf_jit_comp.c : BPF JIT compiler
2  *
3  * Copyright (C) 2011 Eric Dumazet (eric.dumazet@gmail.com)
4  *
5  * This program is free software; you can redistribute it and/or
6  * modify it under the terms of the GNU General Public License
7  * as published by the Free Software Foundation; version 2
8  * of the License.
9  */
10 #include <linux/moduleloader.h>
11 #include <asm/cacheflush.h>
12 #include <linux/netdevice.h>
13 #include <linux/filter.h>
14 
15 /*
16  * Conventions :
17  *  EAX : BPF A accumulator
18  *  EBX : BPF X accumulator
19  *  RDI : pointer to skb   (first argument given to JIT function)
20  *  RBP : frame pointer (even if CONFIG_FRAME_POINTER=n)
21  *  ECX,EDX,ESI : scratch registers
22  *  r9d : skb->len - skb->data_len (headlen)
23  *  r8  : skb->data
24  * -8(RBP) : saved RBX value
25  * -16(RBP)..-80(RBP) : BPF_MEMWORDS values
26  */
27 int bpf_jit_enable __read_mostly;
28 
29 /*
30  * assembly code in arch/x86/net/bpf_jit.S
31  */
32 extern u8 sk_load_word[], sk_load_half[], sk_load_byte[], sk_load_byte_msh[];
33 extern u8 sk_load_word_ind[], sk_load_half_ind[], sk_load_byte_ind[];
34 
emit_code(u8 * ptr,u32 bytes,unsigned int len)35 static inline u8 *emit_code(u8 *ptr, u32 bytes, unsigned int len)
36 {
37 	if (len == 1)
38 		*ptr = bytes;
39 	else if (len == 2)
40 		*(u16 *)ptr = bytes;
41 	else {
42 		*(u32 *)ptr = bytes;
43 		barrier();
44 	}
45 	return ptr + len;
46 }
47 
48 #define EMIT(bytes, len)	do { prog = emit_code(prog, bytes, len); } while (0)
49 
50 #define EMIT1(b1)		EMIT(b1, 1)
51 #define EMIT2(b1, b2)		EMIT((b1) + ((b2) << 8), 2)
52 #define EMIT3(b1, b2, b3)	EMIT((b1) + ((b2) << 8) + ((b3) << 16), 3)
53 #define EMIT4(b1, b2, b3, b4)   EMIT((b1) + ((b2) << 8) + ((b3) << 16) + ((b4) << 24), 4)
54 #define EMIT1_off32(b1, off)	do { EMIT1(b1); EMIT(off, 4);} while (0)
55 
56 #define CLEAR_A() EMIT2(0x31, 0xc0) /* xor %eax,%eax */
57 #define CLEAR_X() EMIT2(0x31, 0xdb) /* xor %ebx,%ebx */
58 
is_imm8(int value)59 static inline bool is_imm8(int value)
60 {
61 	return value <= 127 && value >= -128;
62 }
63 
is_near(int offset)64 static inline bool is_near(int offset)
65 {
66 	return offset <= 127 && offset >= -128;
67 }
68 
69 #define EMIT_JMP(offset)						\
70 do {									\
71 	if (offset) {							\
72 		if (is_near(offset))					\
73 			EMIT2(0xeb, offset); /* jmp .+off8 */		\
74 		else							\
75 			EMIT1_off32(0xe9, offset); /* jmp .+off32 */	\
76 	}								\
77 } while (0)
78 
79 /* list of x86 cond jumps opcodes (. + s8)
80  * Add 0x10 (and an extra 0x0f) to generate far jumps (. + s32)
81  */
82 #define X86_JB  0x72
83 #define X86_JAE 0x73
84 #define X86_JE  0x74
85 #define X86_JNE 0x75
86 #define X86_JBE 0x76
87 #define X86_JA  0x77
88 
89 #define EMIT_COND_JMP(op, offset)				\
90 do {								\
91 	if (is_near(offset))					\
92 		EMIT2(op, offset); /* jxx .+off8 */		\
93 	else {							\
94 		EMIT2(0x0f, op + 0x10);				\
95 		EMIT(offset, 4); /* jxx .+off32 */		\
96 	}							\
97 } while (0)
98 
99 #define COND_SEL(CODE, TOP, FOP)	\
100 	case CODE:			\
101 		t_op = TOP;		\
102 		f_op = FOP;		\
103 		goto cond_branch
104 
105 
106 #define SEEN_DATAREF 1 /* might call external helpers */
107 #define SEEN_XREG    2 /* ebx is used */
108 #define SEEN_MEM     4 /* use mem[] for temporary storage */
109 
bpf_flush_icache(void * start,void * end)110 static inline void bpf_flush_icache(void *start, void *end)
111 {
112 	mm_segment_t old_fs = get_fs();
113 
114 	set_fs(KERNEL_DS);
115 	smp_wmb();
116 	flush_icache_range((unsigned long)start, (unsigned long)end);
117 	set_fs(old_fs);
118 }
119 
120 
bpf_jit_compile(struct sk_filter * fp)121 void bpf_jit_compile(struct sk_filter *fp)
122 {
123 	u8 temp[64];
124 	u8 *prog;
125 	unsigned int proglen, oldproglen = 0;
126 	int ilen, i;
127 	int t_offset, f_offset;
128 	u8 t_op, f_op, seen = 0, pass;
129 	u8 *image = NULL;
130 	u8 *func;
131 	int pc_ret0 = -1; /* bpf index of first RET #0 instruction (if any) */
132 	unsigned int cleanup_addr; /* epilogue code offset */
133 	unsigned int *addrs;
134 	const struct sock_filter *filter = fp->insns;
135 	int flen = fp->len;
136 
137 	if (!bpf_jit_enable)
138 		return;
139 
140 	addrs = kmalloc(flen * sizeof(*addrs), GFP_KERNEL);
141 	if (addrs == NULL)
142 		return;
143 
144 	/* Before first pass, make a rough estimation of addrs[]
145 	 * each bpf instruction is translated to less than 64 bytes
146 	 */
147 	for (proglen = 0, i = 0; i < flen; i++) {
148 		proglen += 64;
149 		addrs[i] = proglen;
150 	}
151 	cleanup_addr = proglen; /* epilogue address */
152 
153 	for (pass = 0; pass < 10; pass++) {
154 		u8 seen_or_pass0 = (pass == 0) ? (SEEN_XREG | SEEN_DATAREF | SEEN_MEM) : seen;
155 		/* no prologue/epilogue for trivial filters (RET something) */
156 		proglen = 0;
157 		prog = temp;
158 
159 		if (seen_or_pass0) {
160 			EMIT4(0x55, 0x48, 0x89, 0xe5); /* push %rbp; mov %rsp,%rbp */
161 			EMIT4(0x48, 0x83, 0xec, 96);	/* subq  $96,%rsp	*/
162 			/* note : must save %rbx in case bpf_error is hit */
163 			if (seen_or_pass0 & (SEEN_XREG | SEEN_DATAREF))
164 				EMIT4(0x48, 0x89, 0x5d, 0xf8); /* mov %rbx, -8(%rbp) */
165 			if (seen_or_pass0 & SEEN_XREG)
166 				CLEAR_X(); /* make sure we dont leek kernel memory */
167 
168 			/*
169 			 * If this filter needs to access skb data,
170 			 * loads r9 and r8 with :
171 			 *  r9 = skb->len - skb->data_len
172 			 *  r8 = skb->data
173 			 */
174 			if (seen_or_pass0 & SEEN_DATAREF) {
175 				if (offsetof(struct sk_buff, len) <= 127)
176 					/* mov    off8(%rdi),%r9d */
177 					EMIT4(0x44, 0x8b, 0x4f, offsetof(struct sk_buff, len));
178 				else {
179 					/* mov    off32(%rdi),%r9d */
180 					EMIT3(0x44, 0x8b, 0x8f);
181 					EMIT(offsetof(struct sk_buff, len), 4);
182 				}
183 				if (is_imm8(offsetof(struct sk_buff, data_len)))
184 					/* sub    off8(%rdi),%r9d */
185 					EMIT4(0x44, 0x2b, 0x4f, offsetof(struct sk_buff, data_len));
186 				else {
187 					EMIT3(0x44, 0x2b, 0x8f);
188 					EMIT(offsetof(struct sk_buff, data_len), 4);
189 				}
190 
191 				if (is_imm8(offsetof(struct sk_buff, data)))
192 					/* mov off8(%rdi),%r8 */
193 					EMIT4(0x4c, 0x8b, 0x47, offsetof(struct sk_buff, data));
194 				else {
195 					/* mov off32(%rdi),%r8 */
196 					EMIT3(0x4c, 0x8b, 0x87);
197 					EMIT(offsetof(struct sk_buff, data), 4);
198 				}
199 			}
200 		}
201 
202 		switch (filter[0].code) {
203 		case BPF_S_RET_K:
204 		case BPF_S_LD_W_LEN:
205 		case BPF_S_ANC_PROTOCOL:
206 		case BPF_S_ANC_IFINDEX:
207 		case BPF_S_ANC_MARK:
208 		case BPF_S_ANC_RXHASH:
209 		case BPF_S_ANC_CPU:
210 		case BPF_S_ANC_QUEUE:
211 		case BPF_S_LD_W_ABS:
212 		case BPF_S_LD_H_ABS:
213 		case BPF_S_LD_B_ABS:
214 			/* first instruction sets A register (or is RET 'constant') */
215 			break;
216 		default:
217 			/* make sure we dont leak kernel information to user */
218 			CLEAR_A(); /* A = 0 */
219 		}
220 
221 		for (i = 0; i < flen; i++) {
222 			unsigned int K = filter[i].k;
223 
224 			switch (filter[i].code) {
225 			case BPF_S_ALU_ADD_X: /* A += X; */
226 				seen |= SEEN_XREG;
227 				EMIT2(0x01, 0xd8);		/* add %ebx,%eax */
228 				break;
229 			case BPF_S_ALU_ADD_K: /* A += K; */
230 				if (!K)
231 					break;
232 				if (is_imm8(K))
233 					EMIT3(0x83, 0xc0, K);	/* add imm8,%eax */
234 				else
235 					EMIT1_off32(0x05, K);	/* add imm32,%eax */
236 				break;
237 			case BPF_S_ALU_SUB_X: /* A -= X; */
238 				seen |= SEEN_XREG;
239 				EMIT2(0x29, 0xd8);		/* sub    %ebx,%eax */
240 				break;
241 			case BPF_S_ALU_SUB_K: /* A -= K */
242 				if (!K)
243 					break;
244 				if (is_imm8(K))
245 					EMIT3(0x83, 0xe8, K); /* sub imm8,%eax */
246 				else
247 					EMIT1_off32(0x2d, K); /* sub imm32,%eax */
248 				break;
249 			case BPF_S_ALU_MUL_X: /* A *= X; */
250 				seen |= SEEN_XREG;
251 				EMIT3(0x0f, 0xaf, 0xc3);	/* imul %ebx,%eax */
252 				break;
253 			case BPF_S_ALU_MUL_K: /* A *= K */
254 				if (is_imm8(K))
255 					EMIT3(0x6b, 0xc0, K); /* imul imm8,%eax,%eax */
256 				else {
257 					EMIT2(0x69, 0xc0);		/* imul imm32,%eax */
258 					EMIT(K, 4);
259 				}
260 				break;
261 			case BPF_S_ALU_DIV_X: /* A /= X; */
262 				seen |= SEEN_XREG;
263 				EMIT2(0x85, 0xdb);	/* test %ebx,%ebx */
264 				if (pc_ret0 > 0) {
265 					/* addrs[pc_ret0 - 1] is start address of target
266 					 * (addrs[i] - 4) is the address following this jmp
267 					 * ("xor %edx,%edx; div %ebx" being 4 bytes long)
268 					 */
269 					EMIT_COND_JMP(X86_JE, addrs[pc_ret0 - 1] -
270 								(addrs[i] - 4));
271 				} else {
272 					EMIT_COND_JMP(X86_JNE, 2 + 5);
273 					CLEAR_A();
274 					EMIT1_off32(0xe9, cleanup_addr - (addrs[i] - 4)); /* jmp .+off32 */
275 				}
276 				EMIT4(0x31, 0xd2, 0xf7, 0xf3); /* xor %edx,%edx; div %ebx */
277 				break;
278 			case BPF_S_ALU_DIV_K: /* A = reciprocal_divide(A, K); */
279 				EMIT3(0x48, 0x69, 0xc0); /* imul imm32,%rax,%rax */
280 				EMIT(K, 4);
281 				EMIT4(0x48, 0xc1, 0xe8, 0x20); /* shr $0x20,%rax */
282 				break;
283 			case BPF_S_ALU_AND_X:
284 				seen |= SEEN_XREG;
285 				EMIT2(0x21, 0xd8);		/* and %ebx,%eax */
286 				break;
287 			case BPF_S_ALU_AND_K:
288 				if (K >= 0xFFFFFF00) {
289 					EMIT2(0x24, K & 0xFF); /* and imm8,%al */
290 				} else if (K >= 0xFFFF0000) {
291 					EMIT2(0x66, 0x25);	/* and imm16,%ax */
292 					EMIT2(K, 2);
293 				} else {
294 					EMIT1_off32(0x25, K);	/* and imm32,%eax */
295 				}
296 				break;
297 			case BPF_S_ALU_OR_X:
298 				seen |= SEEN_XREG;
299 				EMIT2(0x09, 0xd8);		/* or %ebx,%eax */
300 				break;
301 			case BPF_S_ALU_OR_K:
302 				if (is_imm8(K))
303 					EMIT3(0x83, 0xc8, K); /* or imm8,%eax */
304 				else
305 					EMIT1_off32(0x0d, K);	/* or imm32,%eax */
306 				break;
307 			case BPF_S_ALU_LSH_X: /* A <<= X; */
308 				seen |= SEEN_XREG;
309 				EMIT4(0x89, 0xd9, 0xd3, 0xe0);	/* mov %ebx,%ecx; shl %cl,%eax */
310 				break;
311 			case BPF_S_ALU_LSH_K:
312 				if (K == 0)
313 					break;
314 				else if (K == 1)
315 					EMIT2(0xd1, 0xe0); /* shl %eax */
316 				else
317 					EMIT3(0xc1, 0xe0, K);
318 				break;
319 			case BPF_S_ALU_RSH_X: /* A >>= X; */
320 				seen |= SEEN_XREG;
321 				EMIT4(0x89, 0xd9, 0xd3, 0xe8);	/* mov %ebx,%ecx; shr %cl,%eax */
322 				break;
323 			case BPF_S_ALU_RSH_K: /* A >>= K; */
324 				if (K == 0)
325 					break;
326 				else if (K == 1)
327 					EMIT2(0xd1, 0xe8); /* shr %eax */
328 				else
329 					EMIT3(0xc1, 0xe8, K);
330 				break;
331 			case BPF_S_ALU_NEG:
332 				EMIT2(0xf7, 0xd8);		/* neg %eax */
333 				break;
334 			case BPF_S_RET_K:
335 				if (!K) {
336 					if (pc_ret0 == -1)
337 						pc_ret0 = i;
338 					CLEAR_A();
339 				} else {
340 					EMIT1_off32(0xb8, K);	/* mov $imm32,%eax */
341 				}
342 				/* fallinto */
343 			case BPF_S_RET_A:
344 				if (seen_or_pass0) {
345 					if (i != flen - 1) {
346 						EMIT_JMP(cleanup_addr - addrs[i]);
347 						break;
348 					}
349 					if (seen_or_pass0 & SEEN_XREG)
350 						EMIT4(0x48, 0x8b, 0x5d, 0xf8);  /* mov  -8(%rbp),%rbx */
351 					EMIT1(0xc9);		/* leaveq */
352 				}
353 				EMIT1(0xc3);		/* ret */
354 				break;
355 			case BPF_S_MISC_TAX: /* X = A */
356 				seen |= SEEN_XREG;
357 				EMIT2(0x89, 0xc3);	/* mov    %eax,%ebx */
358 				break;
359 			case BPF_S_MISC_TXA: /* A = X */
360 				seen |= SEEN_XREG;
361 				EMIT2(0x89, 0xd8);	/* mov    %ebx,%eax */
362 				break;
363 			case BPF_S_LD_IMM: /* A = K */
364 				if (!K)
365 					CLEAR_A();
366 				else
367 					EMIT1_off32(0xb8, K); /* mov $imm32,%eax */
368 				break;
369 			case BPF_S_LDX_IMM: /* X = K */
370 				seen |= SEEN_XREG;
371 				if (!K)
372 					CLEAR_X();
373 				else
374 					EMIT1_off32(0xbb, K); /* mov $imm32,%ebx */
375 				break;
376 			case BPF_S_LD_MEM: /* A = mem[K] : mov off8(%rbp),%eax */
377 				seen |= SEEN_MEM;
378 				EMIT3(0x8b, 0x45, 0xf0 - K*4);
379 				break;
380 			case BPF_S_LDX_MEM: /* X = mem[K] : mov off8(%rbp),%ebx */
381 				seen |= SEEN_XREG | SEEN_MEM;
382 				EMIT3(0x8b, 0x5d, 0xf0 - K*4);
383 				break;
384 			case BPF_S_ST: /* mem[K] = A : mov %eax,off8(%rbp) */
385 				seen |= SEEN_MEM;
386 				EMIT3(0x89, 0x45, 0xf0 - K*4);
387 				break;
388 			case BPF_S_STX: /* mem[K] = X : mov %ebx,off8(%rbp) */
389 				seen |= SEEN_XREG | SEEN_MEM;
390 				EMIT3(0x89, 0x5d, 0xf0 - K*4);
391 				break;
392 			case BPF_S_LD_W_LEN: /*	A = skb->len; */
393 				BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, len) != 4);
394 				if (is_imm8(offsetof(struct sk_buff, len)))
395 					/* mov    off8(%rdi),%eax */
396 					EMIT3(0x8b, 0x47, offsetof(struct sk_buff, len));
397 				else {
398 					EMIT2(0x8b, 0x87);
399 					EMIT(offsetof(struct sk_buff, len), 4);
400 				}
401 				break;
402 			case BPF_S_LDX_W_LEN: /* X = skb->len; */
403 				seen |= SEEN_XREG;
404 				if (is_imm8(offsetof(struct sk_buff, len)))
405 					/* mov off8(%rdi),%ebx */
406 					EMIT3(0x8b, 0x5f, offsetof(struct sk_buff, len));
407 				else {
408 					EMIT2(0x8b, 0x9f);
409 					EMIT(offsetof(struct sk_buff, len), 4);
410 				}
411 				break;
412 			case BPF_S_ANC_PROTOCOL: /* A = ntohs(skb->protocol); */
413 				BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, protocol) != 2);
414 				if (is_imm8(offsetof(struct sk_buff, protocol))) {
415 					/* movzwl off8(%rdi),%eax */
416 					EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, protocol));
417 				} else {
418 					EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */
419 					EMIT(offsetof(struct sk_buff, protocol), 4);
420 				}
421 				EMIT2(0x86, 0xc4); /* ntohs() : xchg   %al,%ah */
422 				break;
423 			case BPF_S_ANC_IFINDEX:
424 				if (is_imm8(offsetof(struct sk_buff, dev))) {
425 					/* movq off8(%rdi),%rax */
426 					EMIT4(0x48, 0x8b, 0x47, offsetof(struct sk_buff, dev));
427 				} else {
428 					EMIT3(0x48, 0x8b, 0x87); /* movq off32(%rdi),%rax */
429 					EMIT(offsetof(struct sk_buff, dev), 4);
430 				}
431 				EMIT3(0x48, 0x85, 0xc0);	/* test %rax,%rax */
432 				EMIT_COND_JMP(X86_JE, cleanup_addr - (addrs[i] - 6));
433 				BUILD_BUG_ON(FIELD_SIZEOF(struct net_device, ifindex) != 4);
434 				EMIT2(0x8b, 0x80);	/* mov off32(%rax),%eax */
435 				EMIT(offsetof(struct net_device, ifindex), 4);
436 				break;
437 			case BPF_S_ANC_MARK:
438 				BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, mark) != 4);
439 				if (is_imm8(offsetof(struct sk_buff, mark))) {
440 					/* mov off8(%rdi),%eax */
441 					EMIT3(0x8b, 0x47, offsetof(struct sk_buff, mark));
442 				} else {
443 					EMIT2(0x8b, 0x87);
444 					EMIT(offsetof(struct sk_buff, mark), 4);
445 				}
446 				break;
447 			case BPF_S_ANC_RXHASH:
448 				BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, rxhash) != 4);
449 				if (is_imm8(offsetof(struct sk_buff, rxhash))) {
450 					/* mov off8(%rdi),%eax */
451 					EMIT3(0x8b, 0x47, offsetof(struct sk_buff, rxhash));
452 				} else {
453 					EMIT2(0x8b, 0x87);
454 					EMIT(offsetof(struct sk_buff, rxhash), 4);
455 				}
456 				break;
457 			case BPF_S_ANC_QUEUE:
458 				BUILD_BUG_ON(FIELD_SIZEOF(struct sk_buff, queue_mapping) != 2);
459 				if (is_imm8(offsetof(struct sk_buff, queue_mapping))) {
460 					/* movzwl off8(%rdi),%eax */
461 					EMIT4(0x0f, 0xb7, 0x47, offsetof(struct sk_buff, queue_mapping));
462 				} else {
463 					EMIT3(0x0f, 0xb7, 0x87); /* movzwl off32(%rdi),%eax */
464 					EMIT(offsetof(struct sk_buff, queue_mapping), 4);
465 				}
466 				break;
467 			case BPF_S_ANC_CPU:
468 #ifdef CONFIG_SMP
469 				EMIT4(0x65, 0x8b, 0x04, 0x25); /* mov %gs:off32,%eax */
470 				EMIT((u32)(unsigned long)&cpu_number, 4); /* A = smp_processor_id(); */
471 #else
472 				CLEAR_A();
473 #endif
474 				break;
475 			case BPF_S_LD_W_ABS:
476 				func = sk_load_word;
477 common_load:			seen |= SEEN_DATAREF;
478 				if ((int)K < 0)
479 					goto out;
480 				t_offset = func - (image + addrs[i]);
481 				EMIT1_off32(0xbe, K); /* mov imm32,%esi */
482 				EMIT1_off32(0xe8, t_offset); /* call */
483 				break;
484 			case BPF_S_LD_H_ABS:
485 				func = sk_load_half;
486 				goto common_load;
487 			case BPF_S_LD_B_ABS:
488 				func = sk_load_byte;
489 				goto common_load;
490 			case BPF_S_LDX_B_MSH:
491 				if ((int)K < 0) {
492 					if (pc_ret0 > 0) {
493 						/* addrs[pc_ret0 - 1] is the start address */
494 						EMIT_JMP(addrs[pc_ret0 - 1] - addrs[i]);
495 						break;
496 					}
497 					CLEAR_A();
498 					EMIT_JMP(cleanup_addr - addrs[i]);
499 					break;
500 				}
501 				seen |= SEEN_DATAREF | SEEN_XREG;
502 				t_offset = sk_load_byte_msh - (image + addrs[i]);
503 				EMIT1_off32(0xbe, K);	/* mov imm32,%esi */
504 				EMIT1_off32(0xe8, t_offset); /* call sk_load_byte_msh */
505 				break;
506 			case BPF_S_LD_W_IND:
507 				func = sk_load_word_ind;
508 common_load_ind:		seen |= SEEN_DATAREF | SEEN_XREG;
509 				t_offset = func - (image + addrs[i]);
510 				EMIT1_off32(0xbe, K);	/* mov imm32,%esi   */
511 				EMIT1_off32(0xe8, t_offset);	/* call sk_load_xxx_ind */
512 				break;
513 			case BPF_S_LD_H_IND:
514 				func = sk_load_half_ind;
515 				goto common_load_ind;
516 			case BPF_S_LD_B_IND:
517 				func = sk_load_byte_ind;
518 				goto common_load_ind;
519 			case BPF_S_JMP_JA:
520 				t_offset = addrs[i + K] - addrs[i];
521 				EMIT_JMP(t_offset);
522 				break;
523 			COND_SEL(BPF_S_JMP_JGT_K, X86_JA, X86_JBE);
524 			COND_SEL(BPF_S_JMP_JGE_K, X86_JAE, X86_JB);
525 			COND_SEL(BPF_S_JMP_JEQ_K, X86_JE, X86_JNE);
526 			COND_SEL(BPF_S_JMP_JSET_K,X86_JNE, X86_JE);
527 			COND_SEL(BPF_S_JMP_JGT_X, X86_JA, X86_JBE);
528 			COND_SEL(BPF_S_JMP_JGE_X, X86_JAE, X86_JB);
529 			COND_SEL(BPF_S_JMP_JEQ_X, X86_JE, X86_JNE);
530 			COND_SEL(BPF_S_JMP_JSET_X,X86_JNE, X86_JE);
531 
532 cond_branch:			f_offset = addrs[i + filter[i].jf] - addrs[i];
533 				t_offset = addrs[i + filter[i].jt] - addrs[i];
534 
535 				/* same targets, can avoid doing the test :) */
536 				if (filter[i].jt == filter[i].jf) {
537 					EMIT_JMP(t_offset);
538 					break;
539 				}
540 
541 				switch (filter[i].code) {
542 				case BPF_S_JMP_JGT_X:
543 				case BPF_S_JMP_JGE_X:
544 				case BPF_S_JMP_JEQ_X:
545 					seen |= SEEN_XREG;
546 					EMIT2(0x39, 0xd8); /* cmp %ebx,%eax */
547 					break;
548 				case BPF_S_JMP_JSET_X:
549 					seen |= SEEN_XREG;
550 					EMIT2(0x85, 0xd8); /* test %ebx,%eax */
551 					break;
552 				case BPF_S_JMP_JEQ_K:
553 					if (K == 0) {
554 						EMIT2(0x85, 0xc0); /* test   %eax,%eax */
555 						break;
556 					}
557 				case BPF_S_JMP_JGT_K:
558 				case BPF_S_JMP_JGE_K:
559 					if (K <= 127)
560 						EMIT3(0x83, 0xf8, K); /* cmp imm8,%eax */
561 					else
562 						EMIT1_off32(0x3d, K); /* cmp imm32,%eax */
563 					break;
564 				case BPF_S_JMP_JSET_K:
565 					if (K <= 0xFF)
566 						EMIT2(0xa8, K); /* test imm8,%al */
567 					else if (!(K & 0xFFFF00FF))
568 						EMIT3(0xf6, 0xc4, K >> 8); /* test imm8,%ah */
569 					else if (K <= 0xFFFF) {
570 						EMIT2(0x66, 0xa9); /* test imm16,%ax */
571 						EMIT(K, 2);
572 					} else {
573 						EMIT1_off32(0xa9, K); /* test imm32,%eax */
574 					}
575 					break;
576 				}
577 				if (filter[i].jt != 0) {
578 					if (filter[i].jf && f_offset)
579 						t_offset += is_near(f_offset) ? 2 : 5;
580 					EMIT_COND_JMP(t_op, t_offset);
581 					if (filter[i].jf)
582 						EMIT_JMP(f_offset);
583 					break;
584 				}
585 				EMIT_COND_JMP(f_op, f_offset);
586 				break;
587 			default:
588 				/* hmm, too complex filter, give up with jit compiler */
589 				goto out;
590 			}
591 			ilen = prog - temp;
592 			if (image) {
593 				if (unlikely(proglen + ilen > oldproglen)) {
594 					pr_err("bpb_jit_compile fatal error\n");
595 					kfree(addrs);
596 					module_free(NULL, image);
597 					return;
598 				}
599 				memcpy(image + proglen, temp, ilen);
600 			}
601 			proglen += ilen;
602 			addrs[i] = proglen;
603 			prog = temp;
604 		}
605 		/* last bpf instruction is always a RET :
606 		 * use it to give the cleanup instruction(s) addr
607 		 */
608 		cleanup_addr = proglen - 1; /* ret */
609 		if (seen_or_pass0)
610 			cleanup_addr -= 1; /* leaveq */
611 		if (seen_or_pass0 & SEEN_XREG)
612 			cleanup_addr -= 4; /* mov  -8(%rbp),%rbx */
613 
614 		if (image) {
615 			if (proglen != oldproglen)
616 				pr_err("bpb_jit_compile proglen=%u != oldproglen=%u\n", proglen, oldproglen);
617 			break;
618 		}
619 		if (proglen == oldproglen) {
620 			image = module_alloc(max_t(unsigned int,
621 						   proglen,
622 						   sizeof(struct work_struct)));
623 			if (!image)
624 				goto out;
625 		}
626 		oldproglen = proglen;
627 	}
628 	if (bpf_jit_enable > 1)
629 		pr_err("flen=%d proglen=%u pass=%d image=%p\n",
630 		       flen, proglen, pass, image);
631 
632 	if (image) {
633 		if (bpf_jit_enable > 1)
634 			print_hex_dump(KERN_ERR, "JIT code: ", DUMP_PREFIX_ADDRESS,
635 				       16, 1, image, proglen, false);
636 
637 		bpf_flush_icache(image, image + proglen);
638 
639 		fp->bpf_func = (void *)image;
640 	}
641 out:
642 	kfree(addrs);
643 	return;
644 }
645 
jit_free_defer(struct work_struct * arg)646 static void jit_free_defer(struct work_struct *arg)
647 {
648 	module_free(NULL, arg);
649 }
650 
651 /* run from softirq, we must use a work_struct to call
652  * module_free() from process context
653  */
bpf_jit_free(struct sk_filter * fp)654 void bpf_jit_free(struct sk_filter *fp)
655 {
656 	if (fp->bpf_func != sk_run_filter) {
657 		struct work_struct *work = (struct work_struct *)fp->bpf_func;
658 
659 		INIT_WORK(work, jit_free_defer);
660 		schedule_work(work);
661 	}
662 }
663