1 // SPDX-License-Identifier: GPL-2.0-only
2 #define pr_fmt(fmt) "SMP alternatives: " fmt
21 #include <asm/text-patching.h>
36 #define MAX_PATCH_LEN (255-1)
45 __setup("debug-alternative", debug_alt);
54 __setup("noreplace-smp", setup_noreplace_smp);
71 		for (j = 0; j < (len) - 1; j++)				\
80  * add to the array the offset that is equal to the sum of all sizes of
212 		 * the "k8_nops" than with the SDM-recommended NOPs.  in arch_init_ideal_nops()
267 		len -= noplen;  in add_nops()
276  * Are we looking at a near JMP with a 1 or 4-byte displacement.
290 	if (a->replacementlen != 5)  in recompute_jump()
296 	next_rip = repl_insn + a->replacementlen;  in recompute_jump()
299 	n_dspl = tgt_rip - orig_insn;  in recompute_jump()
303 	if (tgt_rip - orig_insn >= 0) {  in recompute_jump()
304 		if (n_dspl - 2 <= 127)  in recompute_jump()
308 	/* negative offset */  in recompute_jump()
310 		if (((n_dspl - 2) & 0xff) == (n_dspl - 2))  in recompute_jump()
317 	n_dspl -= 2;  in recompute_jump()
327 	n_dspl -= 5;  in recompute_jump()
349 	for (i = 0; i < a->padlen; i++) {  in optimize_nops()
355 	add_nops(instr + (a->instrlen - a->padlen), a->padlen);  in optimize_nops()
358 	DUMP_BYTES(instr, a->instrlen, "%px: [%d:%d) optimized NOPs: ",  in optimize_nops()
359 		   instr, a->instrlen - a->padlen, a->padlen);  in optimize_nops()
364  * before SMP is initialized to avoid SMP problems with self modifying code.
379 	DPRINTK("alt table %px, -> %px", start, end);  in apply_alternatives()
392 		instr = (u8 *)&a->instr_offset + a->instr_offset;  in apply_alternatives()
393 		replacement = (u8 *)&a->repl_offset + a->repl_offset;  in apply_alternatives()
394 		BUG_ON(a->instrlen > sizeof(insn_buff));  in apply_alternatives()
395 		BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32);  in apply_alternatives()
396 		if (!boot_cpu_has(a->cpuid)) {  in apply_alternatives()
397 			if (a->padlen > 1)  in apply_alternatives()
404 			a->cpuid >> 5,  in apply_alternatives()
405 			a->cpuid & 0x1f,  in apply_alternatives()
406 			instr, instr, a->instrlen,  in apply_alternatives()
407 			replacement, a->replacementlen, a->padlen);  in apply_alternatives()
409 		DUMP_BYTES(instr, a->instrlen, "%px: old_insn: ", instr);  in apply_alternatives()
410 		DUMP_BYTES(replacement, a->replacementlen, "%px: rpl_insn: ", replacement);  in apply_alternatives()
412 		memcpy(insn_buff, replacement, a->replacementlen);  in apply_alternatives()
413 		insn_buff_sz = a->replacementlen;  in apply_alternatives()
416 		 * 0xe8 is a relative jump; fix the offset.  in apply_alternatives()
419 		 * accessing uninitialized bytes for zero-length replacements.  in apply_alternatives()
421 		if (a->replacementlen == 5 && *insn_buff == 0xe8) {  in apply_alternatives()
422 			*(s32 *)(insn_buff + 1) += replacement - instr;  in apply_alternatives()
423 			DPRINTK("Fix CALL offset: 0x%x, CALL 0x%lx",  in apply_alternatives()
428 		if (a->replacementlen && is_jmp(replacement[0]))  in apply_alternatives()
431 		if (a->instrlen > a->replacementlen) {  in apply_alternatives()
432 			add_nops(insn_buff + a->replacementlen,  in apply_alternatives()
433 				 a->instrlen - a->replacementlen);  in apply_alternatives()
434 			insn_buff_sz += a->instrlen - a->replacementlen;  in apply_alternatives()
498 	struct smp_alt_module *smp;  in alternatives_smp_module_add()  local
508 	smp = kzalloc(sizeof(*smp), GFP_KERNEL);  in alternatives_smp_module_add()
509 	if (NULL == smp)  in alternatives_smp_module_add()
510 		/* we'll run the (safe but slow) SMP code then ... */  in alternatives_smp_module_add()
513 	smp->mod	= mod;  in alternatives_smp_module_add()
514 	smp->name	= name;  in alternatives_smp_module_add()
515 	smp->locks	= locks;  in alternatives_smp_module_add()
516 	smp->locks_end	= locks_end;  in alternatives_smp_module_add()
517 	smp->text	= text;  in alternatives_smp_module_add()
518 	smp->text_end	= text_end;  in alternatives_smp_module_add()
519 	DPRINTK("locks %p -> %p, text %p -> %p, name %s\n",  in alternatives_smp_module_add()
520 		smp->locks, smp->locks_end,  in alternatives_smp_module_add()
521 		smp->text, smp->text_end, smp->name);  in alternatives_smp_module_add()
523 	list_add_tail(&smp->next, &smp_alt_modules);  in alternatives_smp_module_add()
536 		if (mod != item->mod)  in alternatives_smp_module_del()
538 		list_del(&item->next);  in alternatives_smp_module_del()
555 		pr_info("switching to SMP code\n");  in alternatives_enable_smp()
560 			alternatives_smp_lock(mod->locks, mod->locks_end,  in alternatives_enable_smp()
561 					      mod->text, mod->text_end);  in alternatives_enable_smp()
568  * Return 1 if the address range is reserved for SMP-alternatives.
581 		if (mod->text > text_end || mod->text_end < text_start)  in alternatives_text_reserved()
583 		for (poff = mod->locks; poff < mod->locks_end; poff++) {  in alternatives_text_reserved()
605 		BUG_ON(p->len > MAX_PATCH_LEN);  in apply_paravirt()
607 		memcpy(insn_buff, p->instr, p->len);  in apply_paravirt()
608 		used = pv_ops.init.patch(p->type, insn_buff, (unsigned long)p->instr, p->len);  in apply_paravirt()
610 		BUG_ON(used > p->len);  in apply_paravirt()
613 		add_nops(insn_buff + used, p->len - used);  in apply_paravirt()
614 		text_poke_early(p->instr, insn_buff, p->len);  in apply_paravirt()
622  * Self-test for the INT3 based CALL emulation code.
645 "	.size		int3_magic, .-int3_magic\n"
655 	struct pt_regs *regs = args->regs;  in int3_exception_notify()
663 	if (regs->ip - INT3_INSN_SIZE != int3_selftest_ip)  in int3_exception_notify()
674 		.priority	= INT_MAX-1, /* last */  in int3_selftest()
681 	 * Basically: int3_magic(&val); but really complicated :-)  in int3_selftest()
738 		free_init_pages("SMP alternatives",  in alternative_instructions()
751  * text_poke_early - Update instructions on a live kernel at boot time
770 		 * Modules text is marked initially as non-executable, so the  in text_poke_early()
771 		 * code cannot be running and speculative code-fetches are  in text_poke_early()
796  * temporary page-table mappings that are required for these write operations to
889 	 * flush_tlb_mm_range(), which is intended for non-global PTEs.  in __text_poke()
894 	 * The lock is not really needed, but this allows to avoid open-coding.  in __text_poke()
934 	 * Loading the previous page-table hierarchy requires a serializing  in __text_poke()
936 	 * Xen-PV is assumed to serialize execution in a similar manner.  in __text_poke()
960  * text_poke - Update instructions on a live kernel
983  * text_poke_kgdb - Update instructions on a live kernel by kgdb
1032 	if (!desc || !arch_atomic_inc_not_zero(&desc->refs))  in try_get_desc()
1041 	arch_atomic_dec(&desc->refs);  in put_desc()
1046 	return _stext + tp->rel_addr;  in text_poke_addr()
1054 		return -1;  in patch_cmp()
1087 	ip = (void *) regs->ip - INT3_INSN_SIZE;  in poke_int3_handler()
1092 	if (unlikely(desc->nr_entries > 1)) {  in poke_int3_handler()
1093 		tp = __inline_bsearch(ip, desc->vec, desc->nr_entries,  in poke_int3_handler()
1099 		tp = desc->vec;  in poke_int3_handler()
1104 	len = text_opcode_size(tp->opcode);  in poke_int3_handler()
1107 	switch (tp->opcode) {  in poke_int3_handler()
1120 		int3_emulate_call(regs, (long)ip + tp->rel32);  in poke_int3_handler()
1125 		int3_emulate_jmp(regs, (long)ip + tp->rel32);  in poke_int3_handler()
1144  * text_poke_bp_batch() -- update instructions on live kernel on SMP
1148  * Modify multi-byte instruction by using int3 breakpoint on SMP.
1153  *	- For each entry in the vector:
1154  *		- add a int3 trap to the address that will be patched
1155  *	- sync cores
1156  *	- For each entry in the vector:
1157  *		- update all but the first byte of the patched range
1158  *	- sync cores
1159  *	- For each entry in the vector:
1160  *		- replace the first byte (int3) by the first byte of
1162  *	- sync cores
1202 		if (len - INT3_INSN_SIZE > 0) {  in text_poke_bp_batch()
1205 			       len - INT3_INSN_SIZE);  in text_poke_bp_batch()
1208 				  len - INT3_INSN_SIZE);  in text_poke_bp_batch()
1216 		 *   - write INT3 byte  in text_poke_bp_batch()
1217 		 *   - IPI-SYNC  in text_poke_bp_batch()
1218 		 *   - write instruction tail  in text_poke_bp_batch()
1223 		 *   - emit RECORD_TEXT_POKE with the new instruction  in text_poke_bp_batch()
1224 		 *   - IPI-SYNC  in text_poke_bp_batch()
1225 		 *   - write first byte  in text_poke_bp_batch()
1226 		 *   - IPI-SYNC  in text_poke_bp_batch()
1278 	memcpy((void *)tp->text, opcode, len);  in text_poke_loc_init()
1288 	tp->rel_addr = addr - (void *)_stext;  in text_poke_loc_init()
1289 	tp->opcode = insn.opcode.bytes[0];  in text_poke_loc_init()
1291 	switch (tp->opcode) {  in text_poke_loc_init()
1299 		tp->rel32 = insn.immediate.value;  in text_poke_loc_init()
1304 		case 2: /* NOP2 -- emulate as JMP8+0 */  in text_poke_loc_init()
1306 			tp->opcode = JMP8_INSN_OPCODE;  in text_poke_loc_init()
1307 			tp->rel32 = 0;  in text_poke_loc_init()
1310 		case 5: /* NOP5 -- emulate as JMP32+0 */  in text_poke_loc_init()
1312 			tp->opcode = JMP32_INSN_OPCODE;  in text_poke_loc_init()
1313 			tp->rel32 = 0;  in text_poke_loc_init()
1337 	tp = &tp_vec[tp_vec_nr - 1];  in tp_order_fail()
1373  * text_poke_bp() -- update instructions on live kernel on SMP