Lines Matching +full:smp +full:- +full:offset
1 // SPDX-License-Identifier: GPL-2.0-only
2 #define pr_fmt(fmt) "SMP alternatives: " fmt
22 #include <asm/text-patching.h>
33 #include <asm/asm-prototypes.h>
42 #define MAX_PATCH_LEN (255-1)
63 __setup("debug-alternative", debug_alt);
72 __setup("noreplace-smp", setup_noreplace_smp);
89 for (j = 0; j < (len) - 1; j++) \
146 int offset = 0; in its_init_thunk() local
161 offset = 1; in its_init_thunk()
167 reg -= 8; in its_init_thunk()
173 return thunk + offset; in its_init_thunk()
198 for (int i = 0; i < mod->its_num_pages; i++) { in its_fini_mod()
199 void *page = mod->its_page_array[i]; in its_fini_mod()
209 for (int i = 0; i < mod->its_num_pages; i++) { in its_free_mod()
210 void *page = mod->its_page_array[i]; in its_free_mod()
213 kfree(mod->its_page_array); in its_free_mod()
226 void *tmp = krealloc(its_mod->its_page_array, in its_alloc()
227 (its_mod->its_num_pages+1) * sizeof(void *), in its_alloc()
232 its_mod->its_page_array = tmp; in its_alloc()
233 its_mod->its_page_array[its_mod->its_num_pages++] = page; in its_alloc()
258 if (!its_page || (its_offset + size - 1) >= PAGE_SIZE) { in its_allocate_thunk()
270 * of a cacheline, then update the offset to reach the upper half. in its_allocate_thunk()
272 if ((its_offset + size - 1) % 64 < 32) in its_allocate_thunk()
273 its_offset = ((its_offset - 1) | 0x3F) + 33; in its_allocate_thunk()
288 return thunk - 2; in its_static_thunk()
303 * eventually text-poked into the kernel image.
313 * for every single-byte NOP, try to generate the maximally available NOP of
315 * each single-byte NOPs). If @len to fill out is > ASM_NOP_MAX, pad with INT3 and
355 if (insn->opcode.bytes[0] == 0x90 && in insn_is_nop()
356 (!insn->prefixes.nbytes || insn->prefixes.bytes[0] != 0xF3)) in insn_is_nop()
360 if (insn->opcode.bytes[0] == 0x0F && insn->opcode.bytes[1] == 0x1F) in insn_is_nop()
369 * Find the offset of the first non-NOP instruction starting at @offset
372 static int skip_nops(u8 *buf, int offset, int len) in skip_nops() argument
376 for (; offset < len; offset += insn.length) { in skip_nops()
377 if (insn_decode_kernel(&insn, &buf[offset])) in skip_nops()
384 return offset; in skip_nops()
410 add_nop(buf + nop, next - nop); in optimize_nops()
423 * The source offset is:
425 * src_imm = target - src_next_ip (1)
427 * and the target offset is:
429 * dst_imm = target - dst_next_ip (2)
437 * dst_imm = (src_imm + src_next_ip) - dst_next_ip (3)
448 * dst_imm = src_imm + (src + ip_offset) - (dst + ip_offset)
449 * = src_imm + src - dst + ip_offset - ip_offset
450 * = src_imm + src - dst (5)
459 BUG_ON((v >> 31) != (v >> (n_-1))); \
476 bool need_reloc(unsigned long offset, u8 *src, size_t src_len) in need_reloc() argument
478 u8 *target = src + offset; in need_reloc()
510 repl - instr); in __apply_relocation()
518 imm += repl - instr; in __apply_relocation()
519 imm += JMP32_INSN_SIZE - JMP8_INSN_SIZE; in __apply_relocation()
524 memset(&buf[i+2], INT3_INSN_OPCODE, insn.length - 2); in __apply_relocation()
534 repl - instr); in __apply_relocation()
546 /* Low-level backend functions usable from alternative code replacements. */
568 if (a->replacementlen != 5 || insn_buff[0] != CALL_INSN_OPCODE) { in alt_replace_call()
569 pr_err("ALT_FLAG_DIRECT_CALL set for a non-call replacement instruction\n"); in alt_replace_call()
573 if (a->instrlen != 6 || in alt_replace_call()
585 target = *(void **)(instr + a->instrlen + disp); in alt_replace_call()
594 /* (BUG_func - .) + (target - BUG_func) := target - . */ in alt_replace_call()
595 *(s32 *)(insn_buff + 1) += target - bug; in alt_replace_call()
605 return (u8 *)&i->instr_offset + i->instr_offset; in instr_va()
610 * before SMP is initialized to avoid SMP problems with self modifying code.
625 DPRINTK(ALT, "alt table %px, -> %px", start, end); in apply_alternatives()
631 * conversion and triggers a false-positive out-of-bound report. in apply_alternatives()
656 u8 len = max(a->instrlen, b->instrlen); in apply_alternatives()
657 a->instrlen = b->instrlen = len; in apply_alternatives()
661 replacement = (u8 *)&a->repl_offset + a->repl_offset; in apply_alternatives()
662 BUG_ON(a->instrlen > sizeof(insn_buff)); in apply_alternatives()
663 BUG_ON(a->cpuid >= (NCAPINTS + NBUGINTS) * 32); in apply_alternatives()
667 * - feature is present in apply_alternatives()
668 * - feature not present but ALT_FLAG_NOT is set to mean, in apply_alternatives()
671 if (!boot_cpu_has(a->cpuid) == !(a->flags & ALT_FLAG_NOT)) { in apply_alternatives()
672 memcpy(insn_buff, instr, a->instrlen); in apply_alternatives()
673 optimize_nops(instr, insn_buff, a->instrlen); in apply_alternatives()
674 text_poke_early(instr, insn_buff, a->instrlen); in apply_alternatives()
679 a->cpuid >> 5, in apply_alternatives()
680 a->cpuid & 0x1f, in apply_alternatives()
681 instr, instr, a->instrlen, in apply_alternatives()
682 replacement, a->replacementlen, a->flags); in apply_alternatives()
684 memcpy(insn_buff, replacement, a->replacementlen); in apply_alternatives()
685 insn_buff_sz = a->replacementlen; in apply_alternatives()
687 if (a->flags & ALT_FLAG_DIRECT_CALL) { in apply_alternatives()
693 for (; insn_buff_sz < a->instrlen; insn_buff_sz++) in apply_alternatives()
696 apply_relocation(insn_buff, instr, a->instrlen, replacement, a->replacementlen); in apply_alternatives()
698 DUMP_BYTES(ALT, instr, a->instrlen, "%px: old_insn: ", instr); in apply_alternatives()
699 DUMP_BYTES(ALT, replacement, a->replacementlen, "%px: rpl_insn: ", replacement); in apply_alternatives()
710 /* Jcc.d32 second opcode byte is in the range: 0x80-0x8f */ in is_jcc32()
711 return insn->opcode.bytes[0] == 0x0f && (insn->opcode.bytes[1] & 0xf0) == 0x80; in is_jcc32()
735 return -1; in emit_indirect()
740 reg -= 8; in emit_indirect()
755 u8 op = insn->opcode.bytes[0]; in __emit_trampoline()
760 * tail-calls. Deal with them. in __emit_trampoline()
764 op = insn->opcode.bytes[1]; in __emit_trampoline()
768 if (insn->length == 6) in __emit_trampoline()
769 bytes[i++] = 0x2e; /* CS-prefix */ in __emit_trampoline()
789 return -1; in __emit_trampoline()
792 WARN_ON_ONCE(i != insn->length); in __emit_trampoline()
816 /* Check if an indirect branch is at ITS-unsafe address */
825 /* Lower-half of the cacheline? */ in cpu_wants_indirect_its_thunk_at()
861 target = addr + insn->length + insn->immediate.value; in patch_retpoline()
862 reg = target - __x86_indirect_thunk_array; in patch_retpoline()
865 return -1; in patch_retpoline()
875 return -1; in patch_retpoline()
878 op = insn->opcode.bytes[0]; in patch_retpoline()
894 cc = insn->opcode.bytes[1] & 0xf; in patch_retpoline()
898 bytes[i++] = insn->length - 2; /* sizeof(Jcc.d8) == 2 */ in patch_retpoline()
915 * Check if the address of last byte of emitted-indirect is in in patch_retpoline()
916 * lower-half of the cacheline. Such branches need ITS mitigation. in patch_retpoline()
933 if (op == JMP32_INSN_OPCODE && i < insn->length) in patch_retpoline()
936 for (; i < insn->length;) in patch_retpoline()
943 * Generated by 'objtool --retpoline'.
974 if (dest[-1] == 0xea && (dest[0] & 0xf0) == 0x70) { in apply_retpolines()
1021 * Rewrite the compiler generated return thunk tail-calls.
1044 for (; i < insn->length;) in patch_return()
1073 "missing return thunk: %pS-%pS: %*ph", in apply_returns()
1148 * Generated by: objtool --ibt
1162 poison_cfi(addr - 16); in apply_seal_endbr()
1228 func -= cfi_get_offset(); in cfi_get_func_hash()
1254 if (get_kernel_nofault(disp, func - 4)) in cfi_get_func_arity()
1258 return target - __bhi_args; in cfi_get_func_arity()
1268 * Re-hash the CFI hash with a boot-time seed while making sure the result is
1274 while (unlikely(__is_endbr(hash) || __is_endbr(-hash))) { in cfi_rehash()
1286 return -EINVAL; in cfi_parse_cmdline()
1307 pr_alert("CFI mismatch non-fatal!\n"); in cfi_parse_cmdline()
1355 * movl $(-0x12345678),%r10d // 6 movl $0x12345678,%r10d // 6
1356 * addl $-15(%r11),%r10d // 4 lea -0x10(%r11),%r11 // 4
1388 #define fineibt_preamble_size (fineibt_preamble_end - fineibt_preamble_start)
1389 #define fineibt_preamble_bhi (fineibt_preamble_bhi - fineibt_preamble_start)
1396 * 6: 4d 8d 5b f0 lea -0x10(%r11), %r11
1402 " lea -0x10(%r11), %r11 \n"
1411 #define fineibt_caller_size (fineibt_caller_end - fineibt_caller_start)
1414 #define fineibt_caller_jmp (fineibt_caller_size - 2)
1430 * 6: 45 3b 53 f7 cmp -0x9(%r11), %r10d
1431 * a: 4d 8d 5b <f0> lea -0x10(%r11), %r11
1437 * avoiding a dependency. Again, using a non-taken (backwards) branch
1444 " cmpl -9(%r11), %r10d \n"
1445 " lea -0x10(%r11), %r11 \n"
1458 #define fineibt_paranoid_size (fineibt_paranoid_end - fineibt_paranoid_start)
1459 #define fineibt_paranoid_ind (fineibt_paranoid_ind - fineibt_paranoid_start)
1469 *reg = p[0] - 0xb8; in decode_preamble_hash()
1480 /* 41 ba 88 a9 cb ed mov $(-0x12345678),%r10d */ in decode_caller_hash()
1482 return -*(u32 *)(addr + 2); in decode_caller_hash()
1486 return -*(u32 *)(addr + 2); in decode_caller_hash()
1506 addr -= fineibt_caller_size; in cfi_disable_callers()
1520 * Re-enable kCFI, undo what cfi_disable_callers() did. in cfi_enable_callers()
1529 addr -= fineibt_caller_size; in cfi_enable_callers()
1552 return -EINVAL; in cfi_rand_preamble()
1568 * Crazy scheme to allow arity-1 inline: in cfi_fineibt_bhi_preamble()
1628 return -EINVAL; in cfi_rewrite_preamble()
1668 addr -= fineibt_caller_size; in cfi_rand_callers()
1671 hash = -cfi_rehash(hash); in cfi_rand_callers()
1681 u8 *thunk = (void *)__x86_indirect_its_thunk_array[reg] - 2; in emit_paranoid_trampoline()
1706 addr -= fineibt_caller_size; in cfi_rewrite_callers()
1811 /* place the FineIBT preamble at func()-16 */ in __apply_fineibt()
1816 /* rewrite the callers to target func()-16 */ in __apply_fineibt()
1895 * When regs->ip points to a 0xEA byte in the FineIBT preamble,
1903 unsigned long addr = regs->ip - fineibt_preamble_ud; in decode_fineibt_preamble()
1912 *type = (u32)regs->r10 + hash; in decode_fineibt_preamble()
1915 * Since regs->ip points to the middle of an instruction; it cannot in decode_fineibt_preamble()
1918 regs->ip = *target; in decode_fineibt_preamble()
1927 * regs->ip points to one of the UD2 in __bhi_args[].
1937 if (regs->ip < (unsigned long)__bhi_args || in decode_fineibt_bhi()
1938 regs->ip >= (unsigned long)__bhi_args_end) in decode_fineibt_bhi()
1947 __get_kernel_nofault(&addr, regs->sp, unsigned long, Efault); in decode_fineibt_bhi()
1950 addr -= fineibt_preamble_size; in decode_fineibt_bhi()
1955 *type = (u32)regs->r10 + hash; in decode_fineibt_bhi()
1959 * as such the non-fatal case can use the regular fixup. in decode_fineibt_bhi()
1979 * regs->ip points to a LOCK Jcc.d8 instruction from the fineibt_paranoid_start[]
1985 unsigned long addr = regs->ip - fineibt_paranoid_ud; in decode_fineibt_paranoid()
1990 if (is_cfi_trap(addr + fineibt_caller_size - LEN_UD2)) { in decode_fineibt_paranoid()
1991 *target = regs->r11 + fineibt_preamble_size; in decode_fineibt_paranoid()
1992 *type = regs->r10; in decode_fineibt_paranoid()
2005 * 6: 45 3b 53 f7 cmp -0x9(%r11), %r10d in decode_fineibt_paranoid()
2006 * a: 4d 8d 5b f0 lea -0x10(%r11), %r11 in decode_fineibt_paranoid()
2019 if (is_paranoid_thunk(regs->ip)) { in decode_fineibt_paranoid()
2020 *target = regs->r11 + fineibt_preamble_size; in decode_fineibt_paranoid()
2021 *type = regs->r10; in decode_fineibt_paranoid()
2023 regs->ip = *target; in decode_fineibt_paranoid()
2118 struct smp_alt_module *smp; in alternatives_smp_module_add() local
2128 smp = kzalloc(sizeof(*smp), GFP_KERNEL); in alternatives_smp_module_add()
2129 if (NULL == smp) in alternatives_smp_module_add()
2130 /* we'll run the (safe but slow) SMP code then ... */ in alternatives_smp_module_add()
2133 smp->mod = mod; in alternatives_smp_module_add()
2134 smp->name = name; in alternatives_smp_module_add()
2135 smp->locks = locks; in alternatives_smp_module_add()
2136 smp->locks_end = locks_end; in alternatives_smp_module_add()
2137 smp->text = text; in alternatives_smp_module_add()
2138 smp->text_end = text_end; in alternatives_smp_module_add()
2139 DPRINTK(SMP, "locks %p -> %p, text %p -> %p, name %s\n", in alternatives_smp_module_add()
2140 smp->locks, smp->locks_end, in alternatives_smp_module_add()
2141 smp->text, smp->text_end, smp->name); in alternatives_smp_module_add()
2143 list_add_tail(&smp->next, &smp_alt_modules); in alternatives_smp_module_add()
2156 if (mod != item->mod) in alternatives_smp_module_del()
2158 list_del(&item->next); in alternatives_smp_module_del()
2175 pr_info("switching to SMP code\n"); in alternatives_enable_smp()
2180 alternatives_smp_lock(mod->locks, mod->locks_end, in alternatives_enable_smp()
2181 mod->text, mod->text_end); in alternatives_enable_smp()
2188 * Return 1 if the address range is reserved for SMP-alternatives.
2201 if (mod->text > text_end || mod->text_end < text_start) in alternatives_text_reserved()
2203 for (poff = mod->locks; poff < mod->locks_end; poff++) { in alternatives_text_reserved()
2216 * Self-test for the INT3 based CALL emulation code.
2240 " .size int3_magic, .-int3_magic\n"
2251 struct pt_regs *regs = args->regs; in int3_exception_notify()
2261 if (regs->ip - INT3_INSN_SIZE != selftest) in int3_exception_notify()
2273 .priority = INT_MAX-1, /* last */ in int3_selftest()
2280 * Basically: int3_magic(&val); but really complicated :-) in int3_selftest()
2311 * instruction and additionally on x86_64 we get a RIP-relative LEA: in alt_reloc_selftest()
2314 * call +0 # 5d5: R_X86_64_PLT32 __alt_reloc_selftest-0x4 in alt_reloc_selftest()
2357 /* Keep CET-IBT disabled until caller/callee are patched */ in alternative_instructions()
2371 * Adjust all CALL instructions to point to func()-10, including in alternative_instructions()
2395 free_init_pages("SMP alternatives", in alternative_instructions()
2408 * text_poke_early - Update instructions on a live kernel at boot time
2427 * Modules text is marked initially as non-executable, so the in text_poke_early()
2428 * code cannot be running and speculative code-fetches are in text_poke_early()
2453 * temporary page-table mappings that are required for these write operations to
2564 * flush_tlb_mm_range(), which is intended for non-global PTEs. in __text_poke()
2569 * The lock is not really needed, but this allows to avoid open-coding. in __text_poke()
2609 * Loading the previous page-table hierarchy requires a serializing in __text_poke()
2611 * Xen-PV is assumed to serialize execution in a similar manner. in __text_poke()
2637 * text_poke - Update instructions on a live kernel
2660 * text_poke_kgdb - Update instructions on a live kernel by kgdb
2691 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); in text_poke_copy_locked()
2700 * text_poke_copy - Copy instructions into (an unused part of) RX memory
2720 * text_poke_set - memset into (an unused part of) RX memory
2741 s = min_t(size_t, PAGE_SIZE * 2 - offset_in_page(ptr), len - patched); in text_poke_set()
2789 if (!raw_atomic_inc_not_zero(&desc->refs)) in try_get_desc()
2800 raw_atomic_dec(&desc->refs); in put_desc()
2805 return _stext + tp->rel_addr; in text_poke_addr()
2813 return -1; in patch_cmp()
2831 * bp_desc with non-zero refcount: in poke_int3_handler()
2846 ip = (void *) regs->ip - INT3_INSN_SIZE; in poke_int3_handler()
2851 if (unlikely(desc->nr_entries > 1)) { in poke_int3_handler()
2852 tp = __inline_bsearch(ip, desc->vec, desc->nr_entries, in poke_int3_handler()
2858 tp = desc->vec; in poke_int3_handler()
2863 ip += tp->len; in poke_int3_handler()
2865 switch (tp->opcode) { in poke_int3_handler()
2878 int3_emulate_call(regs, (long)ip + tp->disp); in poke_int3_handler()
2883 int3_emulate_jmp(regs, (long)ip + tp->disp); in poke_int3_handler()
2887 int3_emulate_jcc(regs, tp->opcode & 0xf, (long)ip, tp->disp); in poke_int3_handler()
2906 * text_poke_bp_batch() -- update instructions on live kernel on SMP
2910 * Modify multi-byte instruction by using int3 breakpoint on SMP.
2915 * - For each entry in the vector:
2916 * - add a int3 trap to the address that will be patched
2917 * - sync cores
2918 * - For each entry in the vector:
2919 * - update all but the first byte of the patched range
2920 * - sync cores
2921 * - For each entry in the vector:
2922 * - replace the first byte (int3) by the first byte of
2924 * - sync cores
2939 * ensure reading a non-zero refcount provides up to date bp_desc data. in text_poke_bp_batch()
2978 if (len - INT3_INSN_SIZE > 0) { in text_poke_bp_batch()
2981 len - INT3_INSN_SIZE); in text_poke_bp_batch()
2991 len - INT3_INSN_SIZE); in text_poke_bp_batch()
3000 * - write INT3 byte in text_poke_bp_batch()
3001 * - IPI-SYNC in text_poke_bp_batch()
3002 * - write instruction tail in text_poke_bp_batch()
3007 * - emit RECORD_TEXT_POKE with the new instruction in text_poke_bp_batch()
3008 * - IPI-SYNC in text_poke_bp_batch()
3009 * - write first byte in text_poke_bp_batch()
3010 * - IPI-SYNC in text_poke_bp_batch()
3067 memcpy((void *)tp->text, opcode+i, len-i); in text_poke_loc_init()
3074 tp->rel_addr = addr - (void *)_stext; in text_poke_loc_init()
3075 tp->len = len; in text_poke_loc_init()
3076 tp->opcode = insn.opcode.bytes[0]; in text_poke_loc_init()
3082 tp->opcode = insn.opcode.bytes[1] - 0x10; in text_poke_loc_init()
3085 switch (tp->opcode) { in text_poke_loc_init()
3094 BUG_ON(tp->text[i] != INT3_INSN_OPCODE); in text_poke_loc_init()
3101 switch (tp->opcode) { in text_poke_loc_init()
3110 tp->disp = insn.immediate.value; in text_poke_loc_init()
3115 case 2: /* NOP2 -- emulate as JMP8+0 */ in text_poke_loc_init()
3117 tp->opcode = JMP8_INSN_OPCODE; in text_poke_loc_init()
3118 tp->disp = 0; in text_poke_loc_init()
3121 case 5: /* NOP5 -- emulate as JMP32+0 */ in text_poke_loc_init()
3123 tp->opcode = JMP32_INSN_OPCODE; in text_poke_loc_init()
3124 tp->disp = 0; in text_poke_loc_init()
3148 tp = &tp_vec[tp_vec_nr - 1]; in tp_order_fail()
3179 * text_poke_bp() -- update instructions on live kernel on SMP