1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3  * Based on arch/arm/include/asm/assembler.h, arch/arm/mm/proc-macros.S
4  *
5  * Copyright (C) 1996-2000 Russell King
6  * Copyright (C) 2012 ARM Ltd.
7  */
8 #ifndef __ASSEMBLY__
9 #error "Only include this from assembly code"
10 #endif
11 
12 #ifndef __ASM_ASSEMBLER_H
13 #define __ASM_ASSEMBLER_H
14 
15 #include <asm-generic/export.h>
16 
17 #include <asm/asm-offsets.h>
18 #include <asm/cpufeature.h>
19 #include <asm/cputype.h>
20 #include <asm/debug-monitors.h>
21 #include <asm/page.h>
22 #include <asm/pgtable-hwdef.h>
23 #include <asm/ptrace.h>
24 #include <asm/thread_info.h>
25 
26 	.macro save_and_disable_daif, flags
27 	mrs	\flags, daif
28 	msr	daifset, #0xf
29 	.endm
30 
31 	.macro disable_daif
32 	msr	daifset, #0xf
33 	.endm
34 
35 	.macro enable_daif
36 	msr	daifclr, #0xf
37 	.endm
38 
39 	.macro	restore_daif, flags:req
40 	msr	daif, \flags
41 	.endm
42 
43 	/* Only on aarch64 pstate, PSR_D_BIT is different for aarch32 */
44 	.macro	inherit_daif, pstate:req, tmp:req
45 	and	\tmp, \pstate, #(PSR_D_BIT | PSR_A_BIT | PSR_I_BIT | PSR_F_BIT)
46 	msr	daif, \tmp
47 	.endm
48 
49 	/* IRQ is the lowest priority flag, unconditionally unmask the rest. */
50 	.macro enable_da_f
51 	msr	daifclr, #(8 | 4 | 1)
52 	.endm
53 
54 /*
55  * Save/restore interrupts.
56  */
57 	.macro	save_and_disable_irq, flags
58 	mrs	\flags, daif
59 	msr	daifset, #2
60 	.endm
61 
62 	.macro	restore_irq, flags
63 	msr	daif, \flags
64 	.endm
65 
66 	.macro	enable_dbg
67 	msr	daifclr, #8
68 	.endm
69 
70 	.macro	disable_step_tsk, flgs, tmp
71 	tbz	\flgs, #TIF_SINGLESTEP, 9990f
72 	mrs	\tmp, mdscr_el1
73 	bic	\tmp, \tmp, #DBG_MDSCR_SS
74 	msr	mdscr_el1, \tmp
75 	isb	// Synchronise with enable_dbg
76 9990:
77 	.endm
78 
79 	/* call with daif masked */
80 	.macro	enable_step_tsk, flgs, tmp
81 	tbz	\flgs, #TIF_SINGLESTEP, 9990f
82 	mrs	\tmp, mdscr_el1
83 	orr	\tmp, \tmp, #DBG_MDSCR_SS
84 	msr	mdscr_el1, \tmp
85 9990:
86 	.endm
87 
88 /*
89  * SMP data memory barrier
90  */
91 	.macro	smp_dmb, opt
92 	dmb	\opt
93 	.endm
94 
95 /*
96  * RAS Error Synchronization barrier
97  */
98 	.macro  esb
99 #ifdef CONFIG_ARM64_RAS_EXTN
100 	hint    #16
101 #else
102 	nop
103 #endif
104 	.endm
105 
106 /*
107  * Value prediction barrier
108  */
109 	.macro	csdb
110 	hint	#20
111 	.endm
112 
113 /*
114  * Speculation barrier
115  */
116 	.macro	sb
117 alternative_if_not ARM64_HAS_SB
118 	dsb	nsh
119 	isb
120 alternative_else
121 	SB_BARRIER_INSN
122 	nop
123 alternative_endif
124 	.endm
125 
126 /*
127  * Sanitise a 64-bit bounded index wrt speculation, returning zero if out
128  * of bounds.
129  */
130 	.macro	mask_nospec64, idx, limit, tmp
131 	sub	\tmp, \idx, \limit
132 	bic	\tmp, \tmp, \idx
133 	and	\idx, \idx, \tmp, asr #63
134 	csdb
135 	.endm
136 
137 /*
138  * NOP sequence
139  */
140 	.macro	nops, num
141 	.rept	\num
142 	nop
143 	.endr
144 	.endm
145 
146 /*
147  * Emit an entry into the exception table
148  */
149 	.macro		_asm_extable, from, to
150 	.pushsection	__ex_table, "a"
151 	.align		3
152 	.long		(\from - .), (\to - .)
153 	.popsection
154 	.endm
155 
156 #define USER(l, x...)				\
157 9999:	x;					\
158 	_asm_extable	9999b, l
159 
160 /*
161  * Register aliases.
162  */
163 lr	.req	x30		// link register
164 
165 /*
166  * Vector entry
167  */
168 	 .macro	ventry	label
169 	.align	7
170 	b	\label
171 	.endm
172 
173 /*
174  * Select code when configured for BE.
175  */
176 #ifdef CONFIG_CPU_BIG_ENDIAN
177 #define CPU_BE(code...) code
178 #else
179 #define CPU_BE(code...)
180 #endif
181 
182 /*
183  * Select code when configured for LE.
184  */
185 #ifdef CONFIG_CPU_BIG_ENDIAN
186 #define CPU_LE(code...)
187 #else
188 #define CPU_LE(code...) code
189 #endif
190 
191 /*
192  * Define a macro that constructs a 64-bit value by concatenating two
193  * 32-bit registers. Note that on big endian systems the order of the
194  * registers is swapped.
195  */
196 #ifndef CONFIG_CPU_BIG_ENDIAN
197 	.macro	regs_to_64, rd, lbits, hbits
198 #else
199 	.macro	regs_to_64, rd, hbits, lbits
200 #endif
201 	orr	\rd, \lbits, \hbits, lsl #32
202 	.endm
203 
204 /*
205  * Pseudo-ops for PC-relative adr/ldr/str <reg>, <symbol> where
206  * <symbol> is within the range +/- 4 GB of the PC.
207  */
208 	/*
209 	 * @dst: destination register (64 bit wide)
210 	 * @sym: name of the symbol
211 	 */
212 	.macro	adr_l, dst, sym
213 	adrp	\dst, \sym
214 	add	\dst, \dst, :lo12:\sym
215 	.endm
216 
217 	/*
218 	 * @dst: destination register (32 or 64 bit wide)
219 	 * @sym: name of the symbol
220 	 * @tmp: optional 64-bit scratch register to be used if <dst> is a
221 	 *       32-bit wide register, in which case it cannot be used to hold
222 	 *       the address
223 	 */
224 	.macro	ldr_l, dst, sym, tmp=
225 	.ifb	\tmp
226 	adrp	\dst, \sym
227 	ldr	\dst, [\dst, :lo12:\sym]
228 	.else
229 	adrp	\tmp, \sym
230 	ldr	\dst, [\tmp, :lo12:\sym]
231 	.endif
232 	.endm
233 
234 	/*
235 	 * @src: source register (32 or 64 bit wide)
236 	 * @sym: name of the symbol
237 	 * @tmp: mandatory 64-bit scratch register to calculate the address
238 	 *       while <src> needs to be preserved.
239 	 */
240 	.macro	str_l, src, sym, tmp
241 	adrp	\tmp, \sym
242 	str	\src, [\tmp, :lo12:\sym]
243 	.endm
244 
245 	/*
246 	 * @dst: Result of per_cpu(sym, smp_processor_id()) (can be SP)
247 	 * @sym: The name of the per-cpu variable
248 	 * @tmp: scratch register
249 	 */
250 	.macro adr_this_cpu, dst, sym, tmp
251 	adrp	\tmp, \sym
252 	add	\dst, \tmp, #:lo12:\sym
253 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
254 	mrs	\tmp, tpidr_el1
255 alternative_else
256 	mrs	\tmp, tpidr_el2
257 alternative_endif
258 	add	\dst, \dst, \tmp
259 	.endm
260 
261 	/*
262 	 * @dst: Result of READ_ONCE(per_cpu(sym, smp_processor_id()))
263 	 * @sym: The name of the per-cpu variable
264 	 * @tmp: scratch register
265 	 */
266 	.macro ldr_this_cpu dst, sym, tmp
267 	adr_l	\dst, \sym
268 alternative_if_not ARM64_HAS_VIRT_HOST_EXTN
269 	mrs	\tmp, tpidr_el1
270 alternative_else
271 	mrs	\tmp, tpidr_el2
272 alternative_endif
273 	ldr	\dst, [\dst, \tmp]
274 	.endm
275 
276 /*
277  * vma_vm_mm - get mm pointer from vma pointer (vma->vm_mm)
278  */
279 	.macro	vma_vm_mm, rd, rn
280 	ldr	\rd, [\rn, #VMA_VM_MM]
281 	.endm
282 
283 /*
284  * mmid - get context id from mm pointer (mm->context.id)
285  */
286 	.macro	mmid, rd, rn
287 	ldr	\rd, [\rn, #MM_CONTEXT_ID]
288 	.endm
289 /*
290  * read_ctr - read CTR_EL0. If the system has mismatched register fields,
291  * provide the system wide safe value from arm64_ftr_reg_ctrel0.sys_val
292  */
293 	.macro	read_ctr, reg
294 alternative_if_not ARM64_MISMATCHED_CACHE_TYPE
295 	mrs	\reg, ctr_el0			// read CTR
296 	nop
297 alternative_else
298 	ldr_l	\reg, arm64_ftr_reg_ctrel0 + ARM64_FTR_SYSVAL
299 alternative_endif
300 	.endm
301 
302 
303 /*
304  * raw_dcache_line_size - get the minimum D-cache line size on this CPU
305  * from the CTR register.
306  */
307 	.macro	raw_dcache_line_size, reg, tmp
308 	mrs	\tmp, ctr_el0			// read CTR
309 	ubfm	\tmp, \tmp, #16, #19		// cache line size encoding
310 	mov	\reg, #4			// bytes per word
311 	lsl	\reg, \reg, \tmp		// actual cache line size
312 	.endm
313 
314 /*
315  * dcache_line_size - get the safe D-cache line size across all CPUs
316  */
317 	.macro	dcache_line_size, reg, tmp
318 	read_ctr	\tmp
319 	ubfm		\tmp, \tmp, #16, #19	// cache line size encoding
320 	mov		\reg, #4		// bytes per word
321 	lsl		\reg, \reg, \tmp	// actual cache line size
322 	.endm
323 
324 /*
325  * raw_icache_line_size - get the minimum I-cache line size on this CPU
326  * from the CTR register.
327  */
328 	.macro	raw_icache_line_size, reg, tmp
329 	mrs	\tmp, ctr_el0			// read CTR
330 	and	\tmp, \tmp, #0xf		// cache line size encoding
331 	mov	\reg, #4			// bytes per word
332 	lsl	\reg, \reg, \tmp		// actual cache line size
333 	.endm
334 
335 /*
336  * icache_line_size - get the safe I-cache line size across all CPUs
337  */
338 	.macro	icache_line_size, reg, tmp
339 	read_ctr	\tmp
340 	and		\tmp, \tmp, #0xf	// cache line size encoding
341 	mov		\reg, #4		// bytes per word
342 	lsl		\reg, \reg, \tmp	// actual cache line size
343 	.endm
344 
345 /*
346  * tcr_set_t0sz - update TCR.T0SZ so that we can load the ID map
347  */
348 	.macro	tcr_set_t0sz, valreg, t0sz
349 	bfi	\valreg, \t0sz, #TCR_T0SZ_OFFSET, #TCR_TxSZ_WIDTH
350 	.endm
351 
352 /*
353  * tcr_compute_pa_size - set TCR.(I)PS to the highest supported
354  * ID_AA64MMFR0_EL1.PARange value
355  *
356  *	tcr:		register with the TCR_ELx value to be updated
357  *	pos:		IPS or PS bitfield position
358  *	tmp{0,1}:	temporary registers
359  */
360 	.macro	tcr_compute_pa_size, tcr, pos, tmp0, tmp1
361 	mrs	\tmp0, ID_AA64MMFR0_EL1
362 	// Narrow PARange to fit the PS field in TCR_ELx
363 	ubfx	\tmp0, \tmp0, #ID_AA64MMFR0_PARANGE_SHIFT, #3
364 	mov	\tmp1, #ID_AA64MMFR0_PARANGE_MAX
365 	cmp	\tmp0, \tmp1
366 	csel	\tmp0, \tmp1, \tmp0, hi
367 	bfi	\tcr, \tmp0, \pos, #3
368 	.endm
369 
370 /*
371  * Macro to perform a data cache maintenance for the interval
372  * [kaddr, kaddr + size)
373  *
374  * 	op:		operation passed to dc instruction
375  * 	domain:		domain used in dsb instruciton
376  * 	kaddr:		starting virtual address of the region
377  * 	size:		size of the region
378  * 	Corrupts:	kaddr, size, tmp1, tmp2
379  */
380 	.macro __dcache_op_workaround_clean_cache, op, kaddr
381 alternative_if_not ARM64_WORKAROUND_CLEAN_CACHE
382 	dc	\op, \kaddr
383 alternative_else
384 	dc	civac, \kaddr
385 alternative_endif
386 	.endm
387 
388 	.macro dcache_by_line_op op, domain, kaddr, size, tmp1, tmp2
389 	dcache_line_size \tmp1, \tmp2
390 	add	\size, \kaddr, \size
391 	sub	\tmp2, \tmp1, #1
392 	bic	\kaddr, \kaddr, \tmp2
393 9998:
394 	.ifc	\op, cvau
395 	__dcache_op_workaround_clean_cache \op, \kaddr
396 	.else
397 	.ifc	\op, cvac
398 	__dcache_op_workaround_clean_cache \op, \kaddr
399 	.else
400 	.ifc	\op, cvap
401 	sys	3, c7, c12, 1, \kaddr	// dc cvap
402 	.else
403 	.ifc	\op, cvadp
404 	sys	3, c7, c13, 1, \kaddr	// dc cvadp
405 	.else
406 	dc	\op, \kaddr
407 	.endif
408 	.endif
409 	.endif
410 	.endif
411 	add	\kaddr, \kaddr, \tmp1
412 	cmp	\kaddr, \size
413 	b.lo	9998b
414 	dsb	\domain
415 	.endm
416 
417 /*
418  * Macro to perform an instruction cache maintenance for the interval
419  * [start, end)
420  *
421  * 	start, end:	virtual addresses describing the region
422  *	label:		A label to branch to on user fault.
423  * 	Corrupts:	tmp1, tmp2
424  */
425 	.macro invalidate_icache_by_line start, end, tmp1, tmp2, label
426 	icache_line_size \tmp1, \tmp2
427 	sub	\tmp2, \tmp1, #1
428 	bic	\tmp2, \start, \tmp2
429 9997:
430 USER(\label, ic	ivau, \tmp2)			// invalidate I line PoU
431 	add	\tmp2, \tmp2, \tmp1
432 	cmp	\tmp2, \end
433 	b.lo	9997b
434 	dsb	ish
435 	isb
436 	.endm
437 
438 /*
439  * reset_pmuserenr_el0 - reset PMUSERENR_EL0 if PMUv3 present
440  */
441 	.macro	reset_pmuserenr_el0, tmpreg
442 	mrs	\tmpreg, id_aa64dfr0_el1
443 	sbfx	\tmpreg, \tmpreg, #ID_AA64DFR0_PMUVER_SHIFT, #4
444 	cmp	\tmpreg, #1			// Skip if no PMU present
445 	b.lt	9000f
446 	msr	pmuserenr_el0, xzr		// Disable PMU access from EL0
447 9000:
448 	.endm
449 
450 /*
451  * copy_page - copy src to dest using temp registers t1-t8
452  */
453 	.macro copy_page dest:req src:req t1:req t2:req t3:req t4:req t5:req t6:req t7:req t8:req
454 9998:	ldp	\t1, \t2, [\src]
455 	ldp	\t3, \t4, [\src, #16]
456 	ldp	\t5, \t6, [\src, #32]
457 	ldp	\t7, \t8, [\src, #48]
458 	add	\src, \src, #64
459 	stnp	\t1, \t2, [\dest]
460 	stnp	\t3, \t4, [\dest, #16]
461 	stnp	\t5, \t6, [\dest, #32]
462 	stnp	\t7, \t8, [\dest, #48]
463 	add	\dest, \dest, #64
464 	tst	\src, #(PAGE_SIZE - 1)
465 	b.ne	9998b
466 	.endm
467 
468 /*
469  * Annotate a function as position independent, i.e., safe to be called before
470  * the kernel virtual mapping is activated.
471  */
472 #define ENDPIPROC(x)			\
473 	.globl	__pi_##x;		\
474 	.type 	__pi_##x, %function;	\
475 	.set	__pi_##x, x;		\
476 	.size	__pi_##x, . - x;	\
477 	ENDPROC(x)
478 
479 /*
480  * Annotate a function as being unsuitable for kprobes.
481  */
482 #ifdef CONFIG_KPROBES
483 #define NOKPROBE(x)				\
484 	.pushsection "_kprobe_blacklist", "aw";	\
485 	.quad	x;				\
486 	.popsection;
487 #else
488 #define NOKPROBE(x)
489 #endif
490 
491 #ifdef CONFIG_KASAN
492 #define EXPORT_SYMBOL_NOKASAN(name)
493 #else
494 #define EXPORT_SYMBOL_NOKASAN(name)	EXPORT_SYMBOL(name)
495 #endif
496 
497 	/*
498 	 * Emit a 64-bit absolute little endian symbol reference in a way that
499 	 * ensures that it will be resolved at build time, even when building a
500 	 * PIE binary. This requires cooperation from the linker script, which
501 	 * must emit the lo32/hi32 halves individually.
502 	 */
503 	.macro	le64sym, sym
504 	.long	\sym\()_lo32
505 	.long	\sym\()_hi32
506 	.endm
507 
508 	/*
509 	 * mov_q - move an immediate constant into a 64-bit register using
510 	 *         between 2 and 4 movz/movk instructions (depending on the
511 	 *         magnitude and sign of the operand)
512 	 */
513 	.macro	mov_q, reg, val
514 	.if (((\val) >> 31) == 0 || ((\val) >> 31) == 0x1ffffffff)
515 	movz	\reg, :abs_g1_s:\val
516 	.else
517 	.if (((\val) >> 47) == 0 || ((\val) >> 47) == 0x1ffff)
518 	movz	\reg, :abs_g2_s:\val
519 	.else
520 	movz	\reg, :abs_g3:\val
521 	movk	\reg, :abs_g2_nc:\val
522 	.endif
523 	movk	\reg, :abs_g1_nc:\val
524 	.endif
525 	movk	\reg, :abs_g0_nc:\val
526 	.endm
527 
528 /*
529  * Return the current task_struct.
530  */
531 	.macro	get_current_task, rd
532 	mrs	\rd, sp_el0
533 	.endm
534 
535 /*
536  * Offset ttbr1 to allow for 48-bit kernel VAs set with 52-bit PTRS_PER_PGD.
537  * orr is used as it can cover the immediate value (and is idempotent).
538  * In future this may be nop'ed out when dealing with 52-bit kernel VAs.
539  * 	ttbr: Value of ttbr to set, modified.
540  */
541 	.macro	offset_ttbr1, ttbr
542 #ifdef CONFIG_ARM64_USER_VA_BITS_52
543 	orr	\ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
544 #endif
545 	.endm
546 
547 /*
548  * Perform the reverse of offset_ttbr1.
549  * bic is used as it can cover the immediate value and, in future, won't need
550  * to be nop'ed out when dealing with 52-bit kernel VAs.
551  */
552 	.macro	restore_ttbr1, ttbr
553 #ifdef CONFIG_ARM64_USER_VA_BITS_52
554 	bic	\ttbr, \ttbr, #TTBR1_BADDR_4852_OFFSET
555 #endif
556 	.endm
557 
558 /*
559  * Arrange a physical address in a TTBR register, taking care of 52-bit
560  * addresses.
561  *
562  * 	phys:	physical address, preserved
563  * 	ttbr:	returns the TTBR value
564  */
565 	.macro	phys_to_ttbr, ttbr, phys
566 #ifdef CONFIG_ARM64_PA_BITS_52
567 	orr	\ttbr, \phys, \phys, lsr #46
568 	and	\ttbr, \ttbr, #TTBR_BADDR_MASK_52
569 #else
570 	mov	\ttbr, \phys
571 #endif
572 	.endm
573 
574 	.macro	phys_to_pte, pte, phys
575 #ifdef CONFIG_ARM64_PA_BITS_52
576 	/*
577 	 * We assume \phys is 64K aligned and this is guaranteed by only
578 	 * supporting this configuration with 64K pages.
579 	 */
580 	orr	\pte, \phys, \phys, lsr #36
581 	and	\pte, \pte, #PTE_ADDR_MASK
582 #else
583 	mov	\pte, \phys
584 #endif
585 	.endm
586 
587 	.macro	pte_to_phys, phys, pte
588 #ifdef CONFIG_ARM64_PA_BITS_52
589 	ubfiz	\phys, \pte, #(48 - 16 - 12), #16
590 	bfxil	\phys, \pte, #16, #32
591 	lsl	\phys, \phys, #16
592 #else
593 	and	\phys, \pte, #PTE_ADDR_MASK
594 #endif
595 	.endm
596 
597 /*
598  * tcr_clear_errata_bits - Clear TCR bits that trigger an errata on this CPU.
599  */
600 	.macro	tcr_clear_errata_bits, tcr, tmp1, tmp2
601 #ifdef CONFIG_FUJITSU_ERRATUM_010001
602 	mrs	\tmp1, midr_el1
603 
604 	mov_q	\tmp2, MIDR_FUJITSU_ERRATUM_010001_MASK
605 	and	\tmp1, \tmp1, \tmp2
606 	mov_q	\tmp2, MIDR_FUJITSU_ERRATUM_010001
607 	cmp	\tmp1, \tmp2
608 	b.ne	10f
609 
610 	mov_q	\tmp2, TCR_CLEAR_FUJITSU_ERRATUM_010001
611 	bic	\tcr, \tcr, \tmp2
612 10:
613 #endif /* CONFIG_FUJITSU_ERRATUM_010001 */
614 	.endm
615 
616 /**
617  * Errata workaround prior to disable MMU. Insert an ISB immediately prior
618  * to executing the MSR that will change SCTLR_ELn[M] from a value of 1 to 0.
619  */
620 	.macro pre_disable_mmu_workaround
621 #ifdef CONFIG_QCOM_FALKOR_ERRATUM_E1041
622 	isb
623 #endif
624 	.endm
625 
626 	/*
627 	 * frame_push - Push @regcount callee saved registers to the stack,
628 	 *              starting at x19, as well as x29/x30, and set x29 to
629 	 *              the new value of sp. Add @extra bytes of stack space
630 	 *              for locals.
631 	 */
632 	.macro		frame_push, regcount:req, extra
633 	__frame		st, \regcount, \extra
634 	.endm
635 
636 	/*
637 	 * frame_pop  - Pop the callee saved registers from the stack that were
638 	 *              pushed in the most recent call to frame_push, as well
639 	 *              as x29/x30 and any extra stack space that may have been
640 	 *              allocated.
641 	 */
642 	.macro		frame_pop
643 	__frame		ld
644 	.endm
645 
646 	.macro		__frame_regs, reg1, reg2, op, num
647 	.if		.Lframe_regcount == \num
648 	\op\()r		\reg1, [sp, #(\num + 1) * 8]
649 	.elseif		.Lframe_regcount > \num
650 	\op\()p		\reg1, \reg2, [sp, #(\num + 1) * 8]
651 	.endif
652 	.endm
653 
654 	.macro		__frame, op, regcount, extra=0
655 	.ifc		\op, st
656 	.if		(\regcount) < 0 || (\regcount) > 10
657 	.error		"regcount should be in the range [0 ... 10]"
658 	.endif
659 	.if		((\extra) % 16) != 0
660 	.error		"extra should be a multiple of 16 bytes"
661 	.endif
662 	.ifdef		.Lframe_regcount
663 	.if		.Lframe_regcount != -1
664 	.error		"frame_push/frame_pop may not be nested"
665 	.endif
666 	.endif
667 	.set		.Lframe_regcount, \regcount
668 	.set		.Lframe_extra, \extra
669 	.set		.Lframe_local_offset, ((\regcount + 3) / 2) * 16
670 	stp		x29, x30, [sp, #-.Lframe_local_offset - .Lframe_extra]!
671 	mov		x29, sp
672 	.endif
673 
674 	__frame_regs	x19, x20, \op, 1
675 	__frame_regs	x21, x22, \op, 3
676 	__frame_regs	x23, x24, \op, 5
677 	__frame_regs	x25, x26, \op, 7
678 	__frame_regs	x27, x28, \op, 9
679 
680 	.ifc		\op, ld
681 	.if		.Lframe_regcount == -1
682 	.error		"frame_push/frame_pop may not be nested"
683 	.endif
684 	ldp		x29, x30, [sp], #.Lframe_local_offset + .Lframe_extra
685 	.set		.Lframe_regcount, -1
686 	.endif
687 	.endm
688 
689 /*
690  * Check whether to yield to another runnable task from kernel mode NEON code
691  * (which runs with preemption disabled).
692  *
693  * if_will_cond_yield_neon
694  *        // pre-yield patchup code
695  * do_cond_yield_neon
696  *        // post-yield patchup code
697  * endif_yield_neon    <label>
698  *
699  * where <label> is optional, and marks the point where execution will resume
700  * after a yield has been performed. If omitted, execution resumes right after
701  * the endif_yield_neon invocation. Note that the entire sequence, including
702  * the provided patchup code, will be omitted from the image if CONFIG_PREEMPT
703  * is not defined.
704  *
705  * As a convenience, in the case where no patchup code is required, the above
706  * sequence may be abbreviated to
707  *
708  * cond_yield_neon <label>
709  *
710  * Note that the patchup code does not support assembler directives that change
711  * the output section, any use of such directives is undefined.
712  *
713  * The yield itself consists of the following:
714  * - Check whether the preempt count is exactly 1 and a reschedule is also
715  *   needed. If so, calling of preempt_enable() in kernel_neon_end() will
716  *   trigger a reschedule. If it is not the case, yielding is pointless.
717  * - Disable and re-enable kernel mode NEON, and branch to the yield fixup
718  *   code.
719  *
720  * This macro sequence may clobber all CPU state that is not guaranteed by the
721  * AAPCS to be preserved across an ordinary function call.
722  */
723 
724 	.macro		cond_yield_neon, lbl
725 	if_will_cond_yield_neon
726 	do_cond_yield_neon
727 	endif_yield_neon	\lbl
728 	.endm
729 
730 	.macro		if_will_cond_yield_neon
731 #ifdef CONFIG_PREEMPT
732 	get_current_task	x0
733 	ldr		x0, [x0, #TSK_TI_PREEMPT]
734 	sub		x0, x0, #PREEMPT_DISABLE_OFFSET
735 	cbz		x0, .Lyield_\@
736 	/* fall through to endif_yield_neon */
737 	.subsection	1
738 .Lyield_\@ :
739 #else
740 	.section	".discard.cond_yield_neon", "ax"
741 #endif
742 	.endm
743 
744 	.macro		do_cond_yield_neon
745 	bl		kernel_neon_end
746 	bl		kernel_neon_begin
747 	.endm
748 
749 	.macro		endif_yield_neon, lbl
750 	.ifnb		\lbl
751 	b		\lbl
752 	.else
753 	b		.Lyield_out_\@
754 	.endif
755 	.previous
756 .Lyield_out_\@ :
757 	.endm
758 
759 #endif	/* __ASM_ASSEMBLER_H */
760