1/* SPDX-License-Identifier: GPL-2.0 */
2/*
3 * __get_user functions.
4 *
5 * (C) Copyright 1998 Linus Torvalds
6 * (C) Copyright 2005 Andi Kleen
7 * (C) Copyright 2008 Glauber Costa
8 *
9 * These functions have a non-standard call interface
10 * to make them more efficient, especially as they
11 * return an error value in addition to the "real"
12 * return value.
13 */
14
15/*
16 * __get_user_X
17 *
18 * Inputs:	%[r|e]ax contains the address.
19 *
20 * Outputs:	%[r|e]ax is error code (0 or -EFAULT)
21 *		%[r|e]dx contains zero-extended value
22 *		%ecx contains the high half for 32-bit __get_user_8
23 *
24 *
25 * These functions should not modify any other registers,
26 * as they get called from within inline assembly.
27 */
28
29#include <linux/export.h>
30#include <linux/linkage.h>
31#include <linux/objtool.h>
32#include <asm/page_types.h>
33#include <asm/errno.h>
34#include <asm/asm-offsets.h>
35#include <asm/thread_info.h>
36#include <asm/asm.h>
37#include <asm/smap.h>
38#include <asm/runtime-const.h>
39
40#define ASM_BARRIER_NOSPEC ALTERNATIVE "", "lfence", X86_FEATURE_LFENCE_RDTSC
41
42.macro check_range size:req
43.if IS_ENABLED(CONFIG_X86_64)
44	RUNTIME_CONST_PTR USER_PTR_MAX, rdx
45	cmp %rdx, %rax
46	cmova %rdx, %rax
47.else
48	cmp $TASK_SIZE_MAX-\size+1, %eax
49	jae .Lbad_get_user
50	sbb %edx, %edx		/* array_index_mask_nospec() */
51	and %edx, %eax
52.endif
53.endm
54
55.macro UACCESS op src dst
561:	\op \src,\dst
57	_ASM_EXTABLE_UA(1b, __get_user_handle_exception)
58.endm
59
60
61	.text
62SYM_FUNC_START(__get_user_1)
63	ANNOTATE_NOENDBR
64	check_range size=1
65	ASM_STAC
66	UACCESS movzbl (%_ASM_AX),%edx
67	xor %eax,%eax
68	ASM_CLAC
69	RET
70SYM_FUNC_END(__get_user_1)
71EXPORT_SYMBOL(__get_user_1)
72
73SYM_FUNC_START(__get_user_2)
74	ANNOTATE_NOENDBR
75	check_range size=2
76	ASM_STAC
77	UACCESS movzwl (%_ASM_AX),%edx
78	xor %eax,%eax
79	ASM_CLAC
80	RET
81SYM_FUNC_END(__get_user_2)
82EXPORT_SYMBOL(__get_user_2)
83
84SYM_FUNC_START(__get_user_4)
85	ANNOTATE_NOENDBR
86	check_range size=4
87	ASM_STAC
88	UACCESS movl (%_ASM_AX),%edx
89	xor %eax,%eax
90	ASM_CLAC
91	RET
92SYM_FUNC_END(__get_user_4)
93EXPORT_SYMBOL(__get_user_4)
94
95SYM_FUNC_START(__get_user_8)
96	ANNOTATE_NOENDBR
97#ifndef CONFIG_X86_64
98	xor %ecx,%ecx
99#endif
100	check_range size=8
101	ASM_STAC
102#ifdef CONFIG_X86_64
103	UACCESS movq (%_ASM_AX),%rdx
104#else
105	UACCESS movl (%_ASM_AX),%edx
106	UACCESS movl 4(%_ASM_AX),%ecx
107#endif
108	xor %eax,%eax
109	ASM_CLAC
110	RET
111SYM_FUNC_END(__get_user_8)
112EXPORT_SYMBOL(__get_user_8)
113
114/* .. and the same for __get_user, just without the range checks */
115SYM_FUNC_START(__get_user_nocheck_1)
116	ANNOTATE_NOENDBR
117	ASM_STAC
118	ASM_BARRIER_NOSPEC
119	UACCESS movzbl (%_ASM_AX),%edx
120	xor %eax,%eax
121	ASM_CLAC
122	RET
123SYM_FUNC_END(__get_user_nocheck_1)
124EXPORT_SYMBOL(__get_user_nocheck_1)
125
126SYM_FUNC_START(__get_user_nocheck_2)
127	ANNOTATE_NOENDBR
128	ASM_STAC
129	ASM_BARRIER_NOSPEC
130	UACCESS movzwl (%_ASM_AX),%edx
131	xor %eax,%eax
132	ASM_CLAC
133	RET
134SYM_FUNC_END(__get_user_nocheck_2)
135EXPORT_SYMBOL(__get_user_nocheck_2)
136
137SYM_FUNC_START(__get_user_nocheck_4)
138	ANNOTATE_NOENDBR
139	ASM_STAC
140	ASM_BARRIER_NOSPEC
141	UACCESS movl (%_ASM_AX),%edx
142	xor %eax,%eax
143	ASM_CLAC
144	RET
145SYM_FUNC_END(__get_user_nocheck_4)
146EXPORT_SYMBOL(__get_user_nocheck_4)
147
148SYM_FUNC_START(__get_user_nocheck_8)
149	ANNOTATE_NOENDBR
150	ASM_STAC
151	ASM_BARRIER_NOSPEC
152#ifdef CONFIG_X86_64
153	UACCESS movq (%_ASM_AX),%rdx
154#else
155	xor %ecx,%ecx
156	UACCESS movl (%_ASM_AX),%edx
157	UACCESS movl 4(%_ASM_AX),%ecx
158#endif
159	xor %eax,%eax
160	ASM_CLAC
161	RET
162SYM_FUNC_END(__get_user_nocheck_8)
163EXPORT_SYMBOL(__get_user_nocheck_8)
164
165
166SYM_CODE_START_LOCAL(__get_user_handle_exception)
167	ASM_CLAC
168.Lbad_get_user:
169	xor %edx,%edx
170	mov $(-EFAULT),%_ASM_AX
171	RET
172SYM_CODE_END(__get_user_handle_exception)
173