1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Standard user space access functions based on mvcp/mvcs and doing
4 * interesting things in the secondary space mode.
5 *
6 * Copyright IBM Corp. 2006,2014
7 * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com),
8 * Gerald Schaefer (gerald.schaefer@de.ibm.com)
9 */
10
11 #include <linux/kprobes.h>
12 #include <linux/uaccess.h>
13 #include <linux/export.h>
14 #include <linux/mm.h>
15 #include <asm/asm-extable.h>
16 #include <asm/ctlreg.h>
17 #include <asm/skey.h>
18
19 #ifdef CONFIG_DEBUG_ENTRY
debug_user_asce(int exit)20 void debug_user_asce(int exit)
21 {
22 struct lowcore *lc = get_lowcore();
23 struct ctlreg cr1, cr7;
24
25 local_ctl_store(1, &cr1);
26 local_ctl_store(7, &cr7);
27 if (cr1.val == lc->user_asce.val && cr7.val == lc->user_asce.val)
28 return;
29 panic("incorrect ASCE on kernel %s\n"
30 "cr1: %016lx cr7: %016lx\n"
31 "kernel: %016lx user: %016lx\n",
32 exit ? "exit" : "entry", cr1.val, cr7.val,
33 lc->kernel_asce.val, lc->user_asce.val);
34 }
35 #endif /*CONFIG_DEBUG_ENTRY */
36
37 #define CMPXCHG_USER_KEY_MAX_LOOPS 128
38
__cmpxchg_key_small(void * address,unsigned int * uval,unsigned int old,unsigned int new,unsigned int mask,unsigned long key)39 static nokprobe_inline int __cmpxchg_key_small(void *address, unsigned int *uval,
40 unsigned int old, unsigned int new,
41 unsigned int mask, unsigned long key)
42 {
43 unsigned long count;
44 unsigned int prev;
45 int rc = 0;
46
47 skey_regions_initialize();
48 asm_inline volatile(
49 "20: spka 0(%[key])\n"
50 " llill %[count],%[max_loops]\n"
51 "0: l %[prev],%[address]\n"
52 "1: nr %[prev],%[mask]\n"
53 " xilf %[mask],0xffffffff\n"
54 " or %[new],%[prev]\n"
55 " or %[prev],%[tmp]\n"
56 "2: lr %[tmp],%[prev]\n"
57 "3: cs %[prev],%[new],%[address]\n"
58 "4: jnl 5f\n"
59 " xr %[tmp],%[prev]\n"
60 " xr %[new],%[tmp]\n"
61 " nr %[tmp],%[mask]\n"
62 " jnz 5f\n"
63 " brct %[count],2b\n"
64 "5: spka %[default_key]\n"
65 "21:\n"
66 EX_TABLE_UA_LOAD_REG(0b, 5b, %[rc], %[prev])
67 EX_TABLE_UA_LOAD_REG(1b, 5b, %[rc], %[prev])
68 EX_TABLE_UA_LOAD_REG(3b, 5b, %[rc], %[prev])
69 EX_TABLE_UA_LOAD_REG(4b, 5b, %[rc], %[prev])
70 SKEY_REGION(20b, 21b)
71 : [rc] "+&d" (rc),
72 [prev] "=&d" (prev),
73 [address] "+Q" (*(int *)address),
74 [tmp] "+&d" (old),
75 [new] "+&d" (new),
76 [mask] "+&d" (mask),
77 [count] "=a" (count)
78 : [key] "%[count]" (key << 4),
79 [default_key] "J" (PAGE_DEFAULT_KEY),
80 [max_loops] "J" (CMPXCHG_USER_KEY_MAX_LOOPS)
81 : "memory", "cc");
82 *uval = prev;
83 if (!count)
84 rc = -EAGAIN;
85 return rc;
86 }
87
__cmpxchg_key1(void * addr,unsigned char * uval,unsigned char old,unsigned char new,unsigned long key)88 int __kprobes __cmpxchg_key1(void *addr, unsigned char *uval, unsigned char old,
89 unsigned char new, unsigned long key)
90 {
91 unsigned long address = (unsigned long)addr;
92 unsigned int prev, shift, mask, _old, _new;
93 int rc;
94
95 shift = (3 ^ (address & 3)) << 3;
96 address ^= address & 3;
97 _old = (unsigned int)old << shift;
98 _new = (unsigned int)new << shift;
99 mask = ~(0xff << shift);
100 rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key);
101 *uval = prev >> shift;
102 return rc;
103 }
104 EXPORT_SYMBOL(__cmpxchg_key1);
105
__cmpxchg_key2(void * addr,unsigned short * uval,unsigned short old,unsigned short new,unsigned long key)106 int __kprobes __cmpxchg_key2(void *addr, unsigned short *uval, unsigned short old,
107 unsigned short new, unsigned long key)
108 {
109 unsigned long address = (unsigned long)addr;
110 unsigned int prev, shift, mask, _old, _new;
111 int rc;
112
113 shift = (2 ^ (address & 2)) << 3;
114 address ^= address & 2;
115 _old = (unsigned int)old << shift;
116 _new = (unsigned int)new << shift;
117 mask = ~(0xffff << shift);
118 rc = __cmpxchg_key_small((void *)address, &prev, _old, _new, mask, key);
119 *uval = prev >> shift;
120 return rc;
121 }
122 EXPORT_SYMBOL(__cmpxchg_key2);
123
__cmpxchg_key4(void * address,unsigned int * uval,unsigned int old,unsigned int new,unsigned long key)124 int __kprobes __cmpxchg_key4(void *address, unsigned int *uval, unsigned int old,
125 unsigned int new, unsigned long key)
126 {
127 unsigned int prev = old;
128 int rc = 0;
129
130 skey_regions_initialize();
131 asm_inline volatile(
132 "20: spka 0(%[key])\n"
133 "0: cs %[prev],%[new],%[address]\n"
134 "1: spka %[default_key]\n"
135 "21:\n"
136 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
137 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
138 SKEY_REGION(20b, 21b)
139 : [rc] "+&d" (rc),
140 [prev] "+&d" (prev),
141 [address] "+Q" (*(int *)address)
142 : [new] "d" (new),
143 [key] "a" (key << 4),
144 [default_key] "J" (PAGE_DEFAULT_KEY)
145 : "memory", "cc");
146 *uval = prev;
147 return rc;
148 }
149 EXPORT_SYMBOL(__cmpxchg_key4);
150
__cmpxchg_key8(void * address,unsigned long * uval,unsigned long old,unsigned long new,unsigned long key)151 int __kprobes __cmpxchg_key8(void *address, unsigned long *uval, unsigned long old,
152 unsigned long new, unsigned long key)
153 {
154 unsigned long prev = old;
155 int rc = 0;
156
157 skey_regions_initialize();
158 asm_inline volatile(
159 "20: spka 0(%[key])\n"
160 "0: csg %[prev],%[new],%[address]\n"
161 "1: spka %[default_key]\n"
162 "21:\n"
163 EX_TABLE_UA_LOAD_REG(0b, 1b, %[rc], %[prev])
164 EX_TABLE_UA_LOAD_REG(1b, 1b, %[rc], %[prev])
165 SKEY_REGION(20b, 21b)
166 : [rc] "+&d" (rc),
167 [prev] "+&d" (prev),
168 [address] "+QS" (*(long *)address)
169 : [new] "d" (new),
170 [key] "a" (key << 4),
171 [default_key] "J" (PAGE_DEFAULT_KEY)
172 : "memory", "cc");
173 *uval = prev;
174 return rc;
175 }
176 EXPORT_SYMBOL(__cmpxchg_key8);
177
__cmpxchg_key16(void * address,__uint128_t * uval,__uint128_t old,__uint128_t new,unsigned long key)178 int __kprobes __cmpxchg_key16(void *address, __uint128_t *uval, __uint128_t old,
179 __uint128_t new, unsigned long key)
180 {
181 __uint128_t prev = old;
182 int rc = 0;
183
184 skey_regions_initialize();
185 asm_inline volatile(
186 "20: spka 0(%[key])\n"
187 "0: cdsg %[prev],%[new],%[address]\n"
188 "1: spka %[default_key]\n"
189 "21:\n"
190 EX_TABLE_UA_LOAD_REGPAIR(0b, 1b, %[rc], %[prev])
191 EX_TABLE_UA_LOAD_REGPAIR(1b, 1b, %[rc], %[prev])
192 SKEY_REGION(20b, 21b)
193 : [rc] "+&d" (rc),
194 [prev] "+&d" (prev),
195 [address] "+QS" (*(__int128_t *)address)
196 : [new] "d" (new),
197 [key] "a" (key << 4),
198 [default_key] "J" (PAGE_DEFAULT_KEY)
199 : "memory", "cc");
200 *uval = prev;
201 return rc;
202 }
203 EXPORT_SYMBOL(__cmpxchg_key16);
204