1 /* SPDX-License-Identifier: GPL-2.0 */
2 /*
3 * Copyright (C) 2024 Arm Limited.
4 * All rights reserved.
5 */
6
7 #include <libcflat.h>
8 #include <alloc_page.h>
9 #include <stdlib.h>
10
11 #include <asm/mmu.h>
12 #include <asm/pgtable-hwdef.h>
13 #include <asm/processor.h>
14 #include <asm/sysreg.h>
15 #include <asm/thread_info.h>
16
17
18 /* Tag Check Faults cause a synchronous exception */
19 #define MTE_TCF_SYNC 0b01
20 /* Tag Check Faults are asynchronously accumulated */
21 #define MTE_TCF_ASYNC 0b10
22 /*
23 * Tag Check Faults cause a synchronous exception on reads,
24 * and are asynchronously accumulated on writes
25 */
26 #define MTE_TCF_ASYMM 0b11
27
28 #define MTE_GRANULE_SIZE UL(16)
29 #define MTE_GRANULE_MASK (~(MTE_GRANULE_SIZE - 1))
30 #define MTE_TAG_SHIFT 56
31
32 #define untagged(p) \
33 ({ \
34 unsigned long __in = (unsigned long)(p); \
35 typeof(p) __out = (typeof(p))(__in & ~(MTE_GRANULE_MASK << MTE_TAG_SHIFT)); \
36 \
37 __out; \
38 })
39
40 #define tagged(p, t) \
41 ({ \
42 unsigned long __in = (unsigned long)(untagged(p)); \
43 unsigned long __tag = (unsigned long)(t) << MTE_TAG_SHIFT; \
44 typeof(p) __out = (typeof(p))(__in | __tag); \
45 \
46 __out; \
47 })
48
49 /*
50 * If we use a normal (non hand coded inline assembly) load or store
51 * to access a tagged address, the compiler will reasonably assume
52 * that the access succeeded, and the next instruction may do
53 * something based on that assumption.
54 *
55 * But a test might want the tagged access to fail on purpose, and if
56 * we advance the PC to the next instruction, the one added by the
57 * compiler, we might leave the program in an unexpected state.
58 */
mem_read(unsigned int * addr,unsigned int * res)59 static inline void mem_read(unsigned int *addr, unsigned int *res)
60 {
61 unsigned int r;
62
63 asm volatile ("ldr %w0,[%1]\n"
64 "str %w0,[%2]\n"
65 : "=&r" (r)
66 : "r" (addr), "r" (res) : "memory");
67 }
68
mem_write(unsigned int * addr,unsigned int val)69 static inline void mem_write(unsigned int *addr, unsigned int val)
70 {
71 /* The NOP allows the same exception handler as mem_read() to be used. */
72 asm volatile ("str %w0,[%1]\n"
73 "nop\n"
74 :
75 : "r" (val), "r" (addr)
76 : "memory");
77 }
78
79 static volatile bool mte_exception;
80
mte_fault_handler(struct pt_regs * regs,unsigned int esr)81 static void mte_fault_handler(struct pt_regs *regs, unsigned int esr)
82 {
83 unsigned int dfsc = esr & GENMASK(5, 0);
84 unsigned int fnv = esr & BIT(10);
85
86 if (dfsc == 0b010001) {
87 if (fnv)
88 report_info("Unexpected non-zero FnV");
89 mte_exception = true;
90 }
91
92 /*
93 * mem_read() reads the value from the tagged pointer, then
94 * stores this value in the untagged 'res' pointer. The
95 * function that called mem_read() will want to check that the
96 * initial value of 'res' hasn't changed if a tag check fault
97 * is reported. Skip over two instructions so 'res' isn't
98 * overwritten.
99 */
100 regs->pc += 8;
101 }
102
mmu_set_tagged(pgd_t * pgtable,unsigned long vaddr)103 static inline void mmu_set_tagged(pgd_t *pgtable, unsigned long vaddr)
104 {
105 pteval_t *p_pte = follow_pte(pgtable, untagged(vaddr));
106
107 if (p_pte) {
108 pteval_t entry = *p_pte;
109
110 entry &= ~PTE_ATTRINDX_MASK;
111 entry |= PTE_ATTRINDX(MT_NORMAL_TAGGED);
112
113 WRITE_ONCE(*p_pte, entry);
114 flush_tlb_page(vaddr);
115 } else {
116 report_abort("Cannot find PTE");
117 }
118 }
119
mte_init(void)120 static void mte_init(void)
121 {
122 unsigned long sctlr = read_sysreg(sctlr_el1);
123 unsigned long tcr = read_sysreg(tcr_el1);
124
125 sctlr &= ~SCTLR_EL1_TCF_MASK;
126 sctlr |= SCTLR_EL1_ATA;
127
128 tcr &= ~TCR_TCMA0;
129 tcr |= TCR_TBI0;
130
131 write_sysreg(sctlr, sctlr_el1);
132 write_sysreg(tcr, tcr_el1);
133
134 isb();
135 flush_tlb_all();
136 }
137
mte_set_tcf(unsigned long tcf)138 static inline unsigned long mte_set_tcf(unsigned long tcf)
139 {
140 unsigned long sctlr = read_sysreg(sctlr_el1);
141 unsigned long old = (sctlr & SCTLR_EL1_TCF_MASK) >> SCTLR_EL1_TCF_SHIFT;
142
143 sctlr &= ~(SCTLR_EL1_TCF_MASK | SCTLR_EL1_TCF0_MASK);
144 sctlr |= (tcf << SCTLR_EL1_TCF_SHIFT) & SCTLR_EL1_TCF_MASK;
145
146 write_sysreg(sctlr, sctlr_el1);
147 write_sysreg_s(0, TFSR_EL1);
148 isb();
149
150 return old;
151 }
152
mte_set_tag(void * addr,size_t size,unsigned int tag)153 static inline void mte_set_tag(void *addr, size_t size, unsigned int tag)
154 {
155 #ifdef CC_HAS_MTE
156 unsigned long in = (unsigned long)untagged(addr);
157 unsigned long start = ALIGN_DOWN(in, 16);
158 unsigned long end = ALIGN(in + size, 16);
159
160 for (unsigned long ptr = start; ptr < end; ptr += 16) {
161 asm volatile(".arch armv8.5-a+memtag\n"
162 "stg %0, [%0]"
163 :
164 : "r"(tagged(ptr, tag))
165 : "memory");
166 }
167 #endif
168 }
169
get_clear_tfsr(void)170 static inline unsigned long get_clear_tfsr(void)
171 {
172 unsigned long r;
173
174 dsb(nsh);
175 isb();
176
177 r = read_sysreg_s(TFSR_EL1);
178 write_sysreg_s(0, TFSR_EL1);
179
180 return r;
181 }
182
mte_sync_test(void)183 static void mte_sync_test(void)
184 {
185 unsigned int *mem = tagged(alloc_page(), 1);
186 unsigned int val = 0;
187
188 mmu_set_tagged(current_thread_info()->pgtable, (unsigned long)mem);
189 mte_set_tag(mem, PAGE_SIZE, 1);
190 memset(mem, 0xff, PAGE_SIZE);
191 mte_set_tcf(MTE_TCF_SYNC);
192
193 mte_exception = false;
194
195 install_exception_handler(EL1H_SYNC, ESR_EL1_EC_DABT_EL1, mte_fault_handler);
196
197 mem_read(tagged(mem, 2), &val);
198
199 report((val == 0) && mte_exception && (get_clear_tfsr() == 0), "read");
200
201 mte_exception = false;
202
203 mem_write(tagged(mem, 3), 0xbbbbbbbb);
204
205 report((*mem == 0xffffffff) && mte_exception && (get_clear_tfsr() == 0), "write");
206
207 free_page(untagged(mem));
208 }
209
mte_asymm_test(void)210 static void mte_asymm_test(void)
211 {
212 unsigned int *mem = tagged(alloc_page(), 2);
213 unsigned int val = 0;
214
215 mmu_set_tagged(current_thread_info()->pgtable, (unsigned long)mem);
216 mte_set_tag(mem, PAGE_SIZE, 2);
217 memset(mem, 0xff, PAGE_SIZE);
218 mte_set_tcf(MTE_TCF_ASYMM);
219 mte_exception = false;
220
221 install_exception_handler(EL1H_SYNC, ESR_EL1_EC_DABT_EL1, mte_fault_handler);
222
223 mem_read(tagged(mem, 3), &val);
224 report((val == 0) && mte_exception && (get_clear_tfsr() == 0), "read");
225
226 install_exception_handler(EL1H_SYNC, ESR_EL1_EC_DABT_EL1, NULL);
227
228 mem_write(tagged(mem, 4), 0xaaaaaaaa);
229 report((*mem == 0xaaaaaaaa) && (get_clear_tfsr() == TFSR_EL1_TF0), "write");
230
231 free_page(untagged(mem));
232 }
233
mte_async_test(void)234 static void mte_async_test(void)
235 {
236 unsigned int *mem = tagged(alloc_page(), 3);
237 unsigned int val = 0;
238
239 mmu_set_tagged(current_thread_info()->pgtable, (unsigned long)mem);
240 mte_set_tag(mem, PAGE_SIZE, 3);
241 memset(mem, 0xff, PAGE_SIZE);
242 mte_set_tcf(MTE_TCF_ASYNC);
243
244 mem_read(tagged(mem, 4), &val);
245 report((val == 0xffffffff) && (get_clear_tfsr() == TFSR_EL1_TF0), "read");
246
247 mem_write(tagged(mem, 5), 0xcccccccc);
248 report((*mem == 0xcccccccc) && (get_clear_tfsr() == TFSR_EL1_TF0), "write");
249
250 free_page(untagged(mem));
251 }
252
mte_version(void)253 static unsigned int mte_version(void)
254 {
255 #ifdef CC_HAS_MTE
256 uint64_t r;
257
258 asm volatile("mrs %x0, id_aa64pfr1_el1" : "=r"(r));
259
260 return (r >> ID_AA64PFR1_EL1_MTE_SHIFT) & 0b1111;
261 #else
262 report_info("Compiler lack MTE support");
263 return 0;
264 #endif
265 }
266
main(int argc,char * argv[])267 int main(int argc, char *argv[])
268 {
269
270 unsigned int version = mte_version();
271
272 if (version < 2) {
273 report_skip("No MTE support, skip...\n");
274 return report_summary();
275 }
276
277 if (argc < 2)
278 report_abort("no test specified");
279
280 report_prefix_push("mte");
281
282 mte_init();
283
284 if (strcmp(argv[1], "sync") == 0) {
285 report_prefix_push(argv[1]);
286 mte_sync_test();
287 report_prefix_pop();
288 } else if (strcmp(argv[1], "async") == 0) {
289 report_prefix_push(argv[1]);
290 if (version < 3) {
291 report_skip("No MTE async, skip...\n");
292 return report_summary();
293 }
294 mte_async_test();
295 report_prefix_pop();
296 } else if (strcmp(argv[1], "asymm") == 0) {
297 report_prefix_push(argv[1]);
298 if (version < 3) {
299 report_skip("No MTE asymm, skip...\n");
300 return report_summary();
301 }
302 mte_asymm_test();
303 report_prefix_pop();
304 } else {
305 report_abort("Unknown sub-test '%s'", argv[1]);
306 }
307
308 return report_summary();
309 }
310