1 /* SPDX-License-Identifier: GPL-2.0 */ 2 /* 3 * Copyright (C) 2024 Arm Limited. 4 * All rights reserved. 5 */ 6 7 #include <libcflat.h> 8 #include <alloc_page.h> 9 #include <stdlib.h> 10 11 #include <asm/mmu.h> 12 #include <asm/pgtable-hwdef.h> 13 #include <asm/processor.h> 14 #include <asm/sysreg.h> 15 #include <asm/thread_info.h> 16 17 18 /* Tag Check Faults cause a synchronous exception */ 19 #define MTE_TCF_SYNC 0b01 20 /* Tag Check Faults are asynchronously accumulated */ 21 #define MTE_TCF_ASYNC 0b10 22 /* 23 * Tag Check Faults cause a synchronous exception on reads, 24 * and are asynchronously accumulated on writes 25 */ 26 #define MTE_TCF_ASYMM 0b11 27 28 #define MTE_GRANULE_SIZE UL(16) 29 #define MTE_GRANULE_MASK (~(MTE_GRANULE_SIZE - 1)) 30 #define MTE_TAG_SHIFT 56 31 32 #define untagged(p) \ 33 ({ \ 34 unsigned long __in = (unsigned long)(p); \ 35 typeof(p) __out = (typeof(p))(__in & ~(MTE_GRANULE_MASK << MTE_TAG_SHIFT)); \ 36 \ 37 __out; \ 38 }) 39 40 #define tagged(p, t) \ 41 ({ \ 42 unsigned long __in = (unsigned long)(untagged(p)); \ 43 unsigned long __tag = (unsigned long)(t) << MTE_TAG_SHIFT; \ 44 typeof(p) __out = (typeof(p))(__in | __tag); \ 45 \ 46 __out; \ 47 }) 48 49 /* 50 * If we use a normal (non hand coded inline assembly) load or store 51 * to access a tagged address, the compiler will reasonably assume 52 * that the access succeeded, and the next instruction may do 53 * something based on that assumption. 54 * 55 * But a test might want the tagged access to fail on purpose, and if 56 * we advance the PC to the next instruction, the one added by the 57 * compiler, we might leave the program in an unexpected state. 58 */ 59 static inline void mem_read(unsigned int *addr, unsigned int *res) 60 { 61 unsigned int r; 62 63 asm volatile ("ldr %w0,[%1]\n" 64 "str %w0,[%2]\n" 65 : "=&r" (r) 66 : "r" (addr), "r" (res) : "memory"); 67 } 68 69 static inline void mem_write(unsigned int *addr, unsigned int val) 70 { 71 /* The NOP allows the same exception handler as mem_read() to be used. */ 72 asm volatile ("str %w0,[%1]\n" 73 "nop\n" 74 : 75 : "r" (val), "r" (addr) 76 : "memory"); 77 } 78 79 static volatile bool mte_exception; 80 81 static void mte_fault_handler(struct pt_regs *regs, unsigned int esr) 82 { 83 unsigned int dfsc = esr & GENMASK(5, 0); 84 unsigned int fnv = esr & BIT(10); 85 86 if (dfsc == 0b010001) { 87 if (fnv) 88 report_info("Unexpected non-zero FnV"); 89 mte_exception = true; 90 } 91 92 /* 93 * mem_read() reads the value from the tagged pointer, then 94 * stores this value in the untagged 'res' pointer. The 95 * function that called mem_read() will want to check that the 96 * initial value of 'res' hasn't changed if a tag check fault 97 * is reported. Skip over two instructions so 'res' isn't 98 * overwritten. 99 */ 100 regs->pc += 8; 101 } 102 103 static inline void mmu_set_tagged(pgd_t *pgtable, unsigned long vaddr) 104 { 105 pteval_t *p_pte = follow_pte(pgtable, untagged(vaddr)); 106 107 if (p_pte) { 108 pteval_t entry = *p_pte; 109 110 entry &= ~PTE_ATTRINDX_MASK; 111 entry |= PTE_ATTRINDX(MT_NORMAL_TAGGED); 112 113 WRITE_ONCE(*p_pte, entry); 114 flush_tlb_page(vaddr); 115 } else { 116 report_abort("Cannot find PTE"); 117 } 118 } 119 120 static void mte_init(void) 121 { 122 unsigned long sctlr = read_sysreg(sctlr_el1); 123 unsigned long tcr = read_sysreg(tcr_el1); 124 125 sctlr &= ~SCTLR_EL1_TCF_MASK; 126 sctlr |= SCTLR_EL1_ATA; 127 128 tcr &= ~TCR_TCMA0; 129 tcr |= TCR_TBI0; 130 131 write_sysreg(sctlr, sctlr_el1); 132 write_sysreg(tcr, tcr_el1); 133 134 isb(); 135 flush_tlb_all(); 136 } 137 138 static inline unsigned long mte_set_tcf(unsigned long tcf) 139 { 140 unsigned long sctlr = read_sysreg(sctlr_el1); 141 unsigned long old = (sctlr & SCTLR_EL1_TCF_MASK) >> SCTLR_EL1_TCF_SHIFT; 142 143 sctlr &= ~(SCTLR_EL1_TCF_MASK | SCTLR_EL1_TCF0_MASK); 144 sctlr |= (tcf << SCTLR_EL1_TCF_SHIFT) & SCTLR_EL1_TCF_MASK; 145 146 write_sysreg(sctlr, sctlr_el1); 147 write_sysreg_s(0, TFSR_EL1); 148 isb(); 149 150 return old; 151 } 152 153 static inline void mte_set_tag(void *addr, size_t size, unsigned int tag) 154 { 155 #ifdef CC_HAS_MTE 156 unsigned long in = (unsigned long)untagged(addr); 157 unsigned long start = ALIGN_DOWN(in, 16); 158 unsigned long end = ALIGN(in + size, 16); 159 160 for (unsigned long ptr = start; ptr < end; ptr += 16) { 161 asm volatile(".arch armv8.5-a+memtag\n" 162 "stg %0, [%0]" 163 : 164 : "r"(tagged(ptr, tag)) 165 : "memory"); 166 } 167 #endif 168 } 169 170 static inline unsigned long get_clear_tfsr(void) 171 { 172 unsigned long r; 173 174 dsb(nsh); 175 isb(); 176 177 r = read_sysreg_s(TFSR_EL1); 178 write_sysreg_s(0, TFSR_EL1); 179 180 return r; 181 } 182 183 static void mte_sync_test(void) 184 { 185 unsigned int *mem = tagged(alloc_page(), 1); 186 unsigned int val = 0; 187 188 mmu_set_tagged(current_thread_info()->pgtable, (unsigned long)mem); 189 mte_set_tag(mem, PAGE_SIZE, 1); 190 memset(mem, 0xff, PAGE_SIZE); 191 mte_set_tcf(MTE_TCF_SYNC); 192 193 mte_exception = false; 194 195 install_exception_handler(EL1H_SYNC, ESR_EL1_EC_DABT_EL1, mte_fault_handler); 196 197 mem_read(tagged(mem, 2), &val); 198 199 report((val == 0) && mte_exception && (get_clear_tfsr() == 0), "read"); 200 201 mte_exception = false; 202 203 mem_write(tagged(mem, 3), 0xbbbbbbbb); 204 205 report((*mem == 0xffffffff) && mte_exception && (get_clear_tfsr() == 0), "write"); 206 207 free_page(untagged(mem)); 208 } 209 210 static void mte_asymm_test(void) 211 { 212 unsigned int *mem = tagged(alloc_page(), 2); 213 unsigned int val = 0; 214 215 mmu_set_tagged(current_thread_info()->pgtable, (unsigned long)mem); 216 mte_set_tag(mem, PAGE_SIZE, 2); 217 memset(mem, 0xff, PAGE_SIZE); 218 mte_set_tcf(MTE_TCF_ASYMM); 219 mte_exception = false; 220 221 install_exception_handler(EL1H_SYNC, ESR_EL1_EC_DABT_EL1, mte_fault_handler); 222 223 mem_read(tagged(mem, 3), &val); 224 report((val == 0) && mte_exception && (get_clear_tfsr() == 0), "read"); 225 226 install_exception_handler(EL1H_SYNC, ESR_EL1_EC_DABT_EL1, NULL); 227 228 mem_write(tagged(mem, 4), 0xaaaaaaaa); 229 report((*mem == 0xaaaaaaaa) && (get_clear_tfsr() == TFSR_EL1_TF0), "write"); 230 231 free_page(untagged(mem)); 232 } 233 234 static void mte_async_test(void) 235 { 236 unsigned int *mem = tagged(alloc_page(), 3); 237 unsigned int val = 0; 238 239 mmu_set_tagged(current_thread_info()->pgtable, (unsigned long)mem); 240 mte_set_tag(mem, PAGE_SIZE, 3); 241 memset(mem, 0xff, PAGE_SIZE); 242 mte_set_tcf(MTE_TCF_ASYNC); 243 244 mem_read(tagged(mem, 4), &val); 245 report((val == 0xffffffff) && (get_clear_tfsr() == TFSR_EL1_TF0), "read"); 246 247 mem_write(tagged(mem, 5), 0xcccccccc); 248 report((*mem == 0xcccccccc) && (get_clear_tfsr() == TFSR_EL1_TF0), "write"); 249 250 free_page(untagged(mem)); 251 } 252 253 static unsigned int mte_version(void) 254 { 255 #ifdef CC_HAS_MTE 256 uint64_t r; 257 258 asm volatile("mrs %x0, id_aa64pfr1_el1" : "=r"(r)); 259 260 return (r >> ID_AA64PFR1_EL1_MTE_SHIFT) & 0b1111; 261 #else 262 report_info("Compiler lack MTE support"); 263 return 0; 264 #endif 265 } 266 267 int main(int argc, char *argv[]) 268 { 269 270 unsigned int version = mte_version(); 271 272 if (version < 2) { 273 report_skip("No MTE support, skip...\n"); 274 return report_summary(); 275 } 276 277 if (argc < 2) 278 report_abort("no test specified"); 279 280 report_prefix_push("mte"); 281 282 mte_init(); 283 284 if (strcmp(argv[1], "sync") == 0) { 285 report_prefix_push(argv[1]); 286 mte_sync_test(); 287 report_prefix_pop(); 288 } else if (strcmp(argv[1], "async") == 0) { 289 report_prefix_push(argv[1]); 290 if (version < 3) { 291 report_skip("No MTE async, skip...\n"); 292 return report_summary(); 293 } 294 mte_async_test(); 295 report_prefix_pop(); 296 } else if (strcmp(argv[1], "asymm") == 0) { 297 report_prefix_push(argv[1]); 298 if (version < 3) { 299 report_skip("No MTE asymm, skip...\n"); 300 return report_summary(); 301 } 302 mte_asymm_test(); 303 report_prefix_pop(); 304 } else { 305 report_abort("Unknown sub-test '%s'", argv[1]); 306 } 307 308 return report_summary(); 309 } 310