1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2019 SiFive
4 */
5
6 #include <linux/pagewalk.h>
7 #include <linux/pgtable.h>
8 #include <asm/tlbflush.h>
9 #include <asm/bitops.h>
10 #include <asm/set_memory.h>
11
12 struct pageattr_masks {
13 pgprot_t set_mask;
14 pgprot_t clear_mask;
15 };
16
set_pageattr_masks(unsigned long val,struct mm_walk * walk)17 static unsigned long set_pageattr_masks(unsigned long val, struct mm_walk *walk)
18 {
19 struct pageattr_masks *masks = walk->private;
20 unsigned long new_val = val;
21
22 new_val &= ~(pgprot_val(masks->clear_mask));
23 new_val |= (pgprot_val(masks->set_mask));
24
25 return new_val;
26 }
27
pageattr_pgd_entry(pgd_t * pgd,unsigned long addr,unsigned long next,struct mm_walk * walk)28 static int pageattr_pgd_entry(pgd_t *pgd, unsigned long addr,
29 unsigned long next, struct mm_walk *walk)
30 {
31 pgd_t val = READ_ONCE(*pgd);
32
33 if (pgd_leaf(val)) {
34 val = __pgd(set_pageattr_masks(pgd_val(val), walk));
35 set_pgd(pgd, val);
36 }
37
38 return 0;
39 }
40
pageattr_p4d_entry(p4d_t * p4d,unsigned long addr,unsigned long next,struct mm_walk * walk)41 static int pageattr_p4d_entry(p4d_t *p4d, unsigned long addr,
42 unsigned long next, struct mm_walk *walk)
43 {
44 p4d_t val = READ_ONCE(*p4d);
45
46 if (p4d_leaf(val)) {
47 val = __p4d(set_pageattr_masks(p4d_val(val), walk));
48 set_p4d(p4d, val);
49 }
50
51 return 0;
52 }
53
pageattr_pud_entry(pud_t * pud,unsigned long addr,unsigned long next,struct mm_walk * walk)54 static int pageattr_pud_entry(pud_t *pud, unsigned long addr,
55 unsigned long next, struct mm_walk *walk)
56 {
57 pud_t val = READ_ONCE(*pud);
58
59 if (pud_leaf(val)) {
60 val = __pud(set_pageattr_masks(pud_val(val), walk));
61 set_pud(pud, val);
62 }
63
64 return 0;
65 }
66
pageattr_pmd_entry(pmd_t * pmd,unsigned long addr,unsigned long next,struct mm_walk * walk)67 static int pageattr_pmd_entry(pmd_t *pmd, unsigned long addr,
68 unsigned long next, struct mm_walk *walk)
69 {
70 pmd_t val = READ_ONCE(*pmd);
71
72 if (pmd_leaf(val)) {
73 val = __pmd(set_pageattr_masks(pmd_val(val), walk));
74 set_pmd(pmd, val);
75 }
76
77 return 0;
78 }
79
pageattr_pte_entry(pte_t * pte,unsigned long addr,unsigned long next,struct mm_walk * walk)80 static int pageattr_pte_entry(pte_t *pte, unsigned long addr,
81 unsigned long next, struct mm_walk *walk)
82 {
83 pte_t val = READ_ONCE(*pte);
84
85 val = __pte(set_pageattr_masks(pte_val(val), walk));
86 set_pte(pte, val);
87
88 return 0;
89 }
90
pageattr_pte_hole(unsigned long addr,unsigned long next,int depth,struct mm_walk * walk)91 static int pageattr_pte_hole(unsigned long addr, unsigned long next,
92 int depth, struct mm_walk *walk)
93 {
94 /* Nothing to do here */
95 return 0;
96 }
97
98 static const struct mm_walk_ops pageattr_ops = {
99 .pgd_entry = pageattr_pgd_entry,
100 .p4d_entry = pageattr_p4d_entry,
101 .pud_entry = pageattr_pud_entry,
102 .pmd_entry = pageattr_pmd_entry,
103 .pte_entry = pageattr_pte_entry,
104 .pte_hole = pageattr_pte_hole,
105 .walk_lock = PGWALK_RDLOCK,
106 };
107
__set_memory(unsigned long addr,int numpages,pgprot_t set_mask,pgprot_t clear_mask)108 static int __set_memory(unsigned long addr, int numpages, pgprot_t set_mask,
109 pgprot_t clear_mask)
110 {
111 int ret;
112 unsigned long start = addr;
113 unsigned long end = start + PAGE_SIZE * numpages;
114 struct pageattr_masks masks = {
115 .set_mask = set_mask,
116 .clear_mask = clear_mask
117 };
118
119 if (!numpages)
120 return 0;
121
122 mmap_write_lock(&init_mm);
123 ret = walk_page_range_novma(&init_mm, start, end, &pageattr_ops, NULL,
124 &masks);
125 mmap_write_unlock(&init_mm);
126
127 flush_tlb_kernel_range(start, end);
128
129 return ret;
130 }
131
set_memory_rw_nx(unsigned long addr,int numpages)132 int set_memory_rw_nx(unsigned long addr, int numpages)
133 {
134 return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
135 __pgprot(_PAGE_EXEC));
136 }
137
set_memory_ro(unsigned long addr,int numpages)138 int set_memory_ro(unsigned long addr, int numpages)
139 {
140 return __set_memory(addr, numpages, __pgprot(_PAGE_READ),
141 __pgprot(_PAGE_WRITE));
142 }
143
set_memory_rw(unsigned long addr,int numpages)144 int set_memory_rw(unsigned long addr, int numpages)
145 {
146 return __set_memory(addr, numpages, __pgprot(_PAGE_READ | _PAGE_WRITE),
147 __pgprot(0));
148 }
149
set_memory_x(unsigned long addr,int numpages)150 int set_memory_x(unsigned long addr, int numpages)
151 {
152 return __set_memory(addr, numpages, __pgprot(_PAGE_EXEC), __pgprot(0));
153 }
154
set_memory_nx(unsigned long addr,int numpages)155 int set_memory_nx(unsigned long addr, int numpages)
156 {
157 return __set_memory(addr, numpages, __pgprot(0), __pgprot(_PAGE_EXEC));
158 }
159
set_direct_map_invalid_noflush(struct page * page)160 int set_direct_map_invalid_noflush(struct page *page)
161 {
162 int ret;
163 unsigned long start = (unsigned long)page_address(page);
164 unsigned long end = start + PAGE_SIZE;
165 struct pageattr_masks masks = {
166 .set_mask = __pgprot(0),
167 .clear_mask = __pgprot(_PAGE_PRESENT)
168 };
169
170 mmap_read_lock(&init_mm);
171 ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks);
172 mmap_read_unlock(&init_mm);
173
174 return ret;
175 }
176
set_direct_map_default_noflush(struct page * page)177 int set_direct_map_default_noflush(struct page *page)
178 {
179 int ret;
180 unsigned long start = (unsigned long)page_address(page);
181 unsigned long end = start + PAGE_SIZE;
182 struct pageattr_masks masks = {
183 .set_mask = PAGE_KERNEL,
184 .clear_mask = __pgprot(0)
185 };
186
187 mmap_read_lock(&init_mm);
188 ret = walk_page_range(&init_mm, start, end, &pageattr_ops, &masks);
189 mmap_read_unlock(&init_mm);
190
191 return ret;
192 }
193
194 #ifdef CONFIG_DEBUG_PAGEALLOC
__kernel_map_pages(struct page * page,int numpages,int enable)195 void __kernel_map_pages(struct page *page, int numpages, int enable)
196 {
197 if (!debug_pagealloc_enabled())
198 return;
199
200 if (enable)
201 __set_memory((unsigned long)page_address(page), numpages,
202 __pgprot(_PAGE_PRESENT), __pgprot(0));
203 else
204 __set_memory((unsigned long)page_address(page), numpages,
205 __pgprot(0), __pgprot(_PAGE_PRESENT));
206 }
207 #endif
208
kernel_page_present(struct page * page)209 bool kernel_page_present(struct page *page)
210 {
211 unsigned long addr = (unsigned long)page_address(page);
212 pgd_t *pgd;
213 pud_t *pud;
214 p4d_t *p4d;
215 pmd_t *pmd;
216 pte_t *pte;
217
218 pgd = pgd_offset_k(addr);
219 if (!pgd_present(*pgd))
220 return false;
221 if (pgd_leaf(*pgd))
222 return true;
223
224 p4d = p4d_offset(pgd, addr);
225 if (!p4d_present(*p4d))
226 return false;
227 if (p4d_leaf(*p4d))
228 return true;
229
230 pud = pud_offset(p4d, addr);
231 if (!pud_present(*pud))
232 return false;
233 if (pud_leaf(*pud))
234 return true;
235
236 pmd = pmd_offset(pud, addr);
237 if (!pmd_present(*pmd))
238 return false;
239 if (pmd_leaf(*pmd))
240 return true;
241
242 pte = pte_offset_kernel(pmd, addr);
243 return pte_present(*pte);
244 }
245