1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 /*
4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <pasha.tatashin@soleen.com>
6 */
7 #ifndef __LINUX_PAGE_TABLE_CHECK_H
8 #define __LINUX_PAGE_TABLE_CHECK_H
9
10 #ifdef CONFIG_PAGE_TABLE_CHECK
11 #include <linux/jump_label.h>
12
13 extern struct static_key_true page_table_check_disabled;
14 extern struct page_ext_operations page_table_check_ops;
15
16 void __page_table_check_zero(struct page *page, unsigned int order);
17 void __page_table_check_pte_clear(struct mm_struct *mm, unsigned long addr,
18 pte_t pte);
19 void __page_table_check_pmd_clear(struct mm_struct *mm, unsigned long addr,
20 pmd_t pmd);
21 void __page_table_check_pud_clear(struct mm_struct *mm, unsigned long addr,
22 pud_t pud);
23 void __page_table_check_ptes_set(struct mm_struct *mm, unsigned long addr,
24 pte_t *ptep, pte_t pte, unsigned int nr);
25 void __page_table_check_pmds_set(struct mm_struct *mm, unsigned long addr,
26 pmd_t *pmdp, pmd_t pmd, unsigned int nr);
27 void __page_table_check_puds_set(struct mm_struct *mm, unsigned long addr,
28 pud_t *pudp, pud_t pud, unsigned int nr);
29 void __page_table_check_pte_clear_range(struct mm_struct *mm,
30 unsigned long addr,
31 pmd_t pmd);
32
page_table_check_alloc(struct page * page,unsigned int order)33 static inline void page_table_check_alloc(struct page *page, unsigned int order)
34 {
35 if (static_branch_likely(&page_table_check_disabled))
36 return;
37
38 __page_table_check_zero(page, order);
39 }
40
page_table_check_free(struct page * page,unsigned int order)41 static inline void page_table_check_free(struct page *page, unsigned int order)
42 {
43 if (static_branch_likely(&page_table_check_disabled))
44 return;
45
46 __page_table_check_zero(page, order);
47 }
48
page_table_check_pte_clear(struct mm_struct * mm,unsigned long addr,pte_t pte)49 static inline void page_table_check_pte_clear(struct mm_struct *mm,
50 unsigned long addr, pte_t pte)
51 {
52 if (static_branch_likely(&page_table_check_disabled))
53 return;
54
55 __page_table_check_pte_clear(mm, addr, pte);
56 }
57
page_table_check_pmd_clear(struct mm_struct * mm,unsigned long addr,pmd_t pmd)58 static inline void page_table_check_pmd_clear(struct mm_struct *mm,
59 unsigned long addr, pmd_t pmd)
60 {
61 if (static_branch_likely(&page_table_check_disabled))
62 return;
63
64 __page_table_check_pmd_clear(mm, addr, pmd);
65 }
66
page_table_check_pud_clear(struct mm_struct * mm,unsigned long addr,pud_t pud)67 static inline void page_table_check_pud_clear(struct mm_struct *mm,
68 unsigned long addr, pud_t pud)
69 {
70 if (static_branch_likely(&page_table_check_disabled))
71 return;
72
73 __page_table_check_pud_clear(mm, addr, pud);
74 }
75
page_table_check_ptes_set(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t pte,unsigned int nr)76 static inline void page_table_check_ptes_set(struct mm_struct *mm,
77 unsigned long addr, pte_t *ptep,
78 pte_t pte, unsigned int nr)
79 {
80 if (static_branch_likely(&page_table_check_disabled))
81 return;
82
83 __page_table_check_ptes_set(mm, addr, ptep, pte, nr);
84 }
85
page_table_check_pmds_set(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t pmd,unsigned int nr)86 static inline void page_table_check_pmds_set(struct mm_struct *mm,
87 unsigned long addr, pmd_t *pmdp, pmd_t pmd, unsigned int nr)
88 {
89 if (static_branch_likely(&page_table_check_disabled))
90 return;
91
92 __page_table_check_pmds_set(mm, addr, pmdp, pmd, nr);
93 }
94
page_table_check_puds_set(struct mm_struct * mm,unsigned long addr,pud_t * pudp,pud_t pud,unsigned int nr)95 static inline void page_table_check_puds_set(struct mm_struct *mm,
96 unsigned long addr, pud_t *pudp, pud_t pud, unsigned int nr)
97 {
98 if (static_branch_likely(&page_table_check_disabled))
99 return;
100
101 __page_table_check_puds_set(mm, addr, pudp, pud, nr);
102 }
103
page_table_check_pte_clear_range(struct mm_struct * mm,unsigned long addr,pmd_t pmd)104 static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
105 unsigned long addr,
106 pmd_t pmd)
107 {
108 if (static_branch_likely(&page_table_check_disabled))
109 return;
110
111 __page_table_check_pte_clear_range(mm, addr, pmd);
112 }
113
114 #else
115
page_table_check_alloc(struct page * page,unsigned int order)116 static inline void page_table_check_alloc(struct page *page, unsigned int order)
117 {
118 }
119
page_table_check_free(struct page * page,unsigned int order)120 static inline void page_table_check_free(struct page *page, unsigned int order)
121 {
122 }
123
page_table_check_pte_clear(struct mm_struct * mm,unsigned long addr,pte_t pte)124 static inline void page_table_check_pte_clear(struct mm_struct *mm,
125 unsigned long addr, pte_t pte)
126 {
127 }
128
page_table_check_pmd_clear(struct mm_struct * mm,unsigned long addr,pmd_t pmd)129 static inline void page_table_check_pmd_clear(struct mm_struct *mm,
130 unsigned long addr, pmd_t pmd)
131 {
132 }
133
page_table_check_pud_clear(struct mm_struct * mm,unsigned long addr,pud_t pud)134 static inline void page_table_check_pud_clear(struct mm_struct *mm,
135 unsigned long addr, pud_t pud)
136 {
137 }
138
page_table_check_ptes_set(struct mm_struct * mm,unsigned long addr,pte_t * ptep,pte_t pte,unsigned int nr)139 static inline void page_table_check_ptes_set(struct mm_struct *mm,
140 unsigned long addr, pte_t *ptep,
141 pte_t pte, unsigned int nr)
142 {
143 }
144
page_table_check_pmds_set(struct mm_struct * mm,unsigned long addr,pmd_t * pmdp,pmd_t pmd,unsigned int nr)145 static inline void page_table_check_pmds_set(struct mm_struct *mm,
146 unsigned long addr, pmd_t *pmdp, pmd_t pmd, unsigned int nr)
147 {
148 }
149
page_table_check_puds_set(struct mm_struct * mm,unsigned long addr,pud_t * pudp,pud_t pud,unsigned int nr)150 static inline void page_table_check_puds_set(struct mm_struct *mm,
151 unsigned long addr, pud_t *pudp, pud_t pud, unsigned int nr)
152 {
153 }
154
page_table_check_pte_clear_range(struct mm_struct * mm,unsigned long addr,pmd_t pmd)155 static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
156 unsigned long addr,
157 pmd_t pmd)
158 {
159 }
160
161 #endif /* CONFIG_PAGE_TABLE_CHECK */
162
163 #define page_table_check_pmd_set(mm, addr, pmdp, pmd) page_table_check_pmds_set(mm, addr, pmdp, pmd, 1)
164 #define page_table_check_pud_set(mm, addr, pudp, pud) page_table_check_puds_set(mm, addr, pudp, pud, 1)
165
166 #endif /* __LINUX_PAGE_TABLE_CHECK_H */
167