1 /* SPDX-License-Identifier: GPL-2.0 */
2
3 /*
4 * Copyright (c) 2021, Google LLC.
5 * Pasha Tatashin <pasha.tatashin@soleen.com>
6 */
7 #ifndef __LINUX_PAGE_TABLE_CHECK_H
8 #define __LINUX_PAGE_TABLE_CHECK_H
9
10 #ifdef CONFIG_PAGE_TABLE_CHECK
11 #include <linux/jump_label.h>
12
13 extern struct static_key_true page_table_check_disabled;
14 extern struct page_ext_operations page_table_check_ops;
15
16 void __page_table_check_zero(struct page *page, unsigned int order);
17 void __page_table_check_pte_clear(struct mm_struct *mm, pte_t pte);
18 void __page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd);
19 void __page_table_check_pud_clear(struct mm_struct *mm, pud_t pud);
20 void __page_table_check_ptes_set(struct mm_struct *mm, pte_t *ptep, pte_t pte,
21 unsigned int nr);
22 void __page_table_check_pmds_set(struct mm_struct *mm, pmd_t *pmdp, pmd_t pmd,
23 unsigned int nr);
24 void __page_table_check_puds_set(struct mm_struct *mm, pud_t *pudp, pud_t pud,
25 unsigned int nr);
26 void __page_table_check_pte_clear_range(struct mm_struct *mm,
27 unsigned long addr,
28 pmd_t pmd);
29
page_table_check_alloc(struct page * page,unsigned int order)30 static inline void page_table_check_alloc(struct page *page, unsigned int order)
31 {
32 if (static_branch_likely(&page_table_check_disabled))
33 return;
34
35 __page_table_check_zero(page, order);
36 }
37
page_table_check_free(struct page * page,unsigned int order)38 static inline void page_table_check_free(struct page *page, unsigned int order)
39 {
40 if (static_branch_likely(&page_table_check_disabled))
41 return;
42
43 __page_table_check_zero(page, order);
44 }
45
page_table_check_pte_clear(struct mm_struct * mm,pte_t pte)46 static inline void page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
47 {
48 if (static_branch_likely(&page_table_check_disabled))
49 return;
50
51 __page_table_check_pte_clear(mm, pte);
52 }
53
page_table_check_pmd_clear(struct mm_struct * mm,pmd_t pmd)54 static inline void page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
55 {
56 if (static_branch_likely(&page_table_check_disabled))
57 return;
58
59 __page_table_check_pmd_clear(mm, pmd);
60 }
61
page_table_check_pud_clear(struct mm_struct * mm,pud_t pud)62 static inline void page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
63 {
64 if (static_branch_likely(&page_table_check_disabled))
65 return;
66
67 __page_table_check_pud_clear(mm, pud);
68 }
69
page_table_check_ptes_set(struct mm_struct * mm,pte_t * ptep,pte_t pte,unsigned int nr)70 static inline void page_table_check_ptes_set(struct mm_struct *mm,
71 pte_t *ptep, pte_t pte, unsigned int nr)
72 {
73 if (static_branch_likely(&page_table_check_disabled))
74 return;
75
76 __page_table_check_ptes_set(mm, ptep, pte, nr);
77 }
78
page_table_check_pmds_set(struct mm_struct * mm,pmd_t * pmdp,pmd_t pmd,unsigned int nr)79 static inline void page_table_check_pmds_set(struct mm_struct *mm,
80 pmd_t *pmdp, pmd_t pmd, unsigned int nr)
81 {
82 if (static_branch_likely(&page_table_check_disabled))
83 return;
84
85 __page_table_check_pmds_set(mm, pmdp, pmd, nr);
86 }
87
page_table_check_puds_set(struct mm_struct * mm,pud_t * pudp,pud_t pud,unsigned int nr)88 static inline void page_table_check_puds_set(struct mm_struct *mm,
89 pud_t *pudp, pud_t pud, unsigned int nr)
90 {
91 if (static_branch_likely(&page_table_check_disabled))
92 return;
93
94 __page_table_check_puds_set(mm, pudp, pud, nr);
95 }
96
page_table_check_pte_clear_range(struct mm_struct * mm,unsigned long addr,pmd_t pmd)97 static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
98 unsigned long addr,
99 pmd_t pmd)
100 {
101 if (static_branch_likely(&page_table_check_disabled))
102 return;
103
104 __page_table_check_pte_clear_range(mm, addr, pmd);
105 }
106
107 #else
108
page_table_check_alloc(struct page * page,unsigned int order)109 static inline void page_table_check_alloc(struct page *page, unsigned int order)
110 {
111 }
112
page_table_check_free(struct page * page,unsigned int order)113 static inline void page_table_check_free(struct page *page, unsigned int order)
114 {
115 }
116
page_table_check_pte_clear(struct mm_struct * mm,pte_t pte)117 static inline void page_table_check_pte_clear(struct mm_struct *mm, pte_t pte)
118 {
119 }
120
page_table_check_pmd_clear(struct mm_struct * mm,pmd_t pmd)121 static inline void page_table_check_pmd_clear(struct mm_struct *mm, pmd_t pmd)
122 {
123 }
124
page_table_check_pud_clear(struct mm_struct * mm,pud_t pud)125 static inline void page_table_check_pud_clear(struct mm_struct *mm, pud_t pud)
126 {
127 }
128
page_table_check_ptes_set(struct mm_struct * mm,pte_t * ptep,pte_t pte,unsigned int nr)129 static inline void page_table_check_ptes_set(struct mm_struct *mm,
130 pte_t *ptep, pte_t pte, unsigned int nr)
131 {
132 }
133
page_table_check_pmds_set(struct mm_struct * mm,pmd_t * pmdp,pmd_t pmd,unsigned int nr)134 static inline void page_table_check_pmds_set(struct mm_struct *mm,
135 pmd_t *pmdp, pmd_t pmd, unsigned int nr)
136 {
137 }
138
page_table_check_puds_set(struct mm_struct * mm,pud_t * pudp,pud_t pud,unsigned int nr)139 static inline void page_table_check_puds_set(struct mm_struct *mm,
140 pud_t *pudp, pud_t pud, unsigned int nr)
141 {
142 }
143
page_table_check_pte_clear_range(struct mm_struct * mm,unsigned long addr,pmd_t pmd)144 static inline void page_table_check_pte_clear_range(struct mm_struct *mm,
145 unsigned long addr,
146 pmd_t pmd)
147 {
148 }
149
150 #endif /* CONFIG_PAGE_TABLE_CHECK */
151
152 #define page_table_check_pmd_set(mm, pmdp, pmd) page_table_check_pmds_set(mm, pmdp, pmd, 1)
153 #define page_table_check_pud_set(mm, pudp, pud) page_table_check_puds_set(mm, pudp, pud, 1)
154
155 #endif /* __LINUX_PAGE_TABLE_CHECK_H */
156