1 /* SPDX-License-Identifier: GPL-2.0-only */
2 /*
3 * Copyright (c) 2025, NVIDIA CORPORATION & AFFILIATES
4 *
5 * RISC-V page table
6 *
7 * This is described in Sections:
8 * 12.3. Sv32: Page-Based 32-bit Virtual-Memory Systems
9 * 12.4. Sv39: Page-Based 39-bit Virtual-Memory System
10 * 12.5. Sv48: Page-Based 48-bit Virtual-Memory System
11 * 12.6. Sv57: Page-Based 57-bit Virtual-Memory System
12 * of the "The RISC-V Instruction Set Manual: Volume II"
13 *
14 * This includes the contiguous page extension from:
15 * Chapter 13. "Svnapot" Extension for NAPOT Translation Contiguity,
16 * Version 1.0
17 *
18 * The table format is sign extended and supports leafs in every level. The spec
19 * doesn't talk a lot about levels, but level here is the same as i=LEVELS-1 in
20 * the spec.
21 */
22 #ifndef __GENERIC_PT_FMT_RISCV_H
23 #define __GENERIC_PT_FMT_RISCV_H
24
25 #include "defs_riscv.h"
26 #include "../pt_defs.h"
27
28 #include <linux/bitfield.h>
29 #include <linux/container_of.h>
30 #include <linux/log2.h>
31 #include <linux/sizes.h>
32
33 enum {
34 PT_ITEM_WORD_SIZE = sizeof(pt_riscv_entry_t),
35 #ifdef PT_RISCV_32BIT
36 PT_MAX_VA_ADDRESS_LG2 = 32,
37 PT_MAX_OUTPUT_ADDRESS_LG2 = 34,
38 PT_MAX_TOP_LEVEL = 1,
39 #else
40 PT_MAX_VA_ADDRESS_LG2 = 57,
41 PT_MAX_OUTPUT_ADDRESS_LG2 = 56,
42 PT_MAX_TOP_LEVEL = 4,
43 #endif
44 PT_GRANULE_LG2SZ = 12,
45 PT_TABLEMEM_LG2SZ = 12,
46
47 /* fsc.PPN is 44 bits wide, all PPNs are 4k aligned */
48 PT_TOP_PHYS_MASK = GENMASK_ULL(55, 12),
49 };
50
51 /* PTE bits */
52 enum {
53 RISCVPT_V = BIT(0),
54 RISCVPT_R = BIT(1),
55 RISCVPT_W = BIT(2),
56 RISCVPT_X = BIT(3),
57 RISCVPT_U = BIT(4),
58 RISCVPT_G = BIT(5),
59 RISCVPT_A = BIT(6),
60 RISCVPT_D = BIT(7),
61 RISCVPT_RSW = GENMASK(9, 8),
62 RISCVPT_PPN32 = GENMASK(31, 10),
63
64 RISCVPT_PPN64 = GENMASK_ULL(53, 10),
65 RISCVPT_PPN64_64K = GENMASK_ULL(53, 14),
66 RISCVPT_PBMT = GENMASK_ULL(62, 61),
67 RISCVPT_N = BIT_ULL(63),
68
69 /* Svnapot encodings for ppn[0] */
70 RISCVPT_PPN64_64K_SZ = BIT(13),
71 };
72
73 #ifdef PT_RISCV_32BIT
74 #define RISCVPT_PPN RISCVPT_PPN32
75 #define pt_riscv pt_riscv_32
76 #else
77 #define RISCVPT_PPN RISCVPT_PPN64
78 #define pt_riscv pt_riscv_64
79 #endif
80
81 #define common_to_riscvpt(common_ptr) \
82 container_of_const(common_ptr, struct pt_riscv, common)
83 #define to_riscvpt(pts) common_to_riscvpt((pts)->range->common)
84
riscvpt_table_pa(const struct pt_state * pts)85 static inline pt_oaddr_t riscvpt_table_pa(const struct pt_state *pts)
86 {
87 return oalog2_mul(FIELD_GET(RISCVPT_PPN, pts->entry), PT_GRANULE_LG2SZ);
88 }
89 #define pt_table_pa riscvpt_table_pa
90
riscvpt_entry_oa(const struct pt_state * pts)91 static inline pt_oaddr_t riscvpt_entry_oa(const struct pt_state *pts)
92 {
93 if (pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K) &&
94 pts->entry & RISCVPT_N) {
95 PT_WARN_ON(pts->level != 0);
96 return oalog2_mul(FIELD_GET(RISCVPT_PPN64_64K, pts->entry),
97 ilog2(SZ_64K));
98 }
99 return oalog2_mul(FIELD_GET(RISCVPT_PPN, pts->entry), PT_GRANULE_LG2SZ);
100 }
101 #define pt_entry_oa riscvpt_entry_oa
102
riscvpt_can_have_leaf(const struct pt_state * pts)103 static inline bool riscvpt_can_have_leaf(const struct pt_state *pts)
104 {
105 return true;
106 }
107 #define pt_can_have_leaf riscvpt_can_have_leaf
108
109 /* Body in pt_fmt_defaults.h */
110 static inline unsigned int pt_table_item_lg2sz(const struct pt_state *pts);
111
112 static inline unsigned int
riscvpt_entry_num_contig_lg2(const struct pt_state * pts)113 riscvpt_entry_num_contig_lg2(const struct pt_state *pts)
114 {
115 if (PT_SUPPORTED_FEATURE(PT_FEAT_RISCV_SVNAPOT_64K) &&
116 pts->entry & RISCVPT_N) {
117 PT_WARN_ON(!pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K));
118 PT_WARN_ON(pts->level);
119 return ilog2(16);
120 }
121 return ilog2(1);
122 }
123 #define pt_entry_num_contig_lg2 riscvpt_entry_num_contig_lg2
124
riscvpt_num_items_lg2(const struct pt_state * pts)125 static inline unsigned int riscvpt_num_items_lg2(const struct pt_state *pts)
126 {
127 return PT_TABLEMEM_LG2SZ - ilog2(sizeof(u64));
128 }
129 #define pt_num_items_lg2 riscvpt_num_items_lg2
130
131 static inline unsigned short
riscvpt_contig_count_lg2(const struct pt_state * pts)132 riscvpt_contig_count_lg2(const struct pt_state *pts)
133 {
134 if (pts->level == 0 && pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K))
135 return ilog2(16);
136 return ilog2(1);
137 }
138 #define pt_contig_count_lg2 riscvpt_contig_count_lg2
139
riscvpt_load_entry_raw(struct pt_state * pts)140 static inline enum pt_entry_type riscvpt_load_entry_raw(struct pt_state *pts)
141 {
142 const pt_riscv_entry_t *tablep = pt_cur_table(pts, pt_riscv_entry_t);
143 pt_riscv_entry_t entry;
144
145 pts->entry = entry = READ_ONCE(tablep[pts->index]);
146 if (!(entry & RISCVPT_V))
147 return PT_ENTRY_EMPTY;
148 if (pts->level == 0 ||
149 ((entry & (RISCVPT_X | RISCVPT_W | RISCVPT_R)) != 0))
150 return PT_ENTRY_OA;
151 return PT_ENTRY_TABLE;
152 }
153 #define pt_load_entry_raw riscvpt_load_entry_raw
154
155 static inline void
riscvpt_install_leaf_entry(struct pt_state * pts,pt_oaddr_t oa,unsigned int oasz_lg2,const struct pt_write_attrs * attrs)156 riscvpt_install_leaf_entry(struct pt_state *pts, pt_oaddr_t oa,
157 unsigned int oasz_lg2,
158 const struct pt_write_attrs *attrs)
159 {
160 pt_riscv_entry_t *tablep = pt_cur_table(pts, pt_riscv_entry_t);
161 pt_riscv_entry_t entry;
162
163 if (!pt_check_install_leaf_args(pts, oa, oasz_lg2))
164 return;
165
166 entry = RISCVPT_V |
167 FIELD_PREP(RISCVPT_PPN, log2_div(oa, PT_GRANULE_LG2SZ)) |
168 attrs->descriptor_bits;
169
170 if (pts_feature(pts, PT_FEAT_RISCV_SVNAPOT_64K) && pts->level == 0 &&
171 oasz_lg2 != PT_GRANULE_LG2SZ) {
172 u64 *end;
173
174 entry |= RISCVPT_N | RISCVPT_PPN64_64K_SZ;
175 tablep += pts->index;
176 end = tablep + log2_div(SZ_64K, PT_GRANULE_LG2SZ);
177 for (; tablep != end; tablep++)
178 WRITE_ONCE(*tablep, entry);
179 } else {
180 /* FIXME does riscv need this to be cmpxchg? */
181 WRITE_ONCE(tablep[pts->index], entry);
182 }
183 pts->entry = entry;
184 }
185 #define pt_install_leaf_entry riscvpt_install_leaf_entry
186
riscvpt_install_table(struct pt_state * pts,pt_oaddr_t table_pa,const struct pt_write_attrs * attrs)187 static inline bool riscvpt_install_table(struct pt_state *pts,
188 pt_oaddr_t table_pa,
189 const struct pt_write_attrs *attrs)
190 {
191 pt_riscv_entry_t entry;
192
193 entry = RISCVPT_V |
194 FIELD_PREP(RISCVPT_PPN, log2_div(table_pa, PT_GRANULE_LG2SZ));
195 return pt_table_install64(pts, entry);
196 }
197 #define pt_install_table riscvpt_install_table
198
riscvpt_attr_from_entry(const struct pt_state * pts,struct pt_write_attrs * attrs)199 static inline void riscvpt_attr_from_entry(const struct pt_state *pts,
200 struct pt_write_attrs *attrs)
201 {
202 attrs->descriptor_bits =
203 pts->entry & (RISCVPT_R | RISCVPT_W | RISCVPT_X | RISCVPT_U |
204 RISCVPT_G | RISCVPT_A | RISCVPT_D);
205 }
206 #define pt_attr_from_entry riscvpt_attr_from_entry
207
208 /* --- iommu */
209 #include <linux/generic_pt/iommu.h>
210 #include <linux/iommu.h>
211
212 #define pt_iommu_table pt_iommu_riscv_64
213
214 /* The common struct is in the per-format common struct */
common_from_iommu(struct pt_iommu * iommu_table)215 static inline struct pt_common *common_from_iommu(struct pt_iommu *iommu_table)
216 {
217 return &container_of(iommu_table, struct pt_iommu_table, iommu)
218 ->riscv_64pt.common;
219 }
220
iommu_from_common(struct pt_common * common)221 static inline struct pt_iommu *iommu_from_common(struct pt_common *common)
222 {
223 return &container_of(common, struct pt_iommu_table, riscv_64pt.common)
224 ->iommu;
225 }
226
riscvpt_iommu_set_prot(struct pt_common * common,struct pt_write_attrs * attrs,unsigned int iommu_prot)227 static inline int riscvpt_iommu_set_prot(struct pt_common *common,
228 struct pt_write_attrs *attrs,
229 unsigned int iommu_prot)
230 {
231 u64 pte;
232
233 pte = RISCVPT_A | RISCVPT_U;
234 if (iommu_prot & IOMMU_WRITE)
235 pte |= RISCVPT_W | RISCVPT_R | RISCVPT_D;
236 if (iommu_prot & IOMMU_READ)
237 pte |= RISCVPT_R;
238 if (!(iommu_prot & IOMMU_NOEXEC))
239 pte |= RISCVPT_X;
240
241 /* Caller must specify a supported combination of flags */
242 if (unlikely((pte & (RISCVPT_X | RISCVPT_W | RISCVPT_R)) == 0))
243 return -EOPNOTSUPP;
244
245 attrs->descriptor_bits = pte;
246 return 0;
247 }
248 #define pt_iommu_set_prot riscvpt_iommu_set_prot
249
250 static inline int
riscvpt_iommu_fmt_init(struct pt_iommu_riscv_64 * iommu_table,const struct pt_iommu_riscv_64_cfg * cfg)251 riscvpt_iommu_fmt_init(struct pt_iommu_riscv_64 *iommu_table,
252 const struct pt_iommu_riscv_64_cfg *cfg)
253 {
254 struct pt_riscv *table = &iommu_table->riscv_64pt;
255
256 switch (cfg->common.hw_max_vasz_lg2) {
257 case 39:
258 pt_top_set_level(&table->common, 2);
259 break;
260 case 48:
261 pt_top_set_level(&table->common, 3);
262 break;
263 case 57:
264 pt_top_set_level(&table->common, 4);
265 break;
266 default:
267 return -EINVAL;
268 }
269 table->common.max_oasz_lg2 =
270 min(PT_MAX_OUTPUT_ADDRESS_LG2, cfg->common.hw_max_oasz_lg2);
271 return 0;
272 }
273 #define pt_iommu_fmt_init riscvpt_iommu_fmt_init
274
275 static inline void
riscvpt_iommu_fmt_hw_info(struct pt_iommu_riscv_64 * table,const struct pt_range * top_range,struct pt_iommu_riscv_64_hw_info * info)276 riscvpt_iommu_fmt_hw_info(struct pt_iommu_riscv_64 *table,
277 const struct pt_range *top_range,
278 struct pt_iommu_riscv_64_hw_info *info)
279 {
280 phys_addr_t top_phys = virt_to_phys(top_range->top_table);
281
282 info->ppn = oalog2_div(top_phys, PT_GRANULE_LG2SZ);
283 PT_WARN_ON(top_phys & ~PT_TOP_PHYS_MASK);
284
285 /*
286 * See Table 3. Encodings of iosatp.MODE field" for DC.tx.SXL = 0:
287 * 8 = Sv39 = top level 2
288 * 9 = Sv38 = top level 3
289 * 10 = Sv57 = top level 4
290 */
291 info->fsc_iosatp_mode = top_range->top_level + 6;
292 }
293 #define pt_iommu_fmt_hw_info riscvpt_iommu_fmt_hw_info
294
295 #if defined(GENERIC_PT_KUNIT)
296 static const struct pt_iommu_riscv_64_cfg riscv_64_kunit_fmt_cfgs[] = {
297 [0] = { .common.features = BIT(PT_FEAT_RISCV_SVNAPOT_64K),
298 .common.hw_max_oasz_lg2 = 56,
299 .common.hw_max_vasz_lg2 = 39 },
300 [1] = { .common.features = 0,
301 .common.hw_max_oasz_lg2 = 56,
302 .common.hw_max_vasz_lg2 = 48 },
303 [2] = { .common.features = BIT(PT_FEAT_RISCV_SVNAPOT_64K),
304 .common.hw_max_oasz_lg2 = 56,
305 .common.hw_max_vasz_lg2 = 57 },
306 };
307 #define kunit_fmt_cfgs riscv_64_kunit_fmt_cfgs
308 enum {
309 KUNIT_FMT_FEATURES = BIT(PT_FEAT_RISCV_SVNAPOT_64K),
310 };
311 #endif
312
313 #endif
314