1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Copyright (c) 2022-2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4 *
5 */
6
7 /* Support for NVIDIA specific attributes. */
8
9 #include <linux/io.h>
10 #include <linux/module.h>
11 #include <linux/topology.h>
12
13 #include "arm_cspmu.h"
14
15 #define NV_PCIE_PORT_COUNT 10ULL
16 #define NV_PCIE_FILTER_ID_MASK GENMASK_ULL(NV_PCIE_PORT_COUNT - 1, 0)
17
18 #define NV_NVL_C2C_PORT_COUNT 2ULL
19 #define NV_NVL_C2C_FILTER_ID_MASK GENMASK_ULL(NV_NVL_C2C_PORT_COUNT - 1, 0)
20
21 #define NV_CNVL_PORT_COUNT 4ULL
22 #define NV_CNVL_FILTER_ID_MASK GENMASK_ULL(NV_CNVL_PORT_COUNT - 1, 0)
23
24 #define NV_GENERIC_FILTER_ID_MASK GENMASK_ULL(31, 0)
25
26 #define NV_PRODID_MASK GENMASK(31, 0)
27
28 #define NV_FORMAT_NAME_GENERIC 0
29
30 #define to_nv_cspmu_ctx(cspmu) ((struct nv_cspmu_ctx *)(cspmu->impl.ctx))
31
32 #define NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _num, _suff, _config) \
33 ARM_CSPMU_EVENT_ATTR(_pref##_num##_suff, _config)
34
35 #define NV_CSPMU_EVENT_ATTR_4(_pref, _suff, _config) \
36 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _0_, _suff, _config), \
37 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _1_, _suff, _config + 1), \
38 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _2_, _suff, _config + 2), \
39 NV_CSPMU_EVENT_ATTR_4_INNER(_pref, _3_, _suff, _config + 3)
40
41 struct nv_cspmu_ctx {
42 const char *name;
43 u32 filter_mask;
44 u32 filter_default_val;
45 struct attribute **event_attr;
46 struct attribute **format_attr;
47 };
48
49 static struct attribute *scf_pmu_event_attrs[] = {
50 ARM_CSPMU_EVENT_ATTR(bus_cycles, 0x1d),
51
52 ARM_CSPMU_EVENT_ATTR(scf_cache_allocate, 0xF0),
53 ARM_CSPMU_EVENT_ATTR(scf_cache_refill, 0xF1),
54 ARM_CSPMU_EVENT_ATTR(scf_cache, 0xF2),
55 ARM_CSPMU_EVENT_ATTR(scf_cache_wb, 0xF3),
56
57 NV_CSPMU_EVENT_ATTR_4(socket, rd_data, 0x101),
58 NV_CSPMU_EVENT_ATTR_4(socket, wb_data, 0x109),
59
60 NV_CSPMU_EVENT_ATTR_4(socket, rd_outstanding, 0x115),
61
62 NV_CSPMU_EVENT_ATTR_4(socket, rd_access, 0x12d),
63 NV_CSPMU_EVENT_ATTR_4(socket, wb_access, 0x135),
64 NV_CSPMU_EVENT_ATTR_4(socket, wr_access, 0x139),
65
66 ARM_CSPMU_EVENT_ATTR(gmem_rd_data, 0x16d),
67 ARM_CSPMU_EVENT_ATTR(gmem_rd_access, 0x16e),
68 ARM_CSPMU_EVENT_ATTR(gmem_rd_outstanding, 0x16f),
69 ARM_CSPMU_EVENT_ATTR(gmem_wb_data, 0x173),
70 ARM_CSPMU_EVENT_ATTR(gmem_wb_access, 0x174),
71 ARM_CSPMU_EVENT_ATTR(gmem_wr_data, 0x179),
72 ARM_CSPMU_EVENT_ATTR(gmem_wr_access, 0x17b),
73
74 NV_CSPMU_EVENT_ATTR_4(socket, wr_data, 0x17c),
75
76 ARM_CSPMU_EVENT_ATTR(gmem_wr_total_bytes, 0x1a0),
77 ARM_CSPMU_EVENT_ATTR(remote_socket_wr_total_bytes, 0x1a1),
78 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_data, 0x1a2),
79 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_outstanding, 0x1a3),
80 ARM_CSPMU_EVENT_ATTR(remote_socket_rd_access, 0x1a4),
81
82 ARM_CSPMU_EVENT_ATTR(cmem_rd_data, 0x1a5),
83 ARM_CSPMU_EVENT_ATTR(cmem_rd_access, 0x1a6),
84 ARM_CSPMU_EVENT_ATTR(cmem_rd_outstanding, 0x1a7),
85 ARM_CSPMU_EVENT_ATTR(cmem_wb_data, 0x1ab),
86 ARM_CSPMU_EVENT_ATTR(cmem_wb_access, 0x1ac),
87 ARM_CSPMU_EVENT_ATTR(cmem_wr_data, 0x1b1),
88
89 ARM_CSPMU_EVENT_ATTR(cmem_wr_access, 0x1ca),
90
91 ARM_CSPMU_EVENT_ATTR(cmem_wr_total_bytes, 0x1db),
92
93 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
94 NULL,
95 };
96
97 static struct attribute *mcf_pmu_event_attrs[] = {
98 ARM_CSPMU_EVENT_ATTR(rd_bytes_loc, 0x0),
99 ARM_CSPMU_EVENT_ATTR(rd_bytes_rem, 0x1),
100 ARM_CSPMU_EVENT_ATTR(wr_bytes_loc, 0x2),
101 ARM_CSPMU_EVENT_ATTR(wr_bytes_rem, 0x3),
102 ARM_CSPMU_EVENT_ATTR(total_bytes_loc, 0x4),
103 ARM_CSPMU_EVENT_ATTR(total_bytes_rem, 0x5),
104 ARM_CSPMU_EVENT_ATTR(rd_req_loc, 0x6),
105 ARM_CSPMU_EVENT_ATTR(rd_req_rem, 0x7),
106 ARM_CSPMU_EVENT_ATTR(wr_req_loc, 0x8),
107 ARM_CSPMU_EVENT_ATTR(wr_req_rem, 0x9),
108 ARM_CSPMU_EVENT_ATTR(total_req_loc, 0xa),
109 ARM_CSPMU_EVENT_ATTR(total_req_rem, 0xb),
110 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_loc, 0xc),
111 ARM_CSPMU_EVENT_ATTR(rd_cum_outs_rem, 0xd),
112 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
113 NULL,
114 };
115
116 static struct attribute *generic_pmu_event_attrs[] = {
117 ARM_CSPMU_EVENT_ATTR(cycles, ARM_CSPMU_EVT_CYCLES_DEFAULT),
118 NULL,
119 };
120
121 static struct attribute *scf_pmu_format_attrs[] = {
122 ARM_CSPMU_FORMAT_EVENT_ATTR,
123 NULL,
124 };
125
126 static struct attribute *pcie_pmu_format_attrs[] = {
127 ARM_CSPMU_FORMAT_EVENT_ATTR,
128 ARM_CSPMU_FORMAT_ATTR(root_port, "config1:0-9"),
129 NULL,
130 };
131
132 static struct attribute *nvlink_c2c_pmu_format_attrs[] = {
133 ARM_CSPMU_FORMAT_EVENT_ATTR,
134 ARM_CSPMU_FORMAT_ATTR(port, "config1:0-1"),
135 NULL,
136 };
137
138 static struct attribute *cnvlink_pmu_format_attrs[] = {
139 ARM_CSPMU_FORMAT_EVENT_ATTR,
140 ARM_CSPMU_FORMAT_ATTR(rem_socket, "config1:0-3"),
141 NULL,
142 };
143
144 static struct attribute *generic_pmu_format_attrs[] = {
145 ARM_CSPMU_FORMAT_EVENT_ATTR,
146 ARM_CSPMU_FORMAT_FILTER_ATTR,
147 NULL,
148 };
149
150 static struct attribute **
nv_cspmu_get_event_attrs(const struct arm_cspmu * cspmu)151 nv_cspmu_get_event_attrs(const struct arm_cspmu *cspmu)
152 {
153 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
154
155 return ctx->event_attr;
156 }
157
158 static struct attribute **
nv_cspmu_get_format_attrs(const struct arm_cspmu * cspmu)159 nv_cspmu_get_format_attrs(const struct arm_cspmu *cspmu)
160 {
161 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
162
163 return ctx->format_attr;
164 }
165
166 static const char *
nv_cspmu_get_name(const struct arm_cspmu * cspmu)167 nv_cspmu_get_name(const struct arm_cspmu *cspmu)
168 {
169 const struct nv_cspmu_ctx *ctx = to_nv_cspmu_ctx(cspmu);
170
171 return ctx->name;
172 }
173
nv_cspmu_event_filter(const struct perf_event * event)174 static u32 nv_cspmu_event_filter(const struct perf_event *event)
175 {
176 const struct nv_cspmu_ctx *ctx =
177 to_nv_cspmu_ctx(to_arm_cspmu(event->pmu));
178
179 const u32 filter_val = event->attr.config1 & ctx->filter_mask;
180
181 if (filter_val == 0)
182 return ctx->filter_default_val;
183
184 return filter_val;
185 }
186
nv_cspmu_set_ev_filter(struct arm_cspmu * cspmu,const struct perf_event * event)187 static void nv_cspmu_set_ev_filter(struct arm_cspmu *cspmu,
188 const struct perf_event *event)
189 {
190 u32 filter = nv_cspmu_event_filter(event);
191 u32 offset = PMEVFILTR + (4 * event->hw.idx);
192
193 writel(filter, cspmu->base0 + offset);
194 }
195
nv_cspmu_set_cc_filter(struct arm_cspmu * cspmu,const struct perf_event * event)196 static void nv_cspmu_set_cc_filter(struct arm_cspmu *cspmu,
197 const struct perf_event *event)
198 {
199 u32 filter = nv_cspmu_event_filter(event);
200
201 writel(filter, cspmu->base0 + PMCCFILTR);
202 }
203
204
205 enum nv_cspmu_name_fmt {
206 NAME_FMT_GENERIC,
207 NAME_FMT_SOCKET
208 };
209
210 struct nv_cspmu_match {
211 u32 prodid;
212 u32 prodid_mask;
213 u64 filter_mask;
214 u32 filter_default_val;
215 const char *name_pattern;
216 enum nv_cspmu_name_fmt name_fmt;
217 struct attribute **event_attr;
218 struct attribute **format_attr;
219 };
220
221 static const struct nv_cspmu_match nv_cspmu_match[] = {
222 {
223 .prodid = 0x103,
224 .prodid_mask = NV_PRODID_MASK,
225 .filter_mask = NV_PCIE_FILTER_ID_MASK,
226 .filter_default_val = NV_PCIE_FILTER_ID_MASK,
227 .name_pattern = "nvidia_pcie_pmu_%u",
228 .name_fmt = NAME_FMT_SOCKET,
229 .event_attr = mcf_pmu_event_attrs,
230 .format_attr = pcie_pmu_format_attrs
231 },
232 {
233 .prodid = 0x104,
234 .prodid_mask = NV_PRODID_MASK,
235 .filter_mask = NV_NVL_C2C_FILTER_ID_MASK,
236 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
237 .name_pattern = "nvidia_nvlink_c2c1_pmu_%u",
238 .name_fmt = NAME_FMT_SOCKET,
239 .event_attr = mcf_pmu_event_attrs,
240 .format_attr = nvlink_c2c_pmu_format_attrs
241 },
242 {
243 .prodid = 0x105,
244 .prodid_mask = NV_PRODID_MASK,
245 .filter_mask = NV_NVL_C2C_FILTER_ID_MASK,
246 .filter_default_val = NV_NVL_C2C_FILTER_ID_MASK,
247 .name_pattern = "nvidia_nvlink_c2c0_pmu_%u",
248 .name_fmt = NAME_FMT_SOCKET,
249 .event_attr = mcf_pmu_event_attrs,
250 .format_attr = nvlink_c2c_pmu_format_attrs
251 },
252 {
253 .prodid = 0x106,
254 .prodid_mask = NV_PRODID_MASK,
255 .filter_mask = NV_CNVL_FILTER_ID_MASK,
256 .filter_default_val = NV_CNVL_FILTER_ID_MASK,
257 .name_pattern = "nvidia_cnvlink_pmu_%u",
258 .name_fmt = NAME_FMT_SOCKET,
259 .event_attr = mcf_pmu_event_attrs,
260 .format_attr = cnvlink_pmu_format_attrs
261 },
262 {
263 .prodid = 0x2CF,
264 .prodid_mask = NV_PRODID_MASK,
265 .filter_mask = 0x0,
266 .filter_default_val = 0x0,
267 .name_pattern = "nvidia_scf_pmu_%u",
268 .name_fmt = NAME_FMT_SOCKET,
269 .event_attr = scf_pmu_event_attrs,
270 .format_attr = scf_pmu_format_attrs
271 },
272 {
273 .prodid = 0,
274 .prodid_mask = 0,
275 .filter_mask = NV_GENERIC_FILTER_ID_MASK,
276 .filter_default_val = NV_GENERIC_FILTER_ID_MASK,
277 .name_pattern = "nvidia_uncore_pmu_%u",
278 .name_fmt = NAME_FMT_GENERIC,
279 .event_attr = generic_pmu_event_attrs,
280 .format_attr = generic_pmu_format_attrs
281 },
282 };
283
nv_cspmu_format_name(const struct arm_cspmu * cspmu,const struct nv_cspmu_match * match)284 static char *nv_cspmu_format_name(const struct arm_cspmu *cspmu,
285 const struct nv_cspmu_match *match)
286 {
287 char *name;
288 struct device *dev = cspmu->dev;
289
290 static atomic_t pmu_generic_idx = {0};
291
292 switch (match->name_fmt) {
293 case NAME_FMT_SOCKET: {
294 const int cpu = cpumask_first(&cspmu->associated_cpus);
295 const int socket = cpu_to_node(cpu);
296
297 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
298 socket);
299 break;
300 }
301 case NAME_FMT_GENERIC:
302 name = devm_kasprintf(dev, GFP_KERNEL, match->name_pattern,
303 atomic_fetch_inc(&pmu_generic_idx));
304 break;
305 default:
306 name = NULL;
307 break;
308 }
309
310 return name;
311 }
312
nv_cspmu_init_ops(struct arm_cspmu * cspmu)313 static int nv_cspmu_init_ops(struct arm_cspmu *cspmu)
314 {
315 u32 prodid;
316 struct nv_cspmu_ctx *ctx;
317 struct device *dev = cspmu->dev;
318 struct arm_cspmu_impl_ops *impl_ops = &cspmu->impl.ops;
319 const struct nv_cspmu_match *match = nv_cspmu_match;
320
321 ctx = devm_kzalloc(dev, sizeof(struct nv_cspmu_ctx), GFP_KERNEL);
322 if (!ctx)
323 return -ENOMEM;
324
325 prodid = FIELD_GET(ARM_CSPMU_PMIIDR_PRODUCTID, cspmu->impl.pmiidr);
326
327 /* Find matching PMU. */
328 for (; match->prodid; match++) {
329 const u32 prodid_mask = match->prodid_mask;
330
331 if ((match->prodid & prodid_mask) == (prodid & prodid_mask))
332 break;
333 }
334
335 ctx->name = nv_cspmu_format_name(cspmu, match);
336 ctx->filter_mask = match->filter_mask;
337 ctx->filter_default_val = match->filter_default_val;
338 ctx->event_attr = match->event_attr;
339 ctx->format_attr = match->format_attr;
340
341 cspmu->impl.ctx = ctx;
342
343 /* NVIDIA specific callbacks. */
344 impl_ops->set_cc_filter = nv_cspmu_set_cc_filter;
345 impl_ops->set_ev_filter = nv_cspmu_set_ev_filter;
346 impl_ops->get_event_attrs = nv_cspmu_get_event_attrs;
347 impl_ops->get_format_attrs = nv_cspmu_get_format_attrs;
348 impl_ops->get_name = nv_cspmu_get_name;
349
350 return 0;
351 }
352
353 /* Match all NVIDIA Coresight PMU devices */
354 static const struct arm_cspmu_impl_match nv_cspmu_param = {
355 .pmiidr_val = ARM_CSPMU_IMPL_ID_NVIDIA,
356 .module = THIS_MODULE,
357 .impl_init_ops = nv_cspmu_init_ops
358 };
359
nvidia_cspmu_init(void)360 static int __init nvidia_cspmu_init(void)
361 {
362 int ret;
363
364 ret = arm_cspmu_impl_register(&nv_cspmu_param);
365 if (ret)
366 pr_err("nvidia_cspmu backend registration error: %d\n", ret);
367
368 return ret;
369 }
370
nvidia_cspmu_exit(void)371 static void __exit nvidia_cspmu_exit(void)
372 {
373 arm_cspmu_impl_unregister(&nv_cspmu_param);
374 }
375
376 module_init(nvidia_cspmu_init);
377 module_exit(nvidia_cspmu_exit);
378
379 MODULE_DESCRIPTION("NVIDIA Coresight Architecture Performance Monitor Driver");
380 MODULE_LICENSE("GPL v2");
381