1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2025 Intel Corporation
4 */
5
6 #include <drm/drm_drv.h>
7 #include <linux/device.h>
8
9 #include "xe_device.h"
10 #include "xe_force_wake.h"
11 #include "xe_gt_idle.h"
12 #include "xe_guc_engine_activity.h"
13 #include "xe_hw_engine.h"
14 #include "xe_pm.h"
15 #include "xe_pmu.h"
16
17 /**
18 * DOC: Xe PMU (Performance Monitoring Unit)
19 *
20 * Expose events/counters like GT-C6 residency, GT frequency and per-class-engine
21 * activity to user land via the perf interface. Events are per device.
22 *
23 * All events are listed in sysfs:
24 *
25 * $ ls -ld /sys/bus/event_source/devices/xe_*
26 * $ ls /sys/bus/event_source/devices/xe_0000_00_02.0/events/
27 * $ ls /sys/bus/event_source/devices/xe_0000_00_02.0/format/
28 *
29 * The following format parameters are available to read events,
30 * but only few are valid with each event:
31 *
32 * gt[60:63] Selects gt for the event
33 * engine_class[20:27] Selects engine-class for event
34 * engine_instance[12:19] Selects the engine-instance for the event
35 *
36 * For engine specific events (engine-*), gt, engine_class and engine_instance parameters must be
37 * set as populated by DRM_XE_DEVICE_QUERY_ENGINES.
38 *
39 * For gt specific events (gt-*) gt parameter must be passed. All other parameters will be 0.
40 *
41 * The standard perf tool can be used to grep for a certain event as well.
42 * Example:
43 *
44 * $ perf list | grep gt-c6
45 *
46 * To sample a specific event for a GT at regular intervals:
47 *
48 * $ perf stat -e <event_name,gt=> -I <interval>
49 */
50
51 #define XE_PMU_EVENT_GT_MASK GENMASK_ULL(63, 60)
52 #define XE_PMU_EVENT_ENGINE_CLASS_MASK GENMASK_ULL(27, 20)
53 #define XE_PMU_EVENT_ENGINE_INSTANCE_MASK GENMASK_ULL(19, 12)
54 #define XE_PMU_EVENT_ID_MASK GENMASK_ULL(11, 0)
55
config_to_event_id(u64 config)56 static unsigned int config_to_event_id(u64 config)
57 {
58 return FIELD_GET(XE_PMU_EVENT_ID_MASK, config);
59 }
60
config_to_engine_class(u64 config)61 static unsigned int config_to_engine_class(u64 config)
62 {
63 return FIELD_GET(XE_PMU_EVENT_ENGINE_CLASS_MASK, config);
64 }
65
config_to_engine_instance(u64 config)66 static unsigned int config_to_engine_instance(u64 config)
67 {
68 return FIELD_GET(XE_PMU_EVENT_ENGINE_INSTANCE_MASK, config);
69 }
70
config_to_gt_id(u64 config)71 static unsigned int config_to_gt_id(u64 config)
72 {
73 return FIELD_GET(XE_PMU_EVENT_GT_MASK, config);
74 }
75
76 #define XE_PMU_EVENT_GT_C6_RESIDENCY 0x01
77 #define XE_PMU_EVENT_ENGINE_ACTIVE_TICKS 0x02
78 #define XE_PMU_EVENT_ENGINE_TOTAL_TICKS 0x03
79
event_to_gt(struct perf_event * event)80 static struct xe_gt *event_to_gt(struct perf_event *event)
81 {
82 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
83 u64 gt = config_to_gt_id(event->attr.config);
84
85 return xe_device_get_gt(xe, gt);
86 }
87
event_to_hwe(struct perf_event * event)88 static struct xe_hw_engine *event_to_hwe(struct perf_event *event)
89 {
90 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
91 struct drm_xe_engine_class_instance eci;
92 u64 config = event->attr.config;
93 struct xe_hw_engine *hwe;
94
95 eci.engine_class = config_to_engine_class(config);
96 eci.engine_instance = config_to_engine_instance(config);
97 eci.gt_id = config_to_gt_id(config);
98
99 hwe = xe_hw_engine_lookup(xe, eci);
100 if (!hwe || xe_hw_engine_is_reserved(hwe))
101 return NULL;
102
103 return hwe;
104 }
105
is_engine_event(u64 config)106 static bool is_engine_event(u64 config)
107 {
108 unsigned int event_id = config_to_event_id(config);
109
110 return (event_id == XE_PMU_EVENT_ENGINE_TOTAL_TICKS ||
111 event_id == XE_PMU_EVENT_ENGINE_ACTIVE_TICKS);
112 }
113
event_gt_forcewake(struct perf_event * event)114 static bool event_gt_forcewake(struct perf_event *event)
115 {
116 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
117 u64 config = event->attr.config;
118 struct xe_gt *gt;
119 unsigned int *fw_ref;
120
121 if (!is_engine_event(config))
122 return true;
123
124 gt = xe_device_get_gt(xe, config_to_gt_id(config));
125
126 fw_ref = kzalloc(sizeof(*fw_ref), GFP_KERNEL);
127 if (!fw_ref)
128 return false;
129
130 *fw_ref = xe_force_wake_get(gt_to_fw(gt), XE_FW_GT);
131 if (!*fw_ref) {
132 kfree(fw_ref);
133 return false;
134 }
135
136 event->pmu_private = fw_ref;
137
138 return true;
139 }
140
event_supported(struct xe_pmu * pmu,unsigned int gt,unsigned int id)141 static bool event_supported(struct xe_pmu *pmu, unsigned int gt,
142 unsigned int id)
143 {
144 if (gt >= XE_MAX_GT_PER_TILE)
145 return false;
146
147 return id < sizeof(pmu->supported_events) * BITS_PER_BYTE &&
148 pmu->supported_events & BIT_ULL(id);
149 }
150
event_param_valid(struct perf_event * event)151 static bool event_param_valid(struct perf_event *event)
152 {
153 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
154 unsigned int engine_class, engine_instance;
155 u64 config = event->attr.config;
156 struct xe_gt *gt;
157
158 gt = xe_device_get_gt(xe, config_to_gt_id(config));
159 if (!gt)
160 return false;
161
162 engine_class = config_to_engine_class(config);
163 engine_instance = config_to_engine_instance(config);
164
165 switch (config_to_event_id(config)) {
166 case XE_PMU_EVENT_GT_C6_RESIDENCY:
167 if (engine_class || engine_instance)
168 return false;
169 break;
170 case XE_PMU_EVENT_ENGINE_ACTIVE_TICKS:
171 case XE_PMU_EVENT_ENGINE_TOTAL_TICKS:
172 if (!event_to_hwe(event))
173 return false;
174 break;
175 }
176
177 return true;
178 }
179
xe_pmu_event_destroy(struct perf_event * event)180 static void xe_pmu_event_destroy(struct perf_event *event)
181 {
182 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
183 struct xe_gt *gt;
184 unsigned int *fw_ref = event->pmu_private;
185
186 if (fw_ref) {
187 gt = xe_device_get_gt(xe, config_to_gt_id(event->attr.config));
188 xe_force_wake_put(gt_to_fw(gt), *fw_ref);
189 kfree(fw_ref);
190 event->pmu_private = NULL;
191 }
192
193 drm_WARN_ON(&xe->drm, event->parent);
194 xe_pm_runtime_put(xe);
195 drm_dev_put(&xe->drm);
196 }
197
xe_pmu_event_init(struct perf_event * event)198 static int xe_pmu_event_init(struct perf_event *event)
199 {
200 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
201 struct xe_pmu *pmu = &xe->pmu;
202 unsigned int id, gt;
203
204 if (!pmu->registered)
205 return -ENODEV;
206
207 if (event->attr.type != event->pmu->type)
208 return -ENOENT;
209
210 /* unsupported modes and filters */
211 if (event->attr.sample_period) /* no sampling */
212 return -EINVAL;
213
214 if (event->cpu < 0)
215 return -EINVAL;
216
217 gt = config_to_gt_id(event->attr.config);
218 id = config_to_event_id(event->attr.config);
219 if (!event_supported(pmu, gt, id))
220 return -ENOENT;
221
222 if (has_branch_stack(event))
223 return -EOPNOTSUPP;
224
225 if (!event_param_valid(event))
226 return -ENOENT;
227
228 if (!event->parent) {
229 drm_dev_get(&xe->drm);
230 xe_pm_runtime_get(xe);
231 if (!event_gt_forcewake(event)) {
232 xe_pm_runtime_put(xe);
233 drm_dev_put(&xe->drm);
234 return -EINVAL;
235 }
236 event->destroy = xe_pmu_event_destroy;
237 }
238
239 return 0;
240 }
241
read_engine_events(struct xe_gt * gt,struct perf_event * event)242 static u64 read_engine_events(struct xe_gt *gt, struct perf_event *event)
243 {
244 struct xe_hw_engine *hwe;
245 u64 val = 0;
246
247 hwe = event_to_hwe(event);
248 if (config_to_event_id(event->attr.config) == XE_PMU_EVENT_ENGINE_ACTIVE_TICKS)
249 val = xe_guc_engine_activity_active_ticks(>->uc.guc, hwe);
250 else
251 val = xe_guc_engine_activity_total_ticks(>->uc.guc, hwe);
252
253 return val;
254 }
255
__xe_pmu_event_read(struct perf_event * event)256 static u64 __xe_pmu_event_read(struct perf_event *event)
257 {
258 struct xe_gt *gt = event_to_gt(event);
259
260 if (!gt)
261 return 0;
262
263 switch (config_to_event_id(event->attr.config)) {
264 case XE_PMU_EVENT_GT_C6_RESIDENCY:
265 return xe_gt_idle_residency_msec(>->gtidle);
266 case XE_PMU_EVENT_ENGINE_ACTIVE_TICKS:
267 case XE_PMU_EVENT_ENGINE_TOTAL_TICKS:
268 return read_engine_events(gt, event);
269 }
270
271 return 0;
272 }
273
xe_pmu_event_update(struct perf_event * event)274 static void xe_pmu_event_update(struct perf_event *event)
275 {
276 struct hw_perf_event *hwc = &event->hw;
277 u64 prev, new;
278
279 prev = local64_read(&hwc->prev_count);
280 do {
281 new = __xe_pmu_event_read(event);
282 } while (!local64_try_cmpxchg(&hwc->prev_count, &prev, new));
283
284 local64_add(new - prev, &event->count);
285 }
286
xe_pmu_event_read(struct perf_event * event)287 static void xe_pmu_event_read(struct perf_event *event)
288 {
289 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
290 struct xe_pmu *pmu = &xe->pmu;
291
292 if (!pmu->registered) {
293 event->hw.state = PERF_HES_STOPPED;
294 return;
295 }
296
297 xe_pmu_event_update(event);
298 }
299
xe_pmu_enable(struct perf_event * event)300 static void xe_pmu_enable(struct perf_event *event)
301 {
302 /*
303 * Store the current counter value so we can report the correct delta
304 * for all listeners. Even when the event was already enabled and has
305 * an existing non-zero value.
306 */
307 local64_set(&event->hw.prev_count, __xe_pmu_event_read(event));
308 }
309
xe_pmu_event_start(struct perf_event * event,int flags)310 static void xe_pmu_event_start(struct perf_event *event, int flags)
311 {
312 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
313 struct xe_pmu *pmu = &xe->pmu;
314
315 if (!pmu->registered)
316 return;
317
318 xe_pmu_enable(event);
319 event->hw.state = 0;
320 }
321
xe_pmu_event_stop(struct perf_event * event,int flags)322 static void xe_pmu_event_stop(struct perf_event *event, int flags)
323 {
324 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
325 struct xe_pmu *pmu = &xe->pmu;
326
327 if (pmu->registered)
328 if (flags & PERF_EF_UPDATE)
329 xe_pmu_event_update(event);
330
331 event->hw.state = PERF_HES_STOPPED;
332 }
333
xe_pmu_event_add(struct perf_event * event,int flags)334 static int xe_pmu_event_add(struct perf_event *event, int flags)
335 {
336 struct xe_device *xe = container_of(event->pmu, typeof(*xe), pmu.base);
337 struct xe_pmu *pmu = &xe->pmu;
338
339 if (!pmu->registered)
340 return -ENODEV;
341
342 if (flags & PERF_EF_START)
343 xe_pmu_event_start(event, flags);
344
345 return 0;
346 }
347
xe_pmu_event_del(struct perf_event * event,int flags)348 static void xe_pmu_event_del(struct perf_event *event, int flags)
349 {
350 xe_pmu_event_stop(event, PERF_EF_UPDATE);
351 }
352
353 PMU_FORMAT_ATTR(gt, "config:60-63");
354 PMU_FORMAT_ATTR(engine_class, "config:20-27");
355 PMU_FORMAT_ATTR(engine_instance, "config:12-19");
356 PMU_FORMAT_ATTR(event, "config:0-11");
357
358 static struct attribute *pmu_format_attrs[] = {
359 &format_attr_event.attr,
360 &format_attr_engine_class.attr,
361 &format_attr_engine_instance.attr,
362 &format_attr_gt.attr,
363 NULL,
364 };
365
366 static const struct attribute_group pmu_format_attr_group = {
367 .name = "format",
368 .attrs = pmu_format_attrs,
369 };
370
event_attr_show(struct device * dev,struct device_attribute * attr,char * buf)371 static ssize_t event_attr_show(struct device *dev,
372 struct device_attribute *attr, char *buf)
373 {
374 struct perf_pmu_events_attr *pmu_attr =
375 container_of(attr, struct perf_pmu_events_attr, attr);
376
377 return sprintf(buf, "event=%#04llx\n", pmu_attr->id);
378 }
379
380 #define XE_EVENT_ATTR(name_, v_, id_) \
381 PMU_EVENT_ATTR(name_, pmu_event_ ## v_, id_, event_attr_show)
382
383 #define XE_EVENT_ATTR_UNIT(name_, v_, unit_) \
384 PMU_EVENT_ATTR_STRING(name_.unit, pmu_event_unit_ ## v_, unit_)
385
386 #define XE_EVENT_ATTR_GROUP(v_, id_, ...) \
387 static struct attribute *pmu_attr_ ##v_[] = { \
388 __VA_ARGS__, \
389 NULL \
390 }; \
391 static umode_t is_visible_##v_(struct kobject *kobj, \
392 struct attribute *attr, int idx) \
393 { \
394 struct perf_pmu_events_attr *pmu_attr; \
395 struct xe_pmu *pmu; \
396 \
397 pmu_attr = container_of(attr, typeof(*pmu_attr), attr.attr); \
398 pmu = container_of(dev_get_drvdata(kobj_to_dev(kobj)), \
399 typeof(*pmu), base); \
400 \
401 return event_supported(pmu, 0, id_) ? attr->mode : 0; \
402 } \
403 static const struct attribute_group pmu_group_ ##v_ = { \
404 .name = "events", \
405 .attrs = pmu_attr_ ## v_, \
406 .is_visible = is_visible_ ## v_, \
407 }
408
409 #define XE_EVENT_ATTR_SIMPLE(name_, v_, id_, unit_) \
410 XE_EVENT_ATTR(name_, v_, id_) \
411 XE_EVENT_ATTR_UNIT(name_, v_, unit_) \
412 XE_EVENT_ATTR_GROUP(v_, id_, &pmu_event_ ##v_.attr.attr, \
413 &pmu_event_unit_ ##v_.attr.attr)
414
415 #define XE_EVENT_ATTR_NOUNIT(name_, v_, id_) \
416 XE_EVENT_ATTR(name_, v_, id_) \
417 XE_EVENT_ATTR_GROUP(v_, id_, &pmu_event_ ##v_.attr.attr)
418
419 XE_EVENT_ATTR_SIMPLE(gt-c6-residency, gt_c6_residency, XE_PMU_EVENT_GT_C6_RESIDENCY, "ms");
420 XE_EVENT_ATTR_NOUNIT(engine-active-ticks, engine_active_ticks, XE_PMU_EVENT_ENGINE_ACTIVE_TICKS);
421 XE_EVENT_ATTR_NOUNIT(engine-total-ticks, engine_total_ticks, XE_PMU_EVENT_ENGINE_TOTAL_TICKS);
422
423 static struct attribute *pmu_empty_event_attrs[] = {
424 /* Empty - all events are added as groups with .attr_update() */
425 NULL,
426 };
427
428 static const struct attribute_group pmu_events_attr_group = {
429 .name = "events",
430 .attrs = pmu_empty_event_attrs,
431 };
432
433 static const struct attribute_group *pmu_events_attr_update[] = {
434 &pmu_group_gt_c6_residency,
435 &pmu_group_engine_active_ticks,
436 &pmu_group_engine_total_ticks,
437 NULL,
438 };
439
set_supported_events(struct xe_pmu * pmu)440 static void set_supported_events(struct xe_pmu *pmu)
441 {
442 struct xe_device *xe = container_of(pmu, typeof(*xe), pmu);
443 struct xe_gt *gt = xe_device_get_gt(xe, 0);
444
445 if (!xe->info.skip_guc_pc)
446 pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_GT_C6_RESIDENCY);
447
448 if (xe_guc_engine_activity_supported(>->uc.guc)) {
449 pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_ENGINE_ACTIVE_TICKS);
450 pmu->supported_events |= BIT_ULL(XE_PMU_EVENT_ENGINE_TOTAL_TICKS);
451 }
452 }
453
454 /**
455 * xe_pmu_unregister() - Remove/cleanup PMU registration
456 * @arg: Ptr to pmu
457 */
xe_pmu_unregister(void * arg)458 static void xe_pmu_unregister(void *arg)
459 {
460 struct xe_pmu *pmu = arg;
461 struct xe_device *xe = container_of(pmu, typeof(*xe), pmu);
462
463 if (!pmu->registered)
464 return;
465
466 pmu->registered = false;
467
468 perf_pmu_unregister(&pmu->base);
469 kfree(pmu->name);
470 }
471
472 /**
473 * xe_pmu_register() - Define basic PMU properties for Xe and add event callbacks.
474 * @pmu: the PMU object
475 *
476 * Returns 0 on success and an appropriate error code otherwise
477 */
xe_pmu_register(struct xe_pmu * pmu)478 int xe_pmu_register(struct xe_pmu *pmu)
479 {
480 struct xe_device *xe = container_of(pmu, typeof(*xe), pmu);
481 static const struct attribute_group *attr_groups[] = {
482 &pmu_format_attr_group,
483 &pmu_events_attr_group,
484 NULL
485 };
486 int ret = -ENOMEM;
487 char *name;
488
489 BUILD_BUG_ON(XE_MAX_GT_PER_TILE != XE_PMU_MAX_GT);
490
491 if (IS_SRIOV_VF(xe))
492 return 0;
493
494 name = kasprintf(GFP_KERNEL, "xe_%s",
495 dev_name(xe->drm.dev));
496 if (!name)
497 goto err;
498
499 /* tools/perf reserves colons as special. */
500 strreplace(name, ':', '_');
501
502 pmu->name = name;
503 pmu->base.attr_groups = attr_groups;
504 pmu->base.attr_update = pmu_events_attr_update;
505 pmu->base.scope = PERF_PMU_SCOPE_SYS_WIDE;
506 pmu->base.module = THIS_MODULE;
507 pmu->base.task_ctx_nr = perf_invalid_context;
508 pmu->base.event_init = xe_pmu_event_init;
509 pmu->base.add = xe_pmu_event_add;
510 pmu->base.del = xe_pmu_event_del;
511 pmu->base.start = xe_pmu_event_start;
512 pmu->base.stop = xe_pmu_event_stop;
513 pmu->base.read = xe_pmu_event_read;
514
515 set_supported_events(pmu);
516
517 ret = perf_pmu_register(&pmu->base, pmu->name, -1);
518 if (ret)
519 goto err_name;
520
521 pmu->registered = true;
522
523 return devm_add_action_or_reset(xe->drm.dev, xe_pmu_unregister, pmu);
524
525 err_name:
526 kfree(name);
527 err:
528 drm_err(&xe->drm, "Failed to register PMU (ret=%d)!\n", ret);
529
530 return ret;
531 }
532