1 // SPDX-License-Identifier: GPL-2.0
2 /* Manage affinity to optimize IPIs inside the kernel perf API. */
3 #define _GNU_SOURCE 1
4 #include <sched.h>
5 #include <stdlib.h>
6 #include <linux/bitmap.h>
7 #include <linux/zalloc.h>
8 #include <perf/cpumap.h>
9 #include "perf.h"
10 #include "cpumap.h"
11 #include "affinity.h"
12
get_cpu_set_size(void)13 static int get_cpu_set_size(void)
14 {
15 int sz = cpu__max_cpu().cpu + 8 - 1;
16 /*
17 * sched_getaffinity doesn't like masks smaller than the kernel.
18 * Hopefully that's big enough.
19 */
20 if (sz < 4096)
21 sz = 4096;
22 return sz / 8;
23 }
24
affinity__setup(struct affinity * a)25 int affinity__setup(struct affinity *a)
26 {
27 int cpu_set_size = get_cpu_set_size();
28
29 a->orig_cpus = bitmap_zalloc(cpu_set_size * 8);
30 if (!a->orig_cpus)
31 return -1;
32 sched_getaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus);
33 a->sched_cpus = bitmap_zalloc(cpu_set_size * 8);
34 if (!a->sched_cpus) {
35 zfree(&a->orig_cpus);
36 return -1;
37 }
38 bitmap_zero((unsigned long *)a->sched_cpus, cpu_set_size);
39 a->changed = false;
40 return 0;
41 }
42
43 /*
44 * perf_event_open does an IPI internally to the target CPU.
45 * It is more efficient to change perf's affinity to the target
46 * CPU and then set up all events on that CPU, so we amortize
47 * CPU communication.
48 */
affinity__set(struct affinity * a,int cpu)49 void affinity__set(struct affinity *a, int cpu)
50 {
51 int cpu_set_size = get_cpu_set_size();
52
53 /*
54 * Return:
55 * - if cpu is -1
56 * - restrict out of bound access to sched_cpus
57 */
58 if (cpu == -1 || ((cpu >= (cpu_set_size * 8))))
59 return;
60
61 a->changed = true;
62 __set_bit(cpu, a->sched_cpus);
63 /*
64 * We ignore errors because affinity is just an optimization.
65 * This could happen for example with isolated CPUs or cpusets.
66 * In this case the IPIs inside the kernel's perf API still work.
67 */
68 sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->sched_cpus);
69 __clear_bit(cpu, a->sched_cpus);
70 }
71
__affinity__cleanup(struct affinity * a)72 static void __affinity__cleanup(struct affinity *a)
73 {
74 int cpu_set_size = get_cpu_set_size();
75
76 if (a->changed)
77 sched_setaffinity(0, cpu_set_size, (cpu_set_t *)a->orig_cpus);
78 zfree(&a->sched_cpus);
79 zfree(&a->orig_cpus);
80 }
81
affinity__cleanup(struct affinity * a)82 void affinity__cleanup(struct affinity *a)
83 {
84 if (a != NULL)
85 __affinity__cleanup(a);
86 }
87
cpu_map__set_affinity(const struct perf_cpu_map * cpumap)88 void cpu_map__set_affinity(const struct perf_cpu_map *cpumap)
89 {
90 int cpu_set_size = get_cpu_set_size();
91 unsigned long *cpuset = bitmap_zalloc(cpu_set_size * 8);
92 struct perf_cpu cpu;
93 int idx;
94
95 if (!cpuset)
96 return;
97
98 perf_cpu_map__for_each_cpu_skip_any(cpu, idx, cpumap)
99 __set_bit(cpu.cpu, cpuset);
100
101 sched_setaffinity(0, cpu_set_size, (cpu_set_t *)cpuset);
102 zfree(&cpuset);
103 }
104