1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * turbostat -- show CPU frequency and C-state residency
4 * on modern Intel and AMD processors.
5 *
6 * Copyright (c) 2010 - 2026 Intel Corporation
7 * Len Brown <len.brown@intel.com>
8 */
9
10 #define _GNU_SOURCE
11 #include MSRHEADER
12
13 // copied from arch/x86/include/asm/cpu_device_id.h
14 #define VFM_MODEL_BIT 0
15 #define VFM_FAMILY_BIT 8
16 #define VFM_VENDOR_BIT 16
17 #define VFM_RSVD_BIT 24
18
19 #define VFM_MODEL_MASK GENMASK(VFM_FAMILY_BIT - 1, VFM_MODEL_BIT)
20 #define VFM_FAMILY_MASK GENMASK(VFM_VENDOR_BIT - 1, VFM_FAMILY_BIT)
21 #define VFM_VENDOR_MASK GENMASK(VFM_RSVD_BIT - 1, VFM_VENDOR_BIT)
22
23 #define VFM_MODEL(vfm) (((vfm) & VFM_MODEL_MASK) >> VFM_MODEL_BIT)
24 #define VFM_FAMILY(vfm) (((vfm) & VFM_FAMILY_MASK) >> VFM_FAMILY_BIT)
25 #define VFM_VENDOR(vfm) (((vfm) & VFM_VENDOR_MASK) >> VFM_VENDOR_BIT)
26
27 #define VFM_MAKE(_vendor, _family, _model) ( \
28 ((_model) << VFM_MODEL_BIT) | \
29 ((_family) << VFM_FAMILY_BIT) | \
30 ((_vendor) << VFM_VENDOR_BIT) \
31 )
32 // end copied section
33
34 #define CPUID_LEAF_MODEL_ID 0x1A
35 #define CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT 24
36
37 #define X86_VENDOR_INTEL 0
38
39 #include INTEL_FAMILY_HEADER
40 #include BUILD_BUG_HEADER
41 #include <stdarg.h>
42 #include <stdio.h>
43 #include <err.h>
44 #include <unistd.h>
45 #include <sys/types.h>
46 #include <sys/wait.h>
47 #include <sys/stat.h>
48 #include <sys/select.h>
49 #include <sys/resource.h>
50 #include <sys/mman.h>
51 #include <fcntl.h>
52 #include <signal.h>
53 #include <sys/time.h>
54 #include <stdlib.h>
55 #include <getopt.h>
56 #include <dirent.h>
57 #include <string.h>
58 #include <ctype.h>
59 #include <sched.h>
60 #include <time.h>
61 #include <cpuid.h>
62 #include <sys/capability.h>
63 #include <errno.h>
64 #include <math.h>
65 #include <linux/perf_event.h>
66 #include <asm/unistd.h>
67 #include <stdbool.h>
68 #include <assert.h>
69 #include <linux/kernel.h>
70 #include <limits.h>
71
72 #define UNUSED(x) (void)(x)
73
74 /*
75 * This list matches the column headers, except
76 * 1. built-in only, the sysfs counters are not here -- we learn of those at run-time
77 * 2. Core and CPU are moved to the end, we can't have strings that contain them
78 * matching on them for --show and --hide.
79 */
80
81 /*
82 * buffer size used by sscanf() for added column names
83 * Usually truncated to 7 characters, but also handles 18 columns for raw 64-bit counters
84 */
85 #define NAME_BYTES 20
86 #define PATH_BYTES 128
87 #define PERF_NAME_BYTES 128
88
89 #define MAX_NOFILE 0x8000
90
91 #define COUNTER_KIND_PERF_PREFIX "perf/"
92 #define COUNTER_KIND_PERF_PREFIX_LEN strlen(COUNTER_KIND_PERF_PREFIX)
93 #define PERF_DEV_NAME_BYTES 32
94 #define PERF_EVT_NAME_BYTES 32
95
96 #define INTEL_ECORE_TYPE 0x20
97 #define INTEL_PCORE_TYPE 0x40
98
99 #define ROUND_UP_TO_PAGE_SIZE(n) (((n) + 0x1000UL-1UL) & ~(0x1000UL-1UL))
100
101 enum counter_scope { SCOPE_CPU, SCOPE_CORE, SCOPE_PACKAGE };
102 enum counter_type { COUNTER_ITEMS, COUNTER_CYCLES, COUNTER_SECONDS, COUNTER_USEC, COUNTER_K2M };
103 enum counter_format { FORMAT_RAW, FORMAT_DELTA, FORMAT_PERCENT, FORMAT_AVERAGE };
104 enum counter_source { COUNTER_SOURCE_NONE, COUNTER_SOURCE_PERF, COUNTER_SOURCE_MSR };
105
106 struct perf_counter_info {
107 struct perf_counter_info *next;
108
109 /* How to open the counter / What counter it is. */
110 char device[PERF_DEV_NAME_BYTES];
111 char event[PERF_EVT_NAME_BYTES];
112
113 /* How to show/format the counter. */
114 char name[PERF_NAME_BYTES];
115 unsigned int width;
116 enum counter_scope scope;
117 enum counter_type type;
118 enum counter_format format;
119 double scale;
120
121 /* For reading the counter. */
122 int *fd_perf_per_domain;
123 size_t num_domains;
124 };
125
126 struct sysfs_path {
127 char path[PATH_BYTES];
128 int id;
129 struct sysfs_path *next;
130 };
131
132 struct msr_counter {
133 unsigned int msr_num;
134 char name[NAME_BYTES];
135 struct sysfs_path *sp;
136 unsigned int width;
137 enum counter_type type;
138 enum counter_format format;
139 struct msr_counter *next;
140 unsigned int flags;
141 #define FLAGS_HIDE (1 << 0)
142 #define FLAGS_SHOW (1 << 1)
143 #define SYSFS_PERCPU (1 << 1)
144 };
145 static int use_android_msr_path;
146
147 struct msr_counter bic[] = {
148 { 0x0, "usec", NULL, 0, 0, 0, NULL, 0 },
149 { 0x0, "Time_Of_Day_Seconds", NULL, 0, 0, 0, NULL, 0 },
150 { 0x0, "Package", NULL, 0, 0, 0, NULL, 0 },
151 { 0x0, "Node", NULL, 0, 0, 0, NULL, 0 },
152 { 0x0, "Avg_MHz", NULL, 0, 0, 0, NULL, 0 },
153 { 0x0, "Busy%", NULL, 0, 0, 0, NULL, 0 },
154 { 0x0, "Bzy_MHz", NULL, 0, 0, 0, NULL, 0 },
155 { 0x0, "TSC_MHz", NULL, 0, 0, 0, NULL, 0 },
156 { 0x0, "IRQ", NULL, 0, 0, 0, NULL, 0 },
157 { 0x0, "SMI", NULL, 32, 0, FORMAT_DELTA, NULL, 0 },
158 { 0x0, "cpuidle", NULL, 0, 0, 0, NULL, 0 },
159 { 0x0, "CPU%c1", NULL, 0, 0, 0, NULL, 0 },
160 { 0x0, "CPU%c3", NULL, 0, 0, 0, NULL, 0 },
161 { 0x0, "CPU%c6", NULL, 0, 0, 0, NULL, 0 },
162 { 0x0, "CPU%c7", NULL, 0, 0, 0, NULL, 0 },
163 { 0x0, "ThreadC", NULL, 0, 0, 0, NULL, 0 },
164 { 0x0, "CoreTmp", NULL, 0, 0, 0, NULL, 0 },
165 { 0x0, "CoreCnt", NULL, 0, 0, 0, NULL, 0 },
166 { 0x0, "PkgTmp", NULL, 0, 0, 0, NULL, 0 },
167 { 0x0, "GFX%rc6", NULL, 0, 0, 0, NULL, 0 },
168 { 0x0, "GFXMHz", NULL, 0, 0, 0, NULL, 0 },
169 { 0x0, "Pkg%pc2", NULL, 0, 0, 0, NULL, 0 },
170 { 0x0, "Pkg%pc3", NULL, 0, 0, 0, NULL, 0 },
171 { 0x0, "Pkg%pc6", NULL, 0, 0, 0, NULL, 0 },
172 { 0x0, "Pkg%pc7", NULL, 0, 0, 0, NULL, 0 },
173 { 0x0, "Pkg%pc8", NULL, 0, 0, 0, NULL, 0 },
174 { 0x0, "Pkg%pc9", NULL, 0, 0, 0, NULL, 0 },
175 { 0x0, "Pk%pc10", NULL, 0, 0, 0, NULL, 0 },
176 { 0x0, "CPU%LPI", NULL, 0, 0, 0, NULL, 0 },
177 { 0x0, "SYS%LPI", NULL, 0, 0, 0, NULL, 0 },
178 { 0x0, "PkgWatt", NULL, 0, 0, 0, NULL, 0 },
179 { 0x0, "CorWatt", NULL, 0, 0, 0, NULL, 0 },
180 { 0x0, "GFXWatt", NULL, 0, 0, 0, NULL, 0 },
181 { 0x0, "PkgCnt", NULL, 0, 0, 0, NULL, 0 },
182 { 0x0, "RAMWatt", NULL, 0, 0, 0, NULL, 0 },
183 { 0x0, "PKG_%", NULL, 0, 0, 0, NULL, 0 },
184 { 0x0, "RAM_%", NULL, 0, 0, 0, NULL, 0 },
185 { 0x0, "Pkg_J", NULL, 0, 0, 0, NULL, 0 },
186 { 0x0, "Cor_J", NULL, 0, 0, 0, NULL, 0 },
187 { 0x0, "GFX_J", NULL, 0, 0, 0, NULL, 0 },
188 { 0x0, "RAM_J", NULL, 0, 0, 0, NULL, 0 },
189 { 0x0, "Mod%c6", NULL, 0, 0, 0, NULL, 0 },
190 { 0x0, "Totl%C0", NULL, 0, 0, 0, NULL, 0 },
191 { 0x0, "Any%C0", NULL, 0, 0, 0, NULL, 0 },
192 { 0x0, "GFX%C0", NULL, 0, 0, 0, NULL, 0 },
193 { 0x0, "CPUGFX%", NULL, 0, 0, 0, NULL, 0 },
194 { 0x0, "Core", NULL, 0, 0, 0, NULL, 0 },
195 { 0x0, "CPU", NULL, 0, 0, 0, NULL, 0 },
196 { 0x0, "APIC", NULL, 0, 0, 0, NULL, 0 },
197 { 0x0, "X2APIC", NULL, 0, 0, 0, NULL, 0 },
198 { 0x0, "Die", NULL, 0, 0, 0, NULL, 0 },
199 { 0x0, "L3", NULL, 0, 0, 0, NULL, 0 },
200 { 0x0, "GFXAMHz", NULL, 0, 0, 0, NULL, 0 },
201 { 0x0, "IPC", NULL, 0, 0, 0, NULL, 0 },
202 { 0x0, "CoreThr", NULL, 0, 0, 0, NULL, 0 },
203 { 0x0, "UncMHz", NULL, 0, 0, 0, NULL, 0 },
204 { 0x0, "SAM%mc6", NULL, 0, 0, 0, NULL, 0 },
205 { 0x0, "SAMMHz", NULL, 0, 0, 0, NULL, 0 },
206 { 0x0, "SAMAMHz", NULL, 0, 0, 0, NULL, 0 },
207 { 0x0, "Die%c6", NULL, 0, 0, 0, NULL, 0 },
208 { 0x0, "SysWatt", NULL, 0, 0, 0, NULL, 0 },
209 { 0x0, "Sys_J", NULL, 0, 0, 0, NULL, 0 },
210 { 0x0, "NMI", NULL, 0, 0, 0, NULL, 0 },
211 { 0x0, "CPU%c1e", NULL, 0, 0, 0, NULL, 0 },
212 { 0x0, "pct_idle", NULL, 0, 0, 0, NULL, 0 },
213 { 0x0, "LLCMRPS", NULL, 0, 0, 0, NULL, 0 },
214 { 0x0, "LLC%hit", NULL, 0, 0, 0, NULL, 0 },
215 { 0x0, "L2MRPS", NULL, 0, 0, 0, NULL, 0 },
216 { 0x0, "L2%hit", NULL, 0, 0, 0, NULL, 0 },
217 };
218
219 /* n.b. bic_names must match the order in bic[], above */
220 enum bic_names {
221 BIC_USEC,
222 BIC_TOD,
223 BIC_Package,
224 BIC_Node,
225 BIC_Avg_MHz,
226 BIC_Busy,
227 BIC_Bzy_MHz,
228 BIC_TSC_MHz,
229 BIC_IRQ,
230 BIC_SMI,
231 BIC_cpuidle,
232 BIC_CPU_c1,
233 BIC_CPU_c3,
234 BIC_CPU_c6,
235 BIC_CPU_c7,
236 BIC_ThreadC,
237 BIC_CoreTmp,
238 BIC_CoreCnt,
239 BIC_PkgTmp,
240 BIC_GFX_rc6,
241 BIC_GFXMHz,
242 BIC_Pkgpc2,
243 BIC_Pkgpc3,
244 BIC_Pkgpc6,
245 BIC_Pkgpc7,
246 BIC_Pkgpc8,
247 BIC_Pkgpc9,
248 BIC_Pkgpc10,
249 BIC_CPU_LPI,
250 BIC_SYS_LPI,
251 BIC_PkgWatt,
252 BIC_CorWatt,
253 BIC_GFXWatt,
254 BIC_PkgCnt,
255 BIC_RAMWatt,
256 BIC_PKG__,
257 BIC_RAM__,
258 BIC_Pkg_J,
259 BIC_Cor_J,
260 BIC_GFX_J,
261 BIC_RAM_J,
262 BIC_Mod_c6,
263 BIC_Totl_c0,
264 BIC_Any_c0,
265 BIC_GFX_c0,
266 BIC_CPUGFX,
267 BIC_Core,
268 BIC_CPU,
269 BIC_APIC,
270 BIC_X2APIC,
271 BIC_Die,
272 BIC_L3,
273 BIC_GFXACTMHz,
274 BIC_IPC,
275 BIC_CORE_THROT_CNT,
276 BIC_UNCORE_MHZ,
277 BIC_SAM_mc6,
278 BIC_SAMMHz,
279 BIC_SAMACTMHz,
280 BIC_Diec6,
281 BIC_SysWatt,
282 BIC_Sys_J,
283 BIC_NMI,
284 BIC_CPU_c1e,
285 BIC_pct_idle,
286 BIC_LLC_MRPS,
287 BIC_LLC_HIT,
288 BIC_L2_MRPS,
289 BIC_L2_HIT,
290 MAX_BIC
291 };
292
print_bic_set(char * s,cpu_set_t * set)293 void print_bic_set(char *s, cpu_set_t *set)
294 {
295 int i;
296
297 assert(MAX_BIC < CPU_SETSIZE);
298
299 printf("%s:", s);
300
301 for (i = 0; i < MAX_BIC; ++i) {
302
303 if (CPU_ISSET(i, set))
304 printf(" %s", bic[i].name);
305 }
306 putchar('\n');
307 }
308
309 static cpu_set_t bic_group_topology;
310 static cpu_set_t bic_group_thermal_pwr;
311 static cpu_set_t bic_group_frequency;
312 static cpu_set_t bic_group_hw_idle;
313 static cpu_set_t bic_group_sw_idle;
314 static cpu_set_t bic_group_idle;
315 static cpu_set_t bic_group_cache;
316 static cpu_set_t bic_group_other;
317 static cpu_set_t bic_group_disabled_by_default;
318 static cpu_set_t bic_enabled;
319 static cpu_set_t bic_present;
320
321 /* modify */
322 #define BIC_INIT(set) CPU_ZERO(set)
323
324 #define SET_BIC(COUNTER_NUMBER, set) CPU_SET(COUNTER_NUMBER, set)
325 #define CLR_BIC(COUNTER_NUMBER, set) CPU_CLR(COUNTER_NUMBER, set)
326
327 #define BIC_PRESENT(COUNTER_NUMBER) SET_BIC(COUNTER_NUMBER, &bic_present)
328 #define BIC_NOT_PRESENT(COUNTER_NUMBER) CPU_CLR(COUNTER_NUMBER, &bic_present)
329
330 /* test */
331 #define BIC_IS_ENABLED(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_enabled)
332 #define DO_BIC_READ(COUNTER_NUMBER) CPU_ISSET(COUNTER_NUMBER, &bic_present)
333 #define DO_BIC(COUNTER_NUMBER) (CPU_ISSET(COUNTER_NUMBER, &bic_enabled) && CPU_ISSET(COUNTER_NUMBER, &bic_present))
334
bic_set_all(cpu_set_t * set)335 static void bic_set_all(cpu_set_t *set)
336 {
337 int i;
338
339 assert(MAX_BIC < CPU_SETSIZE);
340
341 for (i = 0; i < MAX_BIC; ++i)
342 SET_BIC(i, set);
343 }
344
345 /*
346 * bic_clear_bits()
347 * clear all the bits from "clr" in "dst"
348 */
bic_clear_bits(cpu_set_t * dst,cpu_set_t * clr)349 static void bic_clear_bits(cpu_set_t *dst, cpu_set_t *clr)
350 {
351 int i;
352
353 assert(MAX_BIC < CPU_SETSIZE);
354
355 for (i = 0; i < MAX_BIC; ++i)
356 if (CPU_ISSET(i, clr))
357 CLR_BIC(i, dst);
358 }
359
bic_groups_init(void)360 static void bic_groups_init(void)
361 {
362 BIC_INIT(&bic_group_topology);
363 SET_BIC(BIC_Package, &bic_group_topology);
364 SET_BIC(BIC_Node, &bic_group_topology);
365 SET_BIC(BIC_CoreCnt, &bic_group_topology);
366 SET_BIC(BIC_PkgCnt, &bic_group_topology);
367 SET_BIC(BIC_Core, &bic_group_topology);
368 SET_BIC(BIC_CPU, &bic_group_topology);
369 SET_BIC(BIC_Die, &bic_group_topology);
370 SET_BIC(BIC_L3, &bic_group_topology);
371
372 BIC_INIT(&bic_group_thermal_pwr);
373 SET_BIC(BIC_CoreTmp, &bic_group_thermal_pwr);
374 SET_BIC(BIC_PkgTmp, &bic_group_thermal_pwr);
375 SET_BIC(BIC_PkgWatt, &bic_group_thermal_pwr);
376 SET_BIC(BIC_CorWatt, &bic_group_thermal_pwr);
377 SET_BIC(BIC_GFXWatt, &bic_group_thermal_pwr);
378 SET_BIC(BIC_RAMWatt, &bic_group_thermal_pwr);
379 SET_BIC(BIC_PKG__, &bic_group_thermal_pwr);
380 SET_BIC(BIC_RAM__, &bic_group_thermal_pwr);
381 SET_BIC(BIC_SysWatt, &bic_group_thermal_pwr);
382
383 BIC_INIT(&bic_group_frequency);
384 SET_BIC(BIC_Avg_MHz, &bic_group_frequency);
385 SET_BIC(BIC_Busy, &bic_group_frequency);
386 SET_BIC(BIC_Bzy_MHz, &bic_group_frequency);
387 SET_BIC(BIC_TSC_MHz, &bic_group_frequency);
388 SET_BIC(BIC_GFXMHz, &bic_group_frequency);
389 SET_BIC(BIC_GFXACTMHz, &bic_group_frequency);
390 SET_BIC(BIC_SAMMHz, &bic_group_frequency);
391 SET_BIC(BIC_SAMACTMHz, &bic_group_frequency);
392 SET_BIC(BIC_UNCORE_MHZ, &bic_group_frequency);
393
394 BIC_INIT(&bic_group_hw_idle);
395 SET_BIC(BIC_Busy, &bic_group_hw_idle);
396 SET_BIC(BIC_CPU_c1, &bic_group_hw_idle);
397 SET_BIC(BIC_CPU_c3, &bic_group_hw_idle);
398 SET_BIC(BIC_CPU_c6, &bic_group_hw_idle);
399 SET_BIC(BIC_CPU_c7, &bic_group_hw_idle);
400 SET_BIC(BIC_GFX_rc6, &bic_group_hw_idle);
401 SET_BIC(BIC_Pkgpc2, &bic_group_hw_idle);
402 SET_BIC(BIC_Pkgpc3, &bic_group_hw_idle);
403 SET_BIC(BIC_Pkgpc6, &bic_group_hw_idle);
404 SET_BIC(BIC_Pkgpc7, &bic_group_hw_idle);
405 SET_BIC(BIC_Pkgpc8, &bic_group_hw_idle);
406 SET_BIC(BIC_Pkgpc9, &bic_group_hw_idle);
407 SET_BIC(BIC_Pkgpc10, &bic_group_hw_idle);
408 SET_BIC(BIC_CPU_LPI, &bic_group_hw_idle);
409 SET_BIC(BIC_SYS_LPI, &bic_group_hw_idle);
410 SET_BIC(BIC_Mod_c6, &bic_group_hw_idle);
411 SET_BIC(BIC_Totl_c0, &bic_group_hw_idle);
412 SET_BIC(BIC_Any_c0, &bic_group_hw_idle);
413 SET_BIC(BIC_GFX_c0, &bic_group_hw_idle);
414 SET_BIC(BIC_CPUGFX, &bic_group_hw_idle);
415 SET_BIC(BIC_SAM_mc6, &bic_group_hw_idle);
416 SET_BIC(BIC_Diec6, &bic_group_hw_idle);
417
418 BIC_INIT(&bic_group_sw_idle);
419 SET_BIC(BIC_Busy, &bic_group_sw_idle);
420 SET_BIC(BIC_cpuidle, &bic_group_sw_idle);
421 SET_BIC(BIC_pct_idle, &bic_group_sw_idle);
422
423 BIC_INIT(&bic_group_idle);
424
425 CPU_OR(&bic_group_idle, &bic_group_idle, &bic_group_hw_idle);
426 SET_BIC(BIC_pct_idle, &bic_group_idle);
427
428 BIC_INIT(&bic_group_cache);
429 SET_BIC(BIC_LLC_MRPS, &bic_group_cache);
430 SET_BIC(BIC_LLC_HIT, &bic_group_cache);
431 SET_BIC(BIC_L2_MRPS, &bic_group_cache);
432 SET_BIC(BIC_L2_HIT, &bic_group_cache);
433
434 BIC_INIT(&bic_group_other);
435 SET_BIC(BIC_IRQ, &bic_group_other);
436 SET_BIC(BIC_NMI, &bic_group_other);
437 SET_BIC(BIC_SMI, &bic_group_other);
438 SET_BIC(BIC_ThreadC, &bic_group_other);
439 SET_BIC(BIC_CoreTmp, &bic_group_other);
440 SET_BIC(BIC_IPC, &bic_group_other);
441
442 BIC_INIT(&bic_group_disabled_by_default);
443 SET_BIC(BIC_USEC, &bic_group_disabled_by_default);
444 SET_BIC(BIC_TOD, &bic_group_disabled_by_default);
445 SET_BIC(BIC_cpuidle, &bic_group_disabled_by_default);
446 SET_BIC(BIC_APIC, &bic_group_disabled_by_default);
447 SET_BIC(BIC_X2APIC, &bic_group_disabled_by_default);
448
449 BIC_INIT(&bic_enabled);
450 bic_set_all(&bic_enabled);
451 bic_clear_bits(&bic_enabled, &bic_group_disabled_by_default);
452
453 BIC_INIT(&bic_present);
454 SET_BIC(BIC_USEC, &bic_present);
455 SET_BIC(BIC_TOD, &bic_present);
456 SET_BIC(BIC_cpuidle, &bic_present);
457 SET_BIC(BIC_APIC, &bic_present);
458 SET_BIC(BIC_X2APIC, &bic_present);
459 SET_BIC(BIC_pct_idle, &bic_present);
460 }
461
462 /*
463 * MSR_PKG_CST_CONFIG_CONTROL decoding for pkg_cstate_limit:
464 * If you change the values, note they are used both in comparisons
465 * (>= PCL__7) and to index pkg_cstate_limit_strings[].
466 */
467 #define PCLUKN 0 /* Unknown */
468 #define PCLRSV 1 /* Reserved */
469 #define PCL__0 2 /* PC0 */
470 #define PCL__1 3 /* PC1 */
471 #define PCL__2 4 /* PC2 */
472 #define PCL__3 5 /* PC3 */
473 #define PCL__4 6 /* PC4 */
474 #define PCL__6 7 /* PC6 */
475 #define PCL_6N 8 /* PC6 No Retention */
476 #define PCL_6R 9 /* PC6 Retention */
477 #define PCL__7 10 /* PC7 */
478 #define PCL_7S 11 /* PC7 Shrink */
479 #define PCL__8 12 /* PC8 */
480 #define PCL__9 13 /* PC9 */
481 #define PCL_10 14 /* PC10 */
482 #define PCLUNL 15 /* Unlimited */
483
484 char *proc_stat = "/proc/stat";
485 FILE *outf;
486 int *fd_percpu;
487 int *fd_instr_count_percpu;
488 int *fd_llc_percpu;
489 int *fd_l2_percpu;
490 struct timeval interval_tv = { 5, 0 };
491 struct timespec interval_ts = { 5, 0 };
492
493 unsigned int num_iterations;
494 unsigned int header_iterations;
495 unsigned int debug;
496 unsigned int quiet;
497 unsigned int shown;
498 unsigned int sums_need_wide_columns;
499 unsigned int rapl_joules;
500 unsigned int valid_rapl_msrs;
501 unsigned int summary_only;
502 unsigned int list_header_only;
503 unsigned int dump_only;
504 unsigned int force_load;
505 unsigned int cpuid_has_aperf_mperf;
506 unsigned int cpuid_has_hv;
507 unsigned int has_aperf_access;
508 unsigned int has_epb;
509 unsigned int has_turbo;
510 unsigned int is_hybrid;
511 unsigned int units = 1000000; /* MHz etc */
512 unsigned int genuine_intel;
513 unsigned int authentic_amd;
514 unsigned int hygon_genuine;
515 unsigned int max_level, max_extended_level;
516 unsigned int has_invariant_tsc;
517 unsigned int aperf_mperf_multiplier = 1;
518 double bclk;
519 double base_hz;
520 unsigned int has_base_hz;
521 double tsc_tweak = 1.0;
522 unsigned int show_pkg_only;
523 unsigned int show_core_only;
524 char *output_buffer, *outp;
525 unsigned int do_dts;
526 unsigned int do_ptm;
527 unsigned int do_ipc;
528 unsigned long long cpuidle_cur_cpu_lpi_us;
529 unsigned long long cpuidle_cur_sys_lpi_us;
530 unsigned int tj_max;
531 unsigned int tj_max_override;
532 double rapl_power_units, rapl_time_units;
533 double rapl_dram_energy_units, rapl_energy_units, rapl_psys_energy_units;
534 double rapl_joule_counter_range;
535 unsigned int crystal_hz;
536 unsigned long long tsc_hz;
537 int master_cpu;
538 unsigned int has_hwp; /* IA32_PM_ENABLE, IA32_HWP_CAPABILITIES */
539 /* IA32_HWP_REQUEST, IA32_HWP_STATUS */
540 unsigned int has_hwp_notify; /* IA32_HWP_INTERRUPT */
541 unsigned int has_hwp_activity_window; /* IA32_HWP_REQUEST[bits 41:32] */
542 unsigned int has_hwp_epp; /* IA32_HWP_REQUEST[bits 31:24] */
543 unsigned int has_hwp_pkg; /* IA32_HWP_REQUEST_PKG */
544 unsigned int first_counter_read = 1;
545
546 static struct timeval procsysfs_tv_begin;
547
548 int ignore_stdin;
549 bool no_msr;
550 bool no_perf;
551
552 enum gfx_sysfs_idx {
553 GFX_rc6,
554 GFX_MHz,
555 GFX_ACTMHz,
556 SAM_mc6,
557 SAM_MHz,
558 SAM_ACTMHz,
559 GFX_MAX
560 };
561
562 struct gfx_sysfs_info {
563 FILE *fp;
564 unsigned int val;
565 unsigned long long val_ull;
566 };
567
568 static struct gfx_sysfs_info gfx_info[GFX_MAX];
569
570 int get_msr(int cpu, off_t offset, unsigned long long *msr);
571 int add_counter(unsigned int msr_num, char *path, char *name,
572 unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int package_num);
573
574 /* Model specific support Start */
575
576 /* List of features that may diverge among different platforms */
577 struct platform_features {
578 bool has_msr_misc_feature_control; /* MSR_MISC_FEATURE_CONTROL */
579 bool has_msr_misc_pwr_mgmt; /* MSR_MISC_PWR_MGMT */
580 bool has_nhm_msrs; /* MSR_PLATFORM_INFO, MSR_IA32_TEMPERATURE_TARGET, MSR_SMI_COUNT, MSR_PKG_CST_CONFIG_CONTROL, MSR_IA32_POWER_CTL, TRL MSRs */
581 bool has_config_tdp; /* MSR_CONFIG_TDP_NOMINAL/LEVEL_1/LEVEL_2/CONTROL, MSR_TURBO_ACTIVATION_RATIO */
582 int bclk_freq; /* CPU base clock */
583 int crystal_freq; /* Crystal clock to use when not available from CPUID.15 */
584 int supported_cstates; /* Core cstates and Package cstates supported */
585 int cst_limit; /* MSR_PKG_CST_CONFIG_CONTROL */
586 bool has_cst_auto_convension; /* AUTOMATIC_CSTATE_CONVERSION bit in MSR_PKG_CST_CONFIG_CONTROL */
587 bool has_irtl_msrs; /* MSR_PKGC3/PKGC6/PKGC7/PKGC8/PKGC9/PKGC10_IRTL */
588 bool has_msr_core_c1_res; /* MSR_CORE_C1_RES */
589 bool has_msr_module_c6_res_ms; /* MSR_MODULE_C6_RES_MS */
590 bool has_msr_c6_demotion_policy_config; /* MSR_CC6_DEMOTION_POLICY_CONFIG/MSR_MC6_DEMOTION_POLICY_CONFIG */
591 bool has_msr_atom_pkg_c6_residency; /* MSR_ATOM_PKG_C6_RESIDENCY */
592 bool has_msr_knl_core_c6_residency; /* MSR_KNL_CORE_C6_RESIDENCY */
593 bool has_ext_cst_msrs; /* MSR_PKG_WEIGHTED_CORE_C0_RES/MSR_PKG_ANY_CORE_C0_RES/MSR_PKG_ANY_GFXE_C0_RES/MSR_PKG_BOTH_CORE_GFXE_C0_RES */
594 bool has_cst_prewake_bit; /* Cstate prewake bit in MSR_IA32_POWER_CTL */
595 int trl_msrs; /* MSR_TURBO_RATIO_LIMIT/LIMIT1/LIMIT2/SECONDARY, Atom TRL MSRs */
596 int plr_msrs; /* MSR_CORE/GFX/RING_PERF_LIMIT_REASONS */
597 int plat_rapl_msrs; /* RAPL PKG/DRAM/CORE/GFX MSRs, AMD RAPL MSRs */
598 bool has_per_core_rapl; /* Indicates cores energy collection is per-core, not per-package. AMD specific for now */
599 bool has_rapl_divisor; /* Divisor for Energy unit raw value from MSR_RAPL_POWER_UNIT */
600 bool has_fixed_rapl_unit; /* Fixed Energy Unit used for DRAM RAPL Domain */
601 bool has_fixed_rapl_psys_unit; /* Fixed Energy Unit used for PSYS RAPL Domain */
602 int rapl_quirk_tdp; /* Hardcoded TDP value when cannot be retrieved from hardware */
603 int tcc_offset_bits; /* TCC Offset bits in MSR_IA32_TEMPERATURE_TARGET */
604 bool enable_tsc_tweak; /* Use CPU Base freq instead of TSC freq for aperf/mperf counter */
605 bool need_perf_multiplier; /* mperf/aperf multiplier */
606 };
607
608 struct platform_data {
609 unsigned int vfm;
610 const struct platform_features *features;
611 };
612
613 /* For BCLK */
614 enum bclk_freq {
615 BCLK_100MHZ = 1,
616 BCLK_133MHZ,
617 BCLK_SLV,
618 };
619
620 #define SLM_BCLK_FREQS 5
621 double slm_freq_table[SLM_BCLK_FREQS] = { 83.3, 100.0, 133.3, 116.7, 80.0 };
622
slm_bclk(void)623 double slm_bclk(void)
624 {
625 unsigned long long msr = 3;
626 unsigned int i;
627 double freq;
628
629 if (get_msr(master_cpu, MSR_FSB_FREQ, &msr))
630 fprintf(outf, "SLM BCLK: unknown\n");
631
632 i = msr & 0xf;
633 if (i >= SLM_BCLK_FREQS) {
634 fprintf(outf, "SLM BCLK[%d] invalid\n", i);
635 i = 3;
636 }
637 freq = slm_freq_table[i];
638
639 if (!quiet)
640 fprintf(outf, "SLM BCLK: %.1f Mhz\n", freq);
641
642 return freq;
643 }
644
645 /* For Package cstate limit */
646 enum package_cstate_limit {
647 CST_LIMIT_NHM = 1,
648 CST_LIMIT_SNB,
649 CST_LIMIT_HSW,
650 CST_LIMIT_SKX,
651 CST_LIMIT_ICX,
652 CST_LIMIT_SLV,
653 CST_LIMIT_AMT,
654 CST_LIMIT_KNL,
655 CST_LIMIT_GMT,
656 };
657
658 /* For Turbo Ratio Limit MSRs */
659 enum turbo_ratio_limit_msrs {
660 TRL_BASE = BIT(0),
661 TRL_LIMIT1 = BIT(1),
662 TRL_LIMIT2 = BIT(2),
663 TRL_ATOM = BIT(3),
664 TRL_KNL = BIT(4),
665 TRL_CORECOUNT = BIT(5),
666 };
667
668 /* For Perf Limit Reason MSRs */
669 enum perf_limit_reason_msrs {
670 PLR_CORE = BIT(0),
671 PLR_GFX = BIT(1),
672 PLR_RING = BIT(2),
673 };
674
675 /* For RAPL MSRs */
676 enum rapl_msrs {
677 RAPL_PKG_POWER_LIMIT = BIT(0), /* 0x610 MSR_PKG_POWER_LIMIT */
678 RAPL_PKG_ENERGY_STATUS = BIT(1), /* 0x611 MSR_PKG_ENERGY_STATUS */
679 RAPL_PKG_PERF_STATUS = BIT(2), /* 0x613 MSR_PKG_PERF_STATUS */
680 RAPL_PKG_POWER_INFO = BIT(3), /* 0x614 MSR_PKG_POWER_INFO */
681 RAPL_DRAM_POWER_LIMIT = BIT(4), /* 0x618 MSR_DRAM_POWER_LIMIT */
682 RAPL_DRAM_ENERGY_STATUS = BIT(5), /* 0x619 MSR_DRAM_ENERGY_STATUS */
683 RAPL_DRAM_PERF_STATUS = BIT(6), /* 0x61b MSR_DRAM_PERF_STATUS */
684 RAPL_DRAM_POWER_INFO = BIT(7), /* 0x61c MSR_DRAM_POWER_INFO */
685 RAPL_CORE_POWER_LIMIT = BIT(8), /* 0x638 MSR_PP0_POWER_LIMIT */
686 RAPL_CORE_ENERGY_STATUS = BIT(9), /* 0x639 MSR_PP0_ENERGY_STATUS */
687 RAPL_CORE_POLICY = BIT(10), /* 0x63a MSR_PP0_POLICY */
688 RAPL_GFX_POWER_LIMIT = BIT(11), /* 0x640 MSR_PP1_POWER_LIMIT */
689 RAPL_GFX_ENERGY_STATUS = BIT(12), /* 0x641 MSR_PP1_ENERGY_STATUS */
690 RAPL_GFX_POLICY = BIT(13), /* 0x642 MSR_PP1_POLICY */
691 RAPL_AMD_PWR_UNIT = BIT(14), /* 0xc0010299 MSR_AMD_RAPL_POWER_UNIT */
692 RAPL_AMD_CORE_ENERGY_STAT = BIT(15), /* 0xc001029a MSR_AMD_CORE_ENERGY_STATUS */
693 RAPL_AMD_PKG_ENERGY_STAT = BIT(16), /* 0xc001029b MSR_AMD_PKG_ENERGY_STATUS */
694 RAPL_PLATFORM_ENERGY_LIMIT = BIT(17), /* 0x64c MSR_PLATFORM_ENERGY_LIMIT */
695 RAPL_PLATFORM_ENERGY_STATUS = BIT(18), /* 0x64d MSR_PLATFORM_ENERGY_STATUS */
696 };
697
698 #define RAPL_PKG (RAPL_PKG_ENERGY_STATUS | RAPL_PKG_POWER_LIMIT)
699 #define RAPL_DRAM (RAPL_DRAM_ENERGY_STATUS | RAPL_DRAM_POWER_LIMIT)
700 #define RAPL_CORE (RAPL_CORE_ENERGY_STATUS | RAPL_CORE_POWER_LIMIT)
701 #define RAPL_GFX (RAPL_GFX_POWER_LIMIT | RAPL_GFX_ENERGY_STATUS)
702 #define RAPL_PSYS (RAPL_PLATFORM_ENERGY_STATUS | RAPL_PLATFORM_ENERGY_LIMIT)
703
704 #define RAPL_PKG_ALL (RAPL_PKG | RAPL_PKG_PERF_STATUS | RAPL_PKG_POWER_INFO)
705 #define RAPL_DRAM_ALL (RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_DRAM_POWER_INFO)
706 #define RAPL_CORE_ALL (RAPL_CORE | RAPL_CORE_POLICY)
707 #define RAPL_GFX_ALL (RAPL_GFX | RAPL_GFX_POLICY)
708
709 #define RAPL_AMD_F17H (RAPL_AMD_PWR_UNIT | RAPL_AMD_CORE_ENERGY_STAT | RAPL_AMD_PKG_ENERGY_STAT)
710
711 /* For Cstates */
712 enum cstates {
713 CC1 = BIT(0),
714 CC3 = BIT(1),
715 CC6 = BIT(2),
716 CC7 = BIT(3),
717 PC2 = BIT(4),
718 PC3 = BIT(5),
719 PC6 = BIT(6),
720 PC7 = BIT(7),
721 PC8 = BIT(8),
722 PC9 = BIT(9),
723 PC10 = BIT(10),
724 };
725
726 static const struct platform_features nhm_features = {
727 .has_msr_misc_pwr_mgmt = 1,
728 .has_nhm_msrs = 1,
729 .bclk_freq = BCLK_133MHZ,
730 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
731 .cst_limit = CST_LIMIT_NHM,
732 .trl_msrs = TRL_BASE,
733 };
734
735 static const struct platform_features nhx_features = {
736 .has_msr_misc_pwr_mgmt = 1,
737 .has_nhm_msrs = 1,
738 .bclk_freq = BCLK_133MHZ,
739 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
740 .cst_limit = CST_LIMIT_NHM,
741 };
742
743 static const struct platform_features snb_features = {
744 .has_msr_misc_feature_control = 1,
745 .has_msr_misc_pwr_mgmt = 1,
746 .has_nhm_msrs = 1,
747 .bclk_freq = BCLK_100MHZ,
748 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
749 .cst_limit = CST_LIMIT_SNB,
750 .has_irtl_msrs = 1,
751 .trl_msrs = TRL_BASE,
752 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
753 };
754
755 static const struct platform_features snx_features = {
756 .has_msr_misc_feature_control = 1,
757 .has_msr_misc_pwr_mgmt = 1,
758 .has_nhm_msrs = 1,
759 .bclk_freq = BCLK_100MHZ,
760 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
761 .cst_limit = CST_LIMIT_SNB,
762 .has_irtl_msrs = 1,
763 .trl_msrs = TRL_BASE,
764 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
765 };
766
767 static const struct platform_features ivb_features = {
768 .has_msr_misc_feature_control = 1,
769 .has_msr_misc_pwr_mgmt = 1,
770 .has_nhm_msrs = 1,
771 .has_config_tdp = 1,
772 .bclk_freq = BCLK_100MHZ,
773 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
774 .cst_limit = CST_LIMIT_SNB,
775 .has_irtl_msrs = 1,
776 .trl_msrs = TRL_BASE,
777 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
778 };
779
780 static const struct platform_features ivx_features = {
781 .has_msr_misc_feature_control = 1,
782 .has_msr_misc_pwr_mgmt = 1,
783 .has_nhm_msrs = 1,
784 .bclk_freq = BCLK_100MHZ,
785 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
786 .cst_limit = CST_LIMIT_SNB,
787 .has_irtl_msrs = 1,
788 .trl_msrs = TRL_BASE | TRL_LIMIT1,
789 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM_ALL,
790 };
791
792 static const struct platform_features hsw_features = {
793 .has_msr_misc_feature_control = 1,
794 .has_msr_misc_pwr_mgmt = 1,
795 .has_nhm_msrs = 1,
796 .has_config_tdp = 1,
797 .bclk_freq = BCLK_100MHZ,
798 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
799 .cst_limit = CST_LIMIT_HSW,
800 .has_irtl_msrs = 1,
801 .trl_msrs = TRL_BASE,
802 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
803 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
804 };
805
806 static const struct platform_features hsx_features = {
807 .has_msr_misc_feature_control = 1,
808 .has_msr_misc_pwr_mgmt = 1,
809 .has_nhm_msrs = 1,
810 .has_config_tdp = 1,
811 .bclk_freq = BCLK_100MHZ,
812 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
813 .cst_limit = CST_LIMIT_HSW,
814 .has_irtl_msrs = 1,
815 .trl_msrs = TRL_BASE | TRL_LIMIT1 | TRL_LIMIT2,
816 .plr_msrs = PLR_CORE | PLR_RING,
817 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
818 .has_fixed_rapl_unit = 1,
819 };
820
821 static const struct platform_features hswl_features = {
822 .has_msr_misc_feature_control = 1,
823 .has_msr_misc_pwr_mgmt = 1,
824 .has_nhm_msrs = 1,
825 .has_config_tdp = 1,
826 .bclk_freq = BCLK_100MHZ,
827 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
828 .cst_limit = CST_LIMIT_HSW,
829 .has_irtl_msrs = 1,
830 .trl_msrs = TRL_BASE,
831 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
832 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
833 };
834
835 static const struct platform_features hswg_features = {
836 .has_msr_misc_feature_control = 1,
837 .has_msr_misc_pwr_mgmt = 1,
838 .has_nhm_msrs = 1,
839 .has_config_tdp = 1,
840 .bclk_freq = BCLK_100MHZ,
841 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
842 .cst_limit = CST_LIMIT_HSW,
843 .has_irtl_msrs = 1,
844 .trl_msrs = TRL_BASE,
845 .plr_msrs = PLR_CORE | PLR_GFX | PLR_RING,
846 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
847 };
848
849 static const struct platform_features bdw_features = {
850 .has_msr_misc_feature_control = 1,
851 .has_msr_misc_pwr_mgmt = 1,
852 .has_nhm_msrs = 1,
853 .has_config_tdp = 1,
854 .bclk_freq = BCLK_100MHZ,
855 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
856 .cst_limit = CST_LIMIT_HSW,
857 .has_irtl_msrs = 1,
858 .trl_msrs = TRL_BASE,
859 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
860 };
861
862 static const struct platform_features bdwg_features = {
863 .has_msr_misc_feature_control = 1,
864 .has_msr_misc_pwr_mgmt = 1,
865 .has_nhm_msrs = 1,
866 .has_config_tdp = 1,
867 .bclk_freq = BCLK_100MHZ,
868 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7,
869 .cst_limit = CST_LIMIT_HSW,
870 .has_irtl_msrs = 1,
871 .trl_msrs = TRL_BASE,
872 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE_ALL | RAPL_GFX | RAPL_PKG_POWER_INFO,
873 };
874
875 static const struct platform_features bdx_features = {
876 .has_msr_misc_feature_control = 1,
877 .has_msr_misc_pwr_mgmt = 1,
878 .has_nhm_msrs = 1,
879 .has_config_tdp = 1,
880 .bclk_freq = BCLK_100MHZ,
881 .supported_cstates = CC1 | CC3 | CC6 | PC2 | PC3 | PC6,
882 .cst_limit = CST_LIMIT_HSW,
883 .has_irtl_msrs = 1,
884 .has_cst_auto_convension = 1,
885 .trl_msrs = TRL_BASE,
886 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
887 .has_fixed_rapl_unit = 1,
888 };
889
890 static const struct platform_features skl_features = {
891 .has_msr_misc_feature_control = 1,
892 .has_msr_misc_pwr_mgmt = 1,
893 .has_nhm_msrs = 1,
894 .has_config_tdp = 1,
895 .bclk_freq = BCLK_100MHZ,
896 .crystal_freq = 24000000,
897 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
898 .cst_limit = CST_LIMIT_HSW,
899 .has_irtl_msrs = 1,
900 .has_ext_cst_msrs = 1,
901 .trl_msrs = TRL_BASE,
902 .tcc_offset_bits = 6,
903 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS,
904 .enable_tsc_tweak = 1,
905 };
906
907 static const struct platform_features cnl_features = {
908 .has_msr_misc_feature_control = 1,
909 .has_msr_misc_pwr_mgmt = 1,
910 .has_nhm_msrs = 1,
911 .has_config_tdp = 1,
912 .bclk_freq = BCLK_100MHZ,
913 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
914 .cst_limit = CST_LIMIT_HSW,
915 .has_irtl_msrs = 1,
916 .has_msr_core_c1_res = 1,
917 .has_ext_cst_msrs = 1,
918 .trl_msrs = TRL_BASE,
919 .tcc_offset_bits = 6,
920 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX | RAPL_PSYS,
921 .enable_tsc_tweak = 1,
922 };
923
924 /* Copied from cnl_features, with PC7/PC9 removed */
925 static const struct platform_features adl_features = {
926 .has_msr_misc_feature_control = cnl_features.has_msr_misc_feature_control,
927 .has_msr_misc_pwr_mgmt = cnl_features.has_msr_misc_pwr_mgmt,
928 .has_nhm_msrs = cnl_features.has_nhm_msrs,
929 .has_config_tdp = cnl_features.has_config_tdp,
930 .bclk_freq = cnl_features.bclk_freq,
931 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC8 | PC10,
932 .cst_limit = cnl_features.cst_limit,
933 .has_irtl_msrs = cnl_features.has_irtl_msrs,
934 .has_msr_core_c1_res = cnl_features.has_msr_core_c1_res,
935 .has_ext_cst_msrs = cnl_features.has_ext_cst_msrs,
936 .trl_msrs = cnl_features.trl_msrs,
937 .tcc_offset_bits = cnl_features.tcc_offset_bits,
938 .plat_rapl_msrs = cnl_features.plat_rapl_msrs,
939 .enable_tsc_tweak = cnl_features.enable_tsc_tweak,
940 };
941
942 /* Copied from adl_features, with PC3/PC8 removed */
943 static const struct platform_features lnl_features = {
944 .has_msr_misc_feature_control = adl_features.has_msr_misc_feature_control,
945 .has_msr_misc_pwr_mgmt = adl_features.has_msr_misc_pwr_mgmt,
946 .has_nhm_msrs = adl_features.has_nhm_msrs,
947 .has_config_tdp = adl_features.has_config_tdp,
948 .bclk_freq = adl_features.bclk_freq,
949 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC6 | PC10,
950 .cst_limit = adl_features.cst_limit,
951 .has_irtl_msrs = adl_features.has_irtl_msrs,
952 .has_msr_core_c1_res = adl_features.has_msr_core_c1_res,
953 .has_ext_cst_msrs = adl_features.has_ext_cst_msrs,
954 .trl_msrs = adl_features.trl_msrs,
955 .tcc_offset_bits = adl_features.tcc_offset_bits,
956 .plat_rapl_msrs = adl_features.plat_rapl_msrs,
957 .enable_tsc_tweak = adl_features.enable_tsc_tweak,
958 };
959
960 static const struct platform_features skx_features = {
961 .has_msr_misc_feature_control = 1,
962 .has_msr_misc_pwr_mgmt = 1,
963 .has_nhm_msrs = 1,
964 .has_config_tdp = 1,
965 .bclk_freq = BCLK_100MHZ,
966 .supported_cstates = CC1 | CC6 | PC2 | PC6,
967 .cst_limit = CST_LIMIT_SKX,
968 .has_irtl_msrs = 1,
969 .has_cst_auto_convension = 1,
970 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
971 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
972 .has_fixed_rapl_unit = 1,
973 };
974
975 static const struct platform_features icx_features = {
976 .has_msr_misc_feature_control = 1,
977 .has_msr_misc_pwr_mgmt = 1,
978 .has_nhm_msrs = 1,
979 .has_config_tdp = 1,
980 .bclk_freq = BCLK_100MHZ,
981 .supported_cstates = CC1 | CC6 | PC2 | PC6,
982 .cst_limit = CST_LIMIT_ICX,
983 .has_msr_core_c1_res = 1,
984 .has_irtl_msrs = 1,
985 .has_cst_prewake_bit = 1,
986 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
987 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
988 .has_fixed_rapl_unit = 1,
989 };
990
991 static const struct platform_features spr_features = {
992 .has_msr_misc_feature_control = 1,
993 .has_msr_misc_pwr_mgmt = 1,
994 .has_nhm_msrs = 1,
995 .has_config_tdp = 1,
996 .bclk_freq = BCLK_100MHZ,
997 .supported_cstates = CC1 | CC6 | PC2 | PC6,
998 .cst_limit = CST_LIMIT_SKX,
999 .has_msr_core_c1_res = 1,
1000 .has_irtl_msrs = 1,
1001 .has_cst_prewake_bit = 1,
1002 .has_fixed_rapl_psys_unit = 1,
1003 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1004 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1005 };
1006
1007 static const struct platform_features dmr_features = {
1008 .has_msr_misc_feature_control = spr_features.has_msr_misc_feature_control,
1009 .has_msr_misc_pwr_mgmt = spr_features.has_msr_misc_pwr_mgmt,
1010 .has_nhm_msrs = spr_features.has_nhm_msrs,
1011 .bclk_freq = spr_features.bclk_freq,
1012 .supported_cstates = spr_features.supported_cstates,
1013 .cst_limit = spr_features.cst_limit,
1014 .has_msr_core_c1_res = spr_features.has_msr_core_c1_res,
1015 .has_cst_prewake_bit = spr_features.has_cst_prewake_bit,
1016 .has_fixed_rapl_psys_unit = spr_features.has_fixed_rapl_psys_unit,
1017 .trl_msrs = spr_features.trl_msrs,
1018 .has_msr_module_c6_res_ms = 1, /* DMR has Dual-Core-Module and MC6 MSR */
1019 .plat_rapl_msrs = 0, /* DMR does not have RAPL MSRs */
1020 .plr_msrs = 0, /* DMR does not have PLR MSRs */
1021 .has_irtl_msrs = 0, /* DMR does not have IRTL MSRs */
1022 .has_config_tdp = 0, /* DMR does not have CTDP MSRs */
1023 };
1024
1025 static const struct platform_features srf_features = {
1026 .has_msr_misc_feature_control = 1,
1027 .has_msr_misc_pwr_mgmt = 1,
1028 .has_nhm_msrs = 1,
1029 .has_config_tdp = 1,
1030 .bclk_freq = BCLK_100MHZ,
1031 .supported_cstates = CC1 | CC6 | PC2 | PC6,
1032 .cst_limit = CST_LIMIT_SKX,
1033 .has_msr_core_c1_res = 1,
1034 .has_msr_module_c6_res_ms = 1,
1035 .has_irtl_msrs = 1,
1036 .has_cst_prewake_bit = 1,
1037 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1038 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1039 };
1040
1041 static const struct platform_features grr_features = {
1042 .has_msr_misc_feature_control = 1,
1043 .has_msr_misc_pwr_mgmt = 1,
1044 .has_nhm_msrs = 1,
1045 .has_config_tdp = 1,
1046 .bclk_freq = BCLK_100MHZ,
1047 .supported_cstates = CC1 | CC6,
1048 .cst_limit = CST_LIMIT_SKX,
1049 .has_msr_core_c1_res = 1,
1050 .has_msr_module_c6_res_ms = 1,
1051 .has_irtl_msrs = 1,
1052 .has_cst_prewake_bit = 1,
1053 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1054 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_PSYS,
1055 };
1056
1057 static const struct platform_features slv_features = {
1058 .has_nhm_msrs = 1,
1059 .bclk_freq = BCLK_SLV,
1060 .supported_cstates = CC1 | CC6 | PC6,
1061 .cst_limit = CST_LIMIT_SLV,
1062 .has_msr_core_c1_res = 1,
1063 .has_msr_module_c6_res_ms = 1,
1064 .has_msr_c6_demotion_policy_config = 1,
1065 .has_msr_atom_pkg_c6_residency = 1,
1066 .trl_msrs = TRL_ATOM,
1067 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE,
1068 .has_rapl_divisor = 1,
1069 .rapl_quirk_tdp = 30,
1070 };
1071
1072 static const struct platform_features slvd_features = {
1073 .has_msr_misc_pwr_mgmt = 1,
1074 .has_nhm_msrs = 1,
1075 .bclk_freq = BCLK_SLV,
1076 .supported_cstates = CC1 | CC6 | PC3 | PC6,
1077 .cst_limit = CST_LIMIT_SLV,
1078 .has_msr_atom_pkg_c6_residency = 1,
1079 .trl_msrs = TRL_BASE,
1080 .plat_rapl_msrs = RAPL_PKG | RAPL_CORE,
1081 .rapl_quirk_tdp = 30,
1082 };
1083
1084 static const struct platform_features amt_features = {
1085 .has_nhm_msrs = 1,
1086 .bclk_freq = BCLK_133MHZ,
1087 .supported_cstates = CC1 | CC3 | CC6 | PC3 | PC6,
1088 .cst_limit = CST_LIMIT_AMT,
1089 .trl_msrs = TRL_BASE,
1090 };
1091
1092 static const struct platform_features gmt_features = {
1093 .has_msr_misc_pwr_mgmt = 1,
1094 .has_nhm_msrs = 1,
1095 .bclk_freq = BCLK_100MHZ,
1096 .crystal_freq = 19200000,
1097 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1098 .cst_limit = CST_LIMIT_GMT,
1099 .has_irtl_msrs = 1,
1100 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1101 .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
1102 };
1103
1104 static const struct platform_features gmtd_features = {
1105 .has_msr_misc_pwr_mgmt = 1,
1106 .has_nhm_msrs = 1,
1107 .bclk_freq = BCLK_100MHZ,
1108 .crystal_freq = 25000000,
1109 .supported_cstates = CC1 | CC6 | PC2 | PC6,
1110 .cst_limit = CST_LIMIT_GMT,
1111 .has_irtl_msrs = 1,
1112 .has_msr_core_c1_res = 1,
1113 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1114 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL | RAPL_CORE_ENERGY_STATUS,
1115 };
1116
1117 static const struct platform_features gmtp_features = {
1118 .has_msr_misc_pwr_mgmt = 1,
1119 .has_nhm_msrs = 1,
1120 .bclk_freq = BCLK_100MHZ,
1121 .crystal_freq = 19200000,
1122 .supported_cstates = CC1 | CC3 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1123 .cst_limit = CST_LIMIT_GMT,
1124 .has_irtl_msrs = 1,
1125 .trl_msrs = TRL_BASE,
1126 .plat_rapl_msrs = RAPL_PKG | RAPL_PKG_POWER_INFO,
1127 };
1128
1129 static const struct platform_features tmt_features = {
1130 .has_msr_misc_pwr_mgmt = 1,
1131 .has_nhm_msrs = 1,
1132 .bclk_freq = BCLK_100MHZ,
1133 .supported_cstates = CC1 | CC6 | CC7 | PC2 | PC3 | PC6 | PC7 | PC8 | PC9 | PC10,
1134 .cst_limit = CST_LIMIT_GMT,
1135 .has_irtl_msrs = 1,
1136 .trl_msrs = TRL_BASE,
1137 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_CORE_ALL | RAPL_DRAM | RAPL_DRAM_PERF_STATUS | RAPL_GFX,
1138 .enable_tsc_tweak = 1,
1139 };
1140
1141 static const struct platform_features tmtd_features = {
1142 .has_msr_misc_pwr_mgmt = 1,
1143 .has_nhm_msrs = 1,
1144 .bclk_freq = BCLK_100MHZ,
1145 .supported_cstates = CC1 | CC6,
1146 .cst_limit = CST_LIMIT_GMT,
1147 .has_irtl_msrs = 1,
1148 .trl_msrs = TRL_BASE | TRL_CORECOUNT,
1149 .plat_rapl_msrs = RAPL_PKG_ALL,
1150 };
1151
1152 static const struct platform_features knl_features = {
1153 .has_msr_misc_pwr_mgmt = 1,
1154 .has_nhm_msrs = 1,
1155 .has_config_tdp = 1,
1156 .bclk_freq = BCLK_100MHZ,
1157 .supported_cstates = CC1 | CC6 | PC3 | PC6,
1158 .cst_limit = CST_LIMIT_KNL,
1159 .has_msr_knl_core_c6_residency = 1,
1160 .trl_msrs = TRL_KNL,
1161 .plat_rapl_msrs = RAPL_PKG_ALL | RAPL_DRAM_ALL,
1162 .has_fixed_rapl_unit = 1,
1163 .need_perf_multiplier = 1,
1164 };
1165
1166 static const struct platform_features default_features = {
1167 };
1168
1169 static const struct platform_features amd_features_with_rapl = {
1170 .plat_rapl_msrs = RAPL_AMD_F17H,
1171 .has_per_core_rapl = 1,
1172 .rapl_quirk_tdp = 280, /* This is the max stock TDP of HEDT/Server Fam17h+ chips */
1173 };
1174
1175 static const struct platform_data turbostat_pdata[] = {
1176 { INTEL_NEHALEM, &nhm_features },
1177 { INTEL_NEHALEM_G, &nhm_features },
1178 { INTEL_NEHALEM_EP, &nhm_features },
1179 { INTEL_NEHALEM_EX, &nhx_features },
1180 { INTEL_WESTMERE, &nhm_features },
1181 { INTEL_WESTMERE_EP, &nhm_features },
1182 { INTEL_WESTMERE_EX, &nhx_features },
1183 { INTEL_SANDYBRIDGE, &snb_features },
1184 { INTEL_SANDYBRIDGE_X, &snx_features },
1185 { INTEL_IVYBRIDGE, &ivb_features },
1186 { INTEL_IVYBRIDGE_X, &ivx_features },
1187 { INTEL_HASWELL, &hsw_features },
1188 { INTEL_HASWELL_X, &hsx_features },
1189 { INTEL_HASWELL_L, &hswl_features },
1190 { INTEL_HASWELL_G, &hswg_features },
1191 { INTEL_BROADWELL, &bdw_features },
1192 { INTEL_BROADWELL_G, &bdwg_features },
1193 { INTEL_BROADWELL_X, &bdx_features },
1194 { INTEL_BROADWELL_D, &bdx_features },
1195 { INTEL_SKYLAKE_L, &skl_features },
1196 { INTEL_SKYLAKE, &skl_features },
1197 { INTEL_SKYLAKE_X, &skx_features },
1198 { INTEL_KABYLAKE_L, &skl_features },
1199 { INTEL_KABYLAKE, &skl_features },
1200 { INTEL_COMETLAKE, &skl_features },
1201 { INTEL_COMETLAKE_L, &skl_features },
1202 { INTEL_CANNONLAKE_L, &cnl_features },
1203 { INTEL_ICELAKE_X, &icx_features },
1204 { INTEL_ICELAKE_D, &icx_features },
1205 { INTEL_ICELAKE_L, &cnl_features },
1206 { INTEL_ICELAKE_NNPI, &cnl_features },
1207 { INTEL_ROCKETLAKE, &cnl_features },
1208 { INTEL_TIGERLAKE_L, &cnl_features },
1209 { INTEL_TIGERLAKE, &cnl_features },
1210 { INTEL_SAPPHIRERAPIDS_X, &spr_features },
1211 { INTEL_EMERALDRAPIDS_X, &spr_features },
1212 { INTEL_GRANITERAPIDS_X, &spr_features },
1213 { INTEL_GRANITERAPIDS_D, &spr_features },
1214 { INTEL_DIAMONDRAPIDS_X, &dmr_features },
1215 { INTEL_LAKEFIELD, &cnl_features },
1216 { INTEL_ALDERLAKE, &adl_features },
1217 { INTEL_ALDERLAKE_L, &adl_features },
1218 { INTEL_RAPTORLAKE, &adl_features },
1219 { INTEL_RAPTORLAKE_P, &adl_features },
1220 { INTEL_RAPTORLAKE_S, &adl_features },
1221 { INTEL_BARTLETTLAKE, &adl_features },
1222 { INTEL_METEORLAKE, &adl_features },
1223 { INTEL_METEORLAKE_L, &adl_features },
1224 { INTEL_ARROWLAKE_H, &adl_features },
1225 { INTEL_ARROWLAKE_U, &adl_features },
1226 { INTEL_ARROWLAKE, &adl_features },
1227 { INTEL_LUNARLAKE_M, &lnl_features },
1228 { INTEL_PANTHERLAKE_L, &lnl_features },
1229 { INTEL_NOVALAKE, &lnl_features },
1230 { INTEL_NOVALAKE_L, &lnl_features },
1231 { INTEL_WILDCATLAKE_L, &lnl_features },
1232 { INTEL_ATOM_SILVERMONT, &slv_features },
1233 { INTEL_ATOM_SILVERMONT_D, &slvd_features },
1234 { INTEL_ATOM_AIRMONT, &amt_features },
1235 { INTEL_ATOM_GOLDMONT, &gmt_features },
1236 { INTEL_ATOM_GOLDMONT_D, &gmtd_features },
1237 { INTEL_ATOM_GOLDMONT_PLUS, &gmtp_features },
1238 { INTEL_ATOM_TREMONT_D, &tmtd_features },
1239 { INTEL_ATOM_TREMONT, &tmt_features },
1240 { INTEL_ATOM_TREMONT_L, &tmt_features },
1241 { INTEL_ATOM_GRACEMONT, &adl_features },
1242 { INTEL_ATOM_CRESTMONT_X, &srf_features },
1243 { INTEL_ATOM_CRESTMONT, &grr_features },
1244 { INTEL_ATOM_DARKMONT_X, &srf_features },
1245 { INTEL_XEON_PHI_KNL, &knl_features },
1246 { INTEL_XEON_PHI_KNM, &knl_features },
1247 /*
1248 * Missing support for
1249 * INTEL_ICELAKE
1250 * INTEL_ATOM_SILVERMONT_MID
1251 * INTEL_ATOM_SILVERMONT_MID2
1252 * INTEL_ATOM_AIRMONT_NP
1253 */
1254 { 0, NULL },
1255 };
1256
1257 struct {
1258 unsigned int uniform;
1259 unsigned int pcore;
1260 unsigned int ecore;
1261 unsigned int lcore;
1262 } perf_pmu_types;
1263
1264 /*
1265 * Events are enumerated in https://github.com/intel/perfmon
1266 * and tools/perf/pmu-events/arch/x86/.../cache.json
1267 */
1268 struct perf_l2_events {
1269 unsigned long long refs; /* L2_REQUEST.ALL */
1270 unsigned long long hits; /* L2_REQUEST.HIT */
1271 };
1272
1273 struct perf_model_support {
1274 unsigned int vfm;
1275 struct perf_l2_events first;
1276 struct perf_l2_events second;
1277 struct perf_l2_events third;
1278 } *perf_model_support;
1279
1280 /* Perf Cache Events */
1281 #define PCE(ext_umask, umask) (((unsigned long long) ext_umask) << 40 | umask << 8 | 0x24)
1282
1283 /*
1284 * Enumerate up to three perf CPU PMU's in a system.
1285 * The first, second, and third columns are populated without skipping, describing
1286 * pcore, ecore, lcore PMUs, in order, if present. (The associated PMU "type" field is
1287 * read from sysfs in all cases.) Eg.
1288 *
1289 * non-hybrid:
1290 * GNR: pcore, {}, {}
1291 * ADL-N: ecore, {}, {}
1292 * hybrid:
1293 * MTL: pcore, ecore, {}%
1294 * ARL-H: pcore, ecore, lcore
1295 * LNL: ecore, ecore%%, {}
1296 *
1297 * % MTL physical lcore share architecture and PMU with ecore, and are thus not enumerated separately.
1298 * %% LNL physical lcore is enumerated by perf as ecore
1299 */
1300 static struct perf_model_support turbostat_perf_model_support[] = {
1301 { INTEL_SAPPHIRERAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} },
1302 { INTEL_EMERALDRAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} },
1303 { INTEL_GRANITERAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} },
1304 { INTEL_GRANITERAPIDS_D, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, {}, {} },
1305 { INTEL_DIAMONDRAPIDS_X, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, {}, {} },
1306
1307 { INTEL_ATOM_GRACEMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* ADL-N */
1308 { INTEL_ATOM_CRESTMONT_X, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* SRF */
1309 { INTEL_ATOM_CRESTMONT, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {}, {} }, /* GRR */
1310 { INTEL_ATOM_DARKMONT_X, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {}, {} }, /* CWF */
1311
1312 { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1313 { INTEL_ALDERLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1314 { INTEL_ALDERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1315 { INTEL_RAPTORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1316 { INTEL_RAPTORLAKE_P, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1317 { INTEL_RAPTORLAKE_S, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1318 { INTEL_METEORLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1319 { INTEL_METEORLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1320 { INTEL_ARROWLAKE_U, { PCE(0x00, 0xFF), PCE(0x00, 0xDF)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)}, {} },
1321
1322 { INTEL_LUNARLAKE_M, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, {} },
1323 { INTEL_ARROWLAKE_H, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, { PCE(0x00, 0x00), PCE(0x00, 0x02)} },
1324 { INTEL_ARROWLAKE, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x00, 0x07), PCE(0x00, 0x02)}, {} },
1325
1326 { INTEL_PANTHERLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} },
1327 { INTEL_WILDCATLAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} },
1328
1329 { INTEL_NOVALAKE, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} },
1330 { INTEL_NOVALAKE_L, { PCE(0x00, 0xFF), PCE(0x00, 0x5F)}, { PCE(0x01, 0xFF), PCE(0x01, 0xBF)}, {} },
1331
1332 { 0, {}, {}, {} }
1333 };
1334
1335 static const struct platform_features *platform;
1336
probe_platform_features(unsigned int family,unsigned int model)1337 void probe_platform_features(unsigned int family, unsigned int model)
1338 {
1339 int i;
1340
1341 if (authentic_amd || hygon_genuine) {
1342 /* fallback to default features on unsupported models */
1343 force_load++;
1344 if (max_extended_level >= 0x80000007) {
1345 unsigned int eax, ebx, ecx, edx;
1346
1347 __cpuid(0x80000007, eax, ebx, ecx, edx);
1348 /* RAPL (Fam 17h+) */
1349 if ((edx & (1 << 14)) && family >= 0x17)
1350 platform = &amd_features_with_rapl;
1351 }
1352 goto end;
1353 }
1354
1355 if (!genuine_intel)
1356 goto end;
1357
1358 for (i = 0; turbostat_pdata[i].features; i++) {
1359 if (VFM_FAMILY(turbostat_pdata[i].vfm) == family && VFM_MODEL(turbostat_pdata[i].vfm) == model) {
1360 platform = turbostat_pdata[i].features;
1361 return;
1362 }
1363 }
1364
1365 end:
1366 if (force_load && !platform) {
1367 fprintf(outf, "Forced to run on unsupported platform!\n");
1368 platform = &default_features;
1369 }
1370
1371 if (platform)
1372 return;
1373
1374 fprintf(stderr, "Unsupported platform detected.\n\tSee RUN THE LATEST VERSION on turbostat(8)\n");
1375 exit(1);
1376 }
1377
init_perf_model_support(unsigned int family,unsigned int model)1378 void init_perf_model_support(unsigned int family, unsigned int model)
1379 {
1380 int i;
1381
1382 if (!genuine_intel)
1383 return;
1384
1385 for (i = 0; turbostat_perf_model_support[i].vfm; i++) {
1386 if (VFM_FAMILY(turbostat_perf_model_support[i].vfm) == family && VFM_MODEL(turbostat_perf_model_support[i].vfm) == model) {
1387 perf_model_support = &turbostat_perf_model_support[i];
1388 return;
1389 }
1390 }
1391 }
1392
1393 /* Model specific support End */
1394
1395 #define TJMAX_DEFAULT 100
1396
1397 /* MSRs that are not yet in the kernel-provided header. */
1398 #define MSR_RAPL_PWR_UNIT 0xc0010299
1399 #define MSR_CORE_ENERGY_STAT 0xc001029a
1400 #define MSR_PKG_ENERGY_STAT 0xc001029b
1401
1402 #define MAX(a, b) ((a) > (b) ? (a) : (b))
1403
1404 int backwards_count;
1405 char *progname;
1406
1407 #define CPU_SUBSET_MAXCPUS 8192 /* need to use before probe... */
1408 cpu_set_t *cpu_present_set, *cpu_possible_set, *cpu_effective_set, *cpu_allowed_set, *cpu_affinity_set, *cpu_subset;
1409 cpu_set_t *perf_pcore_set, *perf_ecore_set, *perf_lcore_set;
1410 size_t cpu_present_setsize, cpu_possible_setsize, cpu_effective_setsize, cpu_allowed_setsize, cpu_affinity_setsize, cpu_subset_size;
1411 #define MAX_ADDED_THREAD_COUNTERS 24
1412 #define MAX_ADDED_CORE_COUNTERS 8
1413 #define MAX_ADDED_PACKAGE_COUNTERS 16
1414 #define PMT_MAX_ADDED_THREAD_COUNTERS 24
1415 #define PMT_MAX_ADDED_CORE_COUNTERS 8
1416 #define PMT_MAX_ADDED_PACKAGE_COUNTERS 16
1417 #define BITMASK_SIZE 32
1418
1419 #define ZERO_ARRAY(arr) (memset(arr, 0, sizeof(arr)) + __must_be_array(arr))
1420
1421 /* Indexes used to map data read from perf and MSRs into global variables */
1422 enum rapl_rci_index {
1423 RAPL_RCI_INDEX_ENERGY_PKG = 0,
1424 RAPL_RCI_INDEX_ENERGY_CORES = 1,
1425 RAPL_RCI_INDEX_DRAM = 2,
1426 RAPL_RCI_INDEX_GFX = 3,
1427 RAPL_RCI_INDEX_PKG_PERF_STATUS = 4,
1428 RAPL_RCI_INDEX_DRAM_PERF_STATUS = 5,
1429 RAPL_RCI_INDEX_CORE_ENERGY = 6,
1430 RAPL_RCI_INDEX_ENERGY_PLATFORM = 7,
1431 NUM_RAPL_COUNTERS,
1432 };
1433
1434 enum rapl_unit {
1435 RAPL_UNIT_INVALID,
1436 RAPL_UNIT_JOULES,
1437 RAPL_UNIT_WATTS,
1438 };
1439
1440 struct rapl_counter_info_t {
1441 unsigned long long data[NUM_RAPL_COUNTERS];
1442 enum counter_source source[NUM_RAPL_COUNTERS];
1443 unsigned long long flags[NUM_RAPL_COUNTERS];
1444 double scale[NUM_RAPL_COUNTERS];
1445 enum rapl_unit unit[NUM_RAPL_COUNTERS];
1446 unsigned long long msr[NUM_RAPL_COUNTERS];
1447 unsigned long long msr_mask[NUM_RAPL_COUNTERS];
1448 int msr_shift[NUM_RAPL_COUNTERS];
1449
1450 int fd_perf;
1451 };
1452
1453 /* struct rapl_counter_info_t for each RAPL domain */
1454 struct rapl_counter_info_t *rapl_counter_info_perdomain;
1455 unsigned int rapl_counter_info_perdomain_size;
1456
1457 #define RAPL_COUNTER_FLAG_PLATFORM_COUNTER (1u << 0)
1458 #define RAPL_COUNTER_FLAG_USE_MSR_SUM (1u << 1)
1459
1460 struct rapl_counter_arch_info {
1461 int feature_mask; /* Mask for testing if the counter is supported on host */
1462 const char *perf_subsys;
1463 const char *perf_name;
1464 unsigned long long msr;
1465 unsigned long long msr_mask;
1466 int msr_shift; /* Positive mean shift right, negative mean shift left */
1467 double *platform_rapl_msr_scale; /* Scale applied to values read by MSR (platform dependent, filled at runtime) */
1468 unsigned int rci_index; /* Maps data from perf counters to global variables */
1469 unsigned int bic_number;
1470 double compat_scale; /* Some counters require constant scaling to be in the same range as other, similar ones */
1471 unsigned long long flags;
1472 };
1473
1474 static const struct rapl_counter_arch_info rapl_counter_arch_infos[] = {
1475 {
1476 .feature_mask = RAPL_PKG,
1477 .perf_subsys = "power",
1478 .perf_name = "energy-pkg",
1479 .msr = MSR_PKG_ENERGY_STATUS,
1480 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1481 .msr_shift = 0,
1482 .platform_rapl_msr_scale = &rapl_energy_units,
1483 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1484 .bic_number = BIC_PkgWatt,
1485 .compat_scale = 1.0,
1486 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1487 },
1488 {
1489 .feature_mask = RAPL_PKG,
1490 .perf_subsys = "power",
1491 .perf_name = "energy-pkg",
1492 .msr = MSR_PKG_ENERGY_STATUS,
1493 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1494 .msr_shift = 0,
1495 .platform_rapl_msr_scale = &rapl_energy_units,
1496 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1497 .bic_number = BIC_Pkg_J,
1498 .compat_scale = 1.0,
1499 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1500 },
1501 {
1502 .feature_mask = RAPL_AMD_F17H,
1503 .perf_subsys = "power",
1504 .perf_name = "energy-pkg",
1505 .msr = MSR_PKG_ENERGY_STAT,
1506 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1507 .msr_shift = 0,
1508 .platform_rapl_msr_scale = &rapl_energy_units,
1509 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1510 .bic_number = BIC_PkgWatt,
1511 .compat_scale = 1.0,
1512 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1513 },
1514 {
1515 .feature_mask = RAPL_AMD_F17H,
1516 .perf_subsys = "power",
1517 .perf_name = "energy-pkg",
1518 .msr = MSR_PKG_ENERGY_STAT,
1519 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1520 .msr_shift = 0,
1521 .platform_rapl_msr_scale = &rapl_energy_units,
1522 .rci_index = RAPL_RCI_INDEX_ENERGY_PKG,
1523 .bic_number = BIC_Pkg_J,
1524 .compat_scale = 1.0,
1525 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1526 },
1527 {
1528 .feature_mask = RAPL_CORE_ENERGY_STATUS,
1529 .perf_subsys = "power",
1530 .perf_name = "energy-cores",
1531 .msr = MSR_PP0_ENERGY_STATUS,
1532 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1533 .msr_shift = 0,
1534 .platform_rapl_msr_scale = &rapl_energy_units,
1535 .rci_index = RAPL_RCI_INDEX_ENERGY_CORES,
1536 .bic_number = BIC_CorWatt,
1537 .compat_scale = 1.0,
1538 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1539 },
1540 {
1541 .feature_mask = RAPL_CORE_ENERGY_STATUS,
1542 .perf_subsys = "power",
1543 .perf_name = "energy-cores",
1544 .msr = MSR_PP0_ENERGY_STATUS,
1545 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1546 .msr_shift = 0,
1547 .platform_rapl_msr_scale = &rapl_energy_units,
1548 .rci_index = RAPL_RCI_INDEX_ENERGY_CORES,
1549 .bic_number = BIC_Cor_J,
1550 .compat_scale = 1.0,
1551 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1552 },
1553 {
1554 .feature_mask = RAPL_DRAM,
1555 .perf_subsys = "power",
1556 .perf_name = "energy-ram",
1557 .msr = MSR_DRAM_ENERGY_STATUS,
1558 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1559 .msr_shift = 0,
1560 .platform_rapl_msr_scale = &rapl_dram_energy_units,
1561 .rci_index = RAPL_RCI_INDEX_DRAM,
1562 .bic_number = BIC_RAMWatt,
1563 .compat_scale = 1.0,
1564 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1565 },
1566 {
1567 .feature_mask = RAPL_DRAM,
1568 .perf_subsys = "power",
1569 .perf_name = "energy-ram",
1570 .msr = MSR_DRAM_ENERGY_STATUS,
1571 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1572 .msr_shift = 0,
1573 .platform_rapl_msr_scale = &rapl_dram_energy_units,
1574 .rci_index = RAPL_RCI_INDEX_DRAM,
1575 .bic_number = BIC_RAM_J,
1576 .compat_scale = 1.0,
1577 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1578 },
1579 {
1580 .feature_mask = RAPL_GFX,
1581 .perf_subsys = "power",
1582 .perf_name = "energy-gpu",
1583 .msr = MSR_PP1_ENERGY_STATUS,
1584 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1585 .msr_shift = 0,
1586 .platform_rapl_msr_scale = &rapl_energy_units,
1587 .rci_index = RAPL_RCI_INDEX_GFX,
1588 .bic_number = BIC_GFXWatt,
1589 .compat_scale = 1.0,
1590 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1591 },
1592 {
1593 .feature_mask = RAPL_GFX,
1594 .perf_subsys = "power",
1595 .perf_name = "energy-gpu",
1596 .msr = MSR_PP1_ENERGY_STATUS,
1597 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1598 .msr_shift = 0,
1599 .platform_rapl_msr_scale = &rapl_energy_units,
1600 .rci_index = RAPL_RCI_INDEX_GFX,
1601 .bic_number = BIC_GFX_J,
1602 .compat_scale = 1.0,
1603 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1604 },
1605 {
1606 .feature_mask = RAPL_PKG_PERF_STATUS,
1607 .perf_subsys = NULL,
1608 .perf_name = NULL,
1609 .msr = MSR_PKG_PERF_STATUS,
1610 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1611 .msr_shift = 0,
1612 .platform_rapl_msr_scale = &rapl_time_units,
1613 .rci_index = RAPL_RCI_INDEX_PKG_PERF_STATUS,
1614 .bic_number = BIC_PKG__,
1615 .compat_scale = 100.0,
1616 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1617 },
1618 {
1619 .feature_mask = RAPL_DRAM_PERF_STATUS,
1620 .perf_subsys = NULL,
1621 .perf_name = NULL,
1622 .msr = MSR_DRAM_PERF_STATUS,
1623 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1624 .msr_shift = 0,
1625 .platform_rapl_msr_scale = &rapl_time_units,
1626 .rci_index = RAPL_RCI_INDEX_DRAM_PERF_STATUS,
1627 .bic_number = BIC_RAM__,
1628 .compat_scale = 100.0,
1629 .flags = RAPL_COUNTER_FLAG_USE_MSR_SUM,
1630 },
1631 {
1632 .feature_mask = RAPL_AMD_F17H,
1633 .perf_subsys = NULL,
1634 .perf_name = NULL,
1635 .msr = MSR_CORE_ENERGY_STAT,
1636 .msr_mask = 0xFFFFFFFF,
1637 .msr_shift = 0,
1638 .platform_rapl_msr_scale = &rapl_energy_units,
1639 .rci_index = RAPL_RCI_INDEX_CORE_ENERGY,
1640 .bic_number = BIC_CorWatt,
1641 .compat_scale = 1.0,
1642 .flags = 0,
1643 },
1644 {
1645 .feature_mask = RAPL_AMD_F17H,
1646 .perf_subsys = NULL,
1647 .perf_name = NULL,
1648 .msr = MSR_CORE_ENERGY_STAT,
1649 .msr_mask = 0xFFFFFFFF,
1650 .msr_shift = 0,
1651 .platform_rapl_msr_scale = &rapl_energy_units,
1652 .rci_index = RAPL_RCI_INDEX_CORE_ENERGY,
1653 .bic_number = BIC_Cor_J,
1654 .compat_scale = 1.0,
1655 .flags = 0,
1656 },
1657 {
1658 .feature_mask = RAPL_PSYS,
1659 .perf_subsys = "power",
1660 .perf_name = "energy-psys",
1661 .msr = MSR_PLATFORM_ENERGY_STATUS,
1662 .msr_mask = 0x00000000FFFFFFFF,
1663 .msr_shift = 0,
1664 .platform_rapl_msr_scale = &rapl_psys_energy_units,
1665 .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM,
1666 .bic_number = BIC_SysWatt,
1667 .compat_scale = 1.0,
1668 .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM,
1669 },
1670 {
1671 .feature_mask = RAPL_PSYS,
1672 .perf_subsys = "power",
1673 .perf_name = "energy-psys",
1674 .msr = MSR_PLATFORM_ENERGY_STATUS,
1675 .msr_mask = 0x00000000FFFFFFFF,
1676 .msr_shift = 0,
1677 .platform_rapl_msr_scale = &rapl_psys_energy_units,
1678 .rci_index = RAPL_RCI_INDEX_ENERGY_PLATFORM,
1679 .bic_number = BIC_Sys_J,
1680 .compat_scale = 1.0,
1681 .flags = RAPL_COUNTER_FLAG_PLATFORM_COUNTER | RAPL_COUNTER_FLAG_USE_MSR_SUM,
1682 },
1683 };
1684
1685 struct rapl_counter {
1686 unsigned long long raw_value;
1687 enum rapl_unit unit;
1688 double scale;
1689 };
1690
1691 /* Indexes used to map data read from perf and MSRs into global variables */
1692 enum ccstate_rci_index {
1693 CCSTATE_RCI_INDEX_C1_RESIDENCY = 0,
1694 CCSTATE_RCI_INDEX_C3_RESIDENCY = 1,
1695 CCSTATE_RCI_INDEX_C6_RESIDENCY = 2,
1696 CCSTATE_RCI_INDEX_C7_RESIDENCY = 3,
1697 PCSTATE_RCI_INDEX_C2_RESIDENCY = 4,
1698 PCSTATE_RCI_INDEX_C3_RESIDENCY = 5,
1699 PCSTATE_RCI_INDEX_C6_RESIDENCY = 6,
1700 PCSTATE_RCI_INDEX_C7_RESIDENCY = 7,
1701 PCSTATE_RCI_INDEX_C8_RESIDENCY = 8,
1702 PCSTATE_RCI_INDEX_C9_RESIDENCY = 9,
1703 PCSTATE_RCI_INDEX_C10_RESIDENCY = 10,
1704 NUM_CSTATE_COUNTERS,
1705 };
1706
1707 struct cstate_counter_info_t {
1708 unsigned long long data[NUM_CSTATE_COUNTERS];
1709 enum counter_source source[NUM_CSTATE_COUNTERS];
1710 unsigned long long msr[NUM_CSTATE_COUNTERS];
1711 int fd_perf_core;
1712 int fd_perf_pkg;
1713 };
1714
1715 struct cstate_counter_info_t *ccstate_counter_info;
1716 unsigned int ccstate_counter_info_size;
1717
1718 #define CSTATE_COUNTER_FLAG_COLLECT_PER_CORE (1u << 0)
1719 #define CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD ((1u << 1) | CSTATE_COUNTER_FLAG_COLLECT_PER_CORE)
1720 #define CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY (1u << 2)
1721
1722 struct cstate_counter_arch_info {
1723 int feature_mask; /* Mask for testing if the counter is supported on host */
1724 const char *perf_subsys;
1725 const char *perf_name;
1726 unsigned long long msr;
1727 unsigned int rci_index; /* Maps data from perf counters to global variables */
1728 unsigned int bic_number;
1729 unsigned long long flags;
1730 int pkg_cstate_limit;
1731 };
1732
1733 static struct cstate_counter_arch_info ccstate_counter_arch_infos[] = {
1734 {
1735 .feature_mask = CC1,
1736 .perf_subsys = "cstate_core",
1737 .perf_name = "c1-residency",
1738 .msr = MSR_CORE_C1_RES,
1739 .rci_index = CCSTATE_RCI_INDEX_C1_RESIDENCY,
1740 .bic_number = BIC_CPU_c1,
1741 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD,
1742 .pkg_cstate_limit = 0,
1743 },
1744 {
1745 .feature_mask = CC3,
1746 .perf_subsys = "cstate_core",
1747 .perf_name = "c3-residency",
1748 .msr = MSR_CORE_C3_RESIDENCY,
1749 .rci_index = CCSTATE_RCI_INDEX_C3_RESIDENCY,
1750 .bic_number = BIC_CPU_c3,
1751 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1752 .pkg_cstate_limit = 0,
1753 },
1754 {
1755 .feature_mask = CC6,
1756 .perf_subsys = "cstate_core",
1757 .perf_name = "c6-residency",
1758 .msr = MSR_CORE_C6_RESIDENCY,
1759 .rci_index = CCSTATE_RCI_INDEX_C6_RESIDENCY,
1760 .bic_number = BIC_CPU_c6,
1761 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1762 .pkg_cstate_limit = 0,
1763 },
1764 {
1765 .feature_mask = CC7,
1766 .perf_subsys = "cstate_core",
1767 .perf_name = "c7-residency",
1768 .msr = MSR_CORE_C7_RESIDENCY,
1769 .rci_index = CCSTATE_RCI_INDEX_C7_RESIDENCY,
1770 .bic_number = BIC_CPU_c7,
1771 .flags = CSTATE_COUNTER_FLAG_COLLECT_PER_CORE | CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY,
1772 .pkg_cstate_limit = 0,
1773 },
1774 {
1775 .feature_mask = PC2,
1776 .perf_subsys = "cstate_pkg",
1777 .perf_name = "c2-residency",
1778 .msr = MSR_PKG_C2_RESIDENCY,
1779 .rci_index = PCSTATE_RCI_INDEX_C2_RESIDENCY,
1780 .bic_number = BIC_Pkgpc2,
1781 .flags = 0,
1782 .pkg_cstate_limit = PCL__2,
1783 },
1784 {
1785 .feature_mask = PC3,
1786 .perf_subsys = "cstate_pkg",
1787 .perf_name = "c3-residency",
1788 .msr = MSR_PKG_C3_RESIDENCY,
1789 .rci_index = PCSTATE_RCI_INDEX_C3_RESIDENCY,
1790 .bic_number = BIC_Pkgpc3,
1791 .flags = 0,
1792 .pkg_cstate_limit = PCL__3,
1793 },
1794 {
1795 .feature_mask = PC6,
1796 .perf_subsys = "cstate_pkg",
1797 .perf_name = "c6-residency",
1798 .msr = MSR_PKG_C6_RESIDENCY,
1799 .rci_index = PCSTATE_RCI_INDEX_C6_RESIDENCY,
1800 .bic_number = BIC_Pkgpc6,
1801 .flags = 0,
1802 .pkg_cstate_limit = PCL__6,
1803 },
1804 {
1805 .feature_mask = PC7,
1806 .perf_subsys = "cstate_pkg",
1807 .perf_name = "c7-residency",
1808 .msr = MSR_PKG_C7_RESIDENCY,
1809 .rci_index = PCSTATE_RCI_INDEX_C7_RESIDENCY,
1810 .bic_number = BIC_Pkgpc7,
1811 .flags = 0,
1812 .pkg_cstate_limit = PCL__7,
1813 },
1814 {
1815 .feature_mask = PC8,
1816 .perf_subsys = "cstate_pkg",
1817 .perf_name = "c8-residency",
1818 .msr = MSR_PKG_C8_RESIDENCY,
1819 .rci_index = PCSTATE_RCI_INDEX_C8_RESIDENCY,
1820 .bic_number = BIC_Pkgpc8,
1821 .flags = 0,
1822 .pkg_cstate_limit = PCL__8,
1823 },
1824 {
1825 .feature_mask = PC9,
1826 .perf_subsys = "cstate_pkg",
1827 .perf_name = "c9-residency",
1828 .msr = MSR_PKG_C9_RESIDENCY,
1829 .rci_index = PCSTATE_RCI_INDEX_C9_RESIDENCY,
1830 .bic_number = BIC_Pkgpc9,
1831 .flags = 0,
1832 .pkg_cstate_limit = PCL__9,
1833 },
1834 {
1835 .feature_mask = PC10,
1836 .perf_subsys = "cstate_pkg",
1837 .perf_name = "c10-residency",
1838 .msr = MSR_PKG_C10_RESIDENCY,
1839 .rci_index = PCSTATE_RCI_INDEX_C10_RESIDENCY,
1840 .bic_number = BIC_Pkgpc10,
1841 .flags = 0,
1842 .pkg_cstate_limit = PCL_10,
1843 },
1844 };
1845
1846 /* Indexes used to map data read from perf and MSRs into global variables */
1847 enum msr_rci_index {
1848 MSR_RCI_INDEX_APERF = 0,
1849 MSR_RCI_INDEX_MPERF = 1,
1850 MSR_RCI_INDEX_SMI = 2,
1851 NUM_MSR_COUNTERS,
1852 };
1853
1854 struct msr_counter_info_t {
1855 unsigned long long data[NUM_MSR_COUNTERS];
1856 enum counter_source source[NUM_MSR_COUNTERS];
1857 unsigned long long msr[NUM_MSR_COUNTERS];
1858 unsigned long long msr_mask[NUM_MSR_COUNTERS];
1859 int fd_perf;
1860 };
1861
1862 struct msr_counter_info_t *msr_counter_info;
1863 unsigned int msr_counter_info_size;
1864
1865 struct msr_counter_arch_info {
1866 const char *perf_subsys;
1867 const char *perf_name;
1868 unsigned long long msr;
1869 unsigned long long msr_mask;
1870 unsigned int rci_index; /* Maps data from perf counters to global variables */
1871 bool needed;
1872 bool present;
1873 };
1874
1875 enum msr_arch_info_index {
1876 MSR_ARCH_INFO_APERF_INDEX = 0,
1877 MSR_ARCH_INFO_MPERF_INDEX = 1,
1878 MSR_ARCH_INFO_SMI_INDEX = 2,
1879 };
1880
1881 static struct msr_counter_arch_info msr_counter_arch_infos[] = {
1882 [MSR_ARCH_INFO_APERF_INDEX] = {
1883 .perf_subsys = "msr",
1884 .perf_name = "aperf",
1885 .msr = MSR_IA32_APERF,
1886 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1887 .rci_index = MSR_RCI_INDEX_APERF,
1888 },
1889
1890 [MSR_ARCH_INFO_MPERF_INDEX] = {
1891 .perf_subsys = "msr",
1892 .perf_name = "mperf",
1893 .msr = MSR_IA32_MPERF,
1894 .msr_mask = 0xFFFFFFFFFFFFFFFF,
1895 .rci_index = MSR_RCI_INDEX_MPERF,
1896 },
1897
1898 [MSR_ARCH_INFO_SMI_INDEX] = {
1899 .perf_subsys = "msr",
1900 .perf_name = "smi",
1901 .msr = MSR_SMI_COUNT,
1902 .msr_mask = 0xFFFFFFFF,
1903 .rci_index = MSR_RCI_INDEX_SMI,
1904 },
1905 };
1906
1907 /* Can be redefined when compiling, useful for testing. */
1908 #ifndef SYSFS_TELEM_PATH
1909 #define SYSFS_TELEM_PATH "/sys/class/intel_pmt"
1910 #endif
1911
1912 #define PMT_COUNTER_MTL_DC6_OFFSET 120
1913 #define PMT_COUNTER_MTL_DC6_LSB 0
1914 #define PMT_COUNTER_MTL_DC6_MSB 63
1915 #define PMT_MTL_DC6_GUID 0x1a067102
1916 #define PMT_MTL_DC6_SEQ 0
1917
1918 #define PMT_COUNTER_CWF_MC1E_OFFSET_BASE 20936
1919 #define PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT 24
1920 #define PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE 12
1921 #define PMT_COUNTER_CWF_CPUS_PER_MODULE 4
1922 #define PMT_COUNTER_CWF_MC1E_LSB 0
1923 #define PMT_COUNTER_CWF_MC1E_MSB 63
1924 #define PMT_CWF_MC1E_GUID 0x14421519
1925
1926 unsigned long long tcore_clock_freq_hz = 800000000;
1927
1928 #define PMT_COUNTER_NAME_SIZE_BYTES 16
1929 #define PMT_COUNTER_TYPE_NAME_SIZE_BYTES 32
1930
1931 struct pmt_mmio {
1932 struct pmt_mmio *next;
1933
1934 unsigned int guid;
1935 unsigned int size;
1936
1937 /* Base pointer to the mmaped memory. */
1938 void *mmio_base;
1939
1940 /*
1941 * Offset to be applied to the mmio_base
1942 * to get the beginning of the PMT counters for given GUID.
1943 */
1944 unsigned long pmt_offset;
1945 } *pmt_mmios;
1946
1947 enum pmt_datatype {
1948 PMT_TYPE_RAW,
1949 PMT_TYPE_XTAL_TIME,
1950 PMT_TYPE_TCORE_CLOCK,
1951 };
1952
1953 struct pmt_domain_info {
1954 /*
1955 * Pointer to the MMIO obtained by applying a counter offset
1956 * to the mmio_base of the mmaped region for the given GUID.
1957 *
1958 * This is where to read the raw value of the counter from.
1959 */
1960 unsigned long *pcounter;
1961 };
1962
1963 struct pmt_counter {
1964 struct pmt_counter *next;
1965
1966 /* PMT metadata */
1967 char name[PMT_COUNTER_NAME_SIZE_BYTES];
1968 enum pmt_datatype type;
1969 enum counter_scope scope;
1970 unsigned int lsb;
1971 unsigned int msb;
1972
1973 /* BIC-like metadata */
1974 enum counter_format format;
1975
1976 unsigned int num_domains;
1977 struct pmt_domain_info *domains;
1978 };
1979
1980 /*
1981 * PMT telemetry directory iterator.
1982 * Used to iterate telemetry files in sysfs in correct order.
1983 */
1984 struct pmt_diriter_t {
1985 DIR *dir;
1986 struct dirent **namelist;
1987 unsigned int num_names;
1988 unsigned int current_name_idx;
1989 };
1990
pmt_telemdir_filter(const struct dirent * e)1991 int pmt_telemdir_filter(const struct dirent *e)
1992 {
1993 unsigned int dummy;
1994
1995 return sscanf(e->d_name, "telem%u", &dummy);
1996 }
1997
pmt_telemdir_sort(const struct dirent ** a,const struct dirent ** b)1998 int pmt_telemdir_sort(const struct dirent **a, const struct dirent **b)
1999 {
2000 unsigned int aidx = 0, bidx = 0;
2001
2002 sscanf((*a)->d_name, "telem%u", &aidx);
2003 sscanf((*b)->d_name, "telem%u", &bidx);
2004
2005 return (aidx > bidx) ? 1 : (aidx < bidx) ? -1 : 0;
2006 }
2007
pmt_diriter_next(struct pmt_diriter_t * iter)2008 const struct dirent *pmt_diriter_next(struct pmt_diriter_t *iter)
2009 {
2010 const struct dirent *ret = NULL;
2011
2012 if (!iter->dir)
2013 return NULL;
2014
2015 if (iter->current_name_idx >= iter->num_names)
2016 return NULL;
2017
2018 ret = iter->namelist[iter->current_name_idx];
2019 ++iter->current_name_idx;
2020
2021 return ret;
2022 }
2023
pmt_diriter_begin(struct pmt_diriter_t * iter,const char * pmt_root_path)2024 const struct dirent *pmt_diriter_begin(struct pmt_diriter_t *iter, const char *pmt_root_path)
2025 {
2026 int num_names = iter->num_names;
2027
2028 if (!iter->dir) {
2029 iter->dir = opendir(pmt_root_path);
2030 if (iter->dir == NULL)
2031 return NULL;
2032
2033 num_names = scandir(pmt_root_path, &iter->namelist, pmt_telemdir_filter, pmt_telemdir_sort);
2034 if (num_names == -1)
2035 return NULL;
2036 }
2037
2038 iter->current_name_idx = 0;
2039 iter->num_names = num_names;
2040
2041 return pmt_diriter_next(iter);
2042 }
2043
pmt_diriter_init(struct pmt_diriter_t * iter)2044 void pmt_diriter_init(struct pmt_diriter_t *iter)
2045 {
2046 memset(iter, 0, sizeof(*iter));
2047 }
2048
pmt_diriter_remove(struct pmt_diriter_t * iter)2049 void pmt_diriter_remove(struct pmt_diriter_t *iter)
2050 {
2051 if (iter->namelist) {
2052 for (unsigned int i = 0; i < iter->num_names; i++) {
2053 free(iter->namelist[i]);
2054 iter->namelist[i] = NULL;
2055 }
2056 }
2057
2058 free(iter->namelist);
2059 iter->namelist = NULL;
2060 iter->num_names = 0;
2061 iter->current_name_idx = 0;
2062
2063 closedir(iter->dir);
2064 iter->dir = NULL;
2065 }
2066
pmt_counter_get_width(const struct pmt_counter * p)2067 unsigned int pmt_counter_get_width(const struct pmt_counter *p)
2068 {
2069 return (p->msb - p->lsb) + 1;
2070 }
2071
pmt_counter_resize_(struct pmt_counter * pcounter,unsigned int new_size)2072 void pmt_counter_resize_(struct pmt_counter *pcounter, unsigned int new_size)
2073 {
2074 struct pmt_domain_info *new_mem;
2075
2076 new_mem = (struct pmt_domain_info *)reallocarray(pcounter->domains, new_size, sizeof(*pcounter->domains));
2077 if (!new_mem) {
2078 fprintf(stderr, "%s: failed to allocate memory for PMT counters\n", __func__);
2079 exit(1);
2080 }
2081
2082 /* Zero initialize just allocated memory. */
2083 const size_t num_new_domains = new_size - pcounter->num_domains;
2084
2085 memset(&new_mem[pcounter->num_domains], 0, num_new_domains * sizeof(*pcounter->domains));
2086
2087 pcounter->num_domains = new_size;
2088 pcounter->domains = new_mem;
2089 }
2090
pmt_counter_resize(struct pmt_counter * pcounter,unsigned int new_size)2091 void pmt_counter_resize(struct pmt_counter *pcounter, unsigned int new_size)
2092 {
2093 /*
2094 * Allocate more memory ahead of time.
2095 *
2096 * Always allocate space for at least 8 elements
2097 * and double the size when growing.
2098 */
2099 if (new_size < 8)
2100 new_size = 8;
2101 new_size = MAX(new_size, pcounter->num_domains * 2);
2102
2103 pmt_counter_resize_(pcounter, new_size);
2104 }
2105
2106 struct llc_stats {
2107 unsigned long long references;
2108 unsigned long long misses;
2109 };
2110 struct l2_stats {
2111 unsigned long long references;
2112 unsigned long long hits;
2113 };
2114 struct thread_data {
2115 struct timeval tv_begin;
2116 struct timeval tv_end;
2117 struct timeval tv_delta;
2118 unsigned long long tsc;
2119 unsigned long long aperf;
2120 unsigned long long mperf;
2121 unsigned long long c1;
2122 unsigned long long instr_count;
2123 unsigned long long irq_count;
2124 unsigned long long nmi_count;
2125 unsigned int smi_count;
2126 struct llc_stats llc;
2127 struct l2_stats l2;
2128 unsigned int cpu_id;
2129 unsigned int apic_id;
2130 unsigned int x2apic_id;
2131 unsigned int flags;
2132 bool is_atom;
2133 unsigned long long counter[MAX_ADDED_THREAD_COUNTERS];
2134 unsigned long long perf_counter[MAX_ADDED_THREAD_COUNTERS];
2135 unsigned long long pmt_counter[PMT_MAX_ADDED_THREAD_COUNTERS];
2136 };
2137
2138 struct core_data {
2139 int first_cpu;
2140 unsigned long long c3;
2141 unsigned long long c6;
2142 unsigned long long c7;
2143 unsigned long long mc6_us; /* duplicate as per-core for now, even though per module */
2144 unsigned int core_temp_c;
2145 struct rapl_counter core_energy; /* MSR_CORE_ENERGY_STAT */
2146 unsigned long long core_throt_cnt;
2147 unsigned long long counter[MAX_ADDED_CORE_COUNTERS];
2148 unsigned long long perf_counter[MAX_ADDED_CORE_COUNTERS];
2149 unsigned long long pmt_counter[PMT_MAX_ADDED_CORE_COUNTERS];
2150 };
2151
2152 struct pkg_data {
2153 int first_cpu;
2154 unsigned long long pc2;
2155 unsigned long long pc3;
2156 unsigned long long pc6;
2157 unsigned long long pc7;
2158 unsigned long long pc8;
2159 unsigned long long pc9;
2160 unsigned long long pc10;
2161 long long cpu_lpi;
2162 long long sys_lpi;
2163 unsigned long long pkg_wtd_core_c0;
2164 unsigned long long pkg_any_core_c0;
2165 unsigned long long pkg_any_gfxe_c0;
2166 unsigned long long pkg_both_core_gfxe_c0;
2167 long long gfx_rc6_ms;
2168 unsigned int gfx_mhz;
2169 unsigned int gfx_act_mhz;
2170 long long sam_mc6_ms;
2171 unsigned int sam_mhz;
2172 unsigned int sam_act_mhz;
2173 struct rapl_counter energy_pkg; /* MSR_PKG_ENERGY_STATUS */
2174 struct rapl_counter energy_dram; /* MSR_DRAM_ENERGY_STATUS */
2175 struct rapl_counter energy_cores; /* MSR_PP0_ENERGY_STATUS */
2176 struct rapl_counter energy_gfx; /* MSR_PP1_ENERGY_STATUS */
2177 struct rapl_counter rapl_pkg_perf_status; /* MSR_PKG_PERF_STATUS */
2178 struct rapl_counter rapl_dram_perf_status; /* MSR_DRAM_PERF_STATUS */
2179 unsigned int pkg_temp_c;
2180 unsigned int uncore_mhz;
2181 unsigned long long die_c6;
2182 unsigned long long counter[MAX_ADDED_PACKAGE_COUNTERS];
2183 unsigned long long perf_counter[MAX_ADDED_PACKAGE_COUNTERS];
2184 unsigned long long pmt_counter[PMT_MAX_ADDED_PACKAGE_COUNTERS];
2185 };
2186
2187 #define ODD_COUNTERS odd.threads, odd.cores, odd.packages
2188 #define EVEN_COUNTERS even.threads, even.cores, even.packages
2189
2190 /*
2191 * The accumulated sum of MSR is defined as a monotonic
2192 * increasing MSR, it will be accumulated periodically,
2193 * despite its register's bit width.
2194 */
2195 enum {
2196 IDX_PKG_ENERGY,
2197 IDX_DRAM_ENERGY,
2198 IDX_PP0_ENERGY,
2199 IDX_PP1_ENERGY,
2200 IDX_PKG_PERF,
2201 IDX_DRAM_PERF,
2202 IDX_PSYS_ENERGY,
2203 IDX_COUNT,
2204 };
2205
2206 int get_msr_sum(int cpu, off_t offset, unsigned long long *msr);
2207
2208 struct msr_sum_array {
2209 /* get_msr_sum() = sum + (get_msr() - last) */
2210 struct {
2211 /*The accumulated MSR value is updated by the timer */
2212 unsigned long long sum;
2213 /*The MSR footprint recorded in last timer */
2214 unsigned long long last;
2215 } entries[IDX_COUNT];
2216 };
2217
2218 /* The percpu MSR sum array.*/
2219 struct msr_sum_array *per_cpu_msr_sum;
2220
idx_to_offset(int idx)2221 off_t idx_to_offset(int idx)
2222 {
2223 off_t offset;
2224
2225 switch (idx) {
2226 case IDX_PKG_ENERGY:
2227 if (platform->plat_rapl_msrs & RAPL_AMD_F17H)
2228 offset = MSR_PKG_ENERGY_STAT;
2229 else
2230 offset = MSR_PKG_ENERGY_STATUS;
2231 break;
2232 case IDX_DRAM_ENERGY:
2233 offset = MSR_DRAM_ENERGY_STATUS;
2234 break;
2235 case IDX_PP0_ENERGY:
2236 offset = MSR_PP0_ENERGY_STATUS;
2237 break;
2238 case IDX_PP1_ENERGY:
2239 offset = MSR_PP1_ENERGY_STATUS;
2240 break;
2241 case IDX_PKG_PERF:
2242 offset = MSR_PKG_PERF_STATUS;
2243 break;
2244 case IDX_DRAM_PERF:
2245 offset = MSR_DRAM_PERF_STATUS;
2246 break;
2247 case IDX_PSYS_ENERGY:
2248 offset = MSR_PLATFORM_ENERGY_STATUS;
2249 break;
2250 default:
2251 offset = -1;
2252 }
2253 return offset;
2254 }
2255
offset_to_idx(off_t offset)2256 int offset_to_idx(off_t offset)
2257 {
2258 int idx;
2259
2260 switch (offset) {
2261 case MSR_PKG_ENERGY_STATUS:
2262 case MSR_PKG_ENERGY_STAT:
2263 idx = IDX_PKG_ENERGY;
2264 break;
2265 case MSR_DRAM_ENERGY_STATUS:
2266 idx = IDX_DRAM_ENERGY;
2267 break;
2268 case MSR_PP0_ENERGY_STATUS:
2269 idx = IDX_PP0_ENERGY;
2270 break;
2271 case MSR_PP1_ENERGY_STATUS:
2272 idx = IDX_PP1_ENERGY;
2273 break;
2274 case MSR_PKG_PERF_STATUS:
2275 idx = IDX_PKG_PERF;
2276 break;
2277 case MSR_DRAM_PERF_STATUS:
2278 idx = IDX_DRAM_PERF;
2279 break;
2280 case MSR_PLATFORM_ENERGY_STATUS:
2281 idx = IDX_PSYS_ENERGY;
2282 break;
2283 default:
2284 idx = -1;
2285 }
2286 return idx;
2287 }
2288
idx_valid(int idx)2289 int idx_valid(int idx)
2290 {
2291 switch (idx) {
2292 case IDX_PKG_ENERGY:
2293 return valid_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H);
2294 case IDX_DRAM_ENERGY:
2295 return valid_rapl_msrs & RAPL_DRAM;
2296 case IDX_PP0_ENERGY:
2297 return valid_rapl_msrs & RAPL_CORE_ENERGY_STATUS;
2298 case IDX_PP1_ENERGY:
2299 return valid_rapl_msrs & RAPL_GFX;
2300 case IDX_PKG_PERF:
2301 return valid_rapl_msrs & RAPL_PKG_PERF_STATUS;
2302 case IDX_DRAM_PERF:
2303 return valid_rapl_msrs & RAPL_DRAM_PERF_STATUS;
2304 case IDX_PSYS_ENERGY:
2305 return valid_rapl_msrs & RAPL_PSYS;
2306 default:
2307 return 0;
2308 }
2309 }
2310
2311 struct sys_counters {
2312 /* MSR added counters */
2313 unsigned int added_thread_counters;
2314 unsigned int added_core_counters;
2315 unsigned int added_package_counters;
2316 struct msr_counter *tp;
2317 struct msr_counter *cp;
2318 struct msr_counter *pp;
2319
2320 /* perf added counters */
2321 unsigned int added_thread_perf_counters;
2322 unsigned int added_core_perf_counters;
2323 unsigned int added_package_perf_counters;
2324 struct perf_counter_info *perf_tp;
2325 struct perf_counter_info *perf_cp;
2326 struct perf_counter_info *perf_pp;
2327
2328 struct pmt_counter *pmt_tp;
2329 struct pmt_counter *pmt_cp;
2330 struct pmt_counter *pmt_pp;
2331 } sys;
2332
free_msr_counters_(struct msr_counter ** pp)2333 static size_t free_msr_counters_(struct msr_counter **pp)
2334 {
2335 struct msr_counter *p = NULL;
2336 size_t num_freed = 0;
2337
2338 while (*pp) {
2339 p = *pp;
2340
2341 if (p->msr_num != 0) {
2342 *pp = p->next;
2343
2344 free(p);
2345 ++num_freed;
2346
2347 continue;
2348 }
2349
2350 pp = &p->next;
2351 }
2352
2353 return num_freed;
2354 }
2355
2356 /*
2357 * Free all added counters accessed via msr.
2358 */
free_sys_msr_counters(void)2359 static void free_sys_msr_counters(void)
2360 {
2361 /* Thread counters */
2362 sys.added_thread_counters -= free_msr_counters_(&sys.tp);
2363
2364 /* Core counters */
2365 sys.added_core_counters -= free_msr_counters_(&sys.cp);
2366
2367 /* Package counters */
2368 sys.added_package_counters -= free_msr_counters_(&sys.pp);
2369 }
2370
2371 struct counters {
2372 struct thread_data *threads;
2373 struct core_data *cores;
2374 struct pkg_data *packages;
2375 } average, even, odd;
2376
2377 struct platform_counters {
2378 struct rapl_counter energy_psys; /* MSR_PLATFORM_ENERGY_STATUS */
2379 } platform_counters_odd, platform_counters_even;
2380
2381 #define MAX_HT_ID 3 /* support SMT-4 */
2382
2383 struct cpu_topology {
2384 int cpu_id;
2385 int core_id; /* unique within a package */
2386 int package_id;
2387 int die_id;
2388 int l3_id;
2389 int physical_node_id;
2390 int logical_node_id; /* 0-based count within the package */
2391 int ht_id; /* unique within a core */
2392 int ht_sibling_cpu_id[MAX_HT_ID + 1];
2393 int type;
2394 cpu_set_t *put_ids; /* Processing Unit/Thread IDs */
2395 } *cpus;
2396
2397 struct topo_params {
2398 int num_packages;
2399 int num_die;
2400 int num_cpus;
2401 int num_cores; /* system wide */
2402 int allowed_packages;
2403 int allowed_cpus;
2404 int allowed_cores;
2405 int max_cpu_num;
2406 int max_core_id; /* within a package */
2407 int max_package_id;
2408 int max_die_id;
2409 int max_l3_id;
2410 int max_node_num;
2411 int nodes_per_pkg;
2412 int cores_per_pkg;
2413 int threads_per_core;
2414 } topo;
2415
2416 struct timeval tv_even, tv_odd, tv_delta;
2417
2418 int *irq_column_2_cpu; /* /proc/interrupts column numbers */
2419 int *irqs_per_cpu; /* indexed by cpu_num */
2420 int *nmi_per_cpu; /* indexed by cpu_num */
2421
2422 void setup_all_buffers(bool startup);
2423
2424 char *sys_lpi_file;
2425 char *sys_lpi_file_sysfs = "/sys/devices/system/cpu/cpuidle/low_power_idle_system_residency_us";
2426 char *sys_lpi_file_debugfs = "/sys/kernel/debug/pmc_core/slp_s0_residency_usec";
2427
cpu_is_not_present(int cpu)2428 int cpu_is_not_present(int cpu)
2429 {
2430 return !CPU_ISSET_S(cpu, cpu_present_setsize, cpu_present_set);
2431 }
2432
cpu_is_not_allowed(int cpu)2433 int cpu_is_not_allowed(int cpu)
2434 {
2435 return !CPU_ISSET_S(cpu, cpu_allowed_setsize, cpu_allowed_set);
2436 }
2437
2438 #define GLOBAL_CORE_ID(core_id, pkg_id) (core_id + pkg_id * (topo.max_core_id + 1))
2439 /*
2440 * run func(thread, core, package) in topology order
2441 * skip non-present cpus
2442 */
2443
2444 #define PER_THREAD_PARAMS struct thread_data *t, struct core_data *c, struct pkg_data *p
2445
for_all_cpus(int (func)(struct thread_data *,struct core_data *,struct pkg_data *),struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base)2446 int for_all_cpus(int (func) (struct thread_data *, struct core_data *, struct pkg_data *),
2447 struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base)
2448 {
2449 int cpu, retval;
2450
2451 retval = 0;
2452
2453 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
2454 struct thread_data *t;
2455 struct core_data *c;
2456 struct pkg_data *p;
2457
2458 int pkg_id = cpus[cpu].package_id;
2459
2460 if (cpu_is_not_allowed(cpu))
2461 continue;
2462
2463 if (cpus[cpu].ht_id > 0) /* skip HT sibling */
2464 continue;
2465
2466 t = &thread_base[cpu];
2467 c = &core_base[GLOBAL_CORE_ID(cpus[cpu].core_id, pkg_id)];
2468 p = &pkg_base[pkg_id];
2469
2470 retval |= func(t, c, p);
2471
2472 /* Handle HT sibling now */
2473 int i;
2474
2475 for (i = MAX_HT_ID; i > 0; --i) { /* ht_id 0 is self */
2476 if (cpus[cpu].ht_sibling_cpu_id[i] <= 0)
2477 continue;
2478 t = &thread_base[cpus[cpu].ht_sibling_cpu_id[i]];
2479
2480 retval |= func(t, c, p);
2481 }
2482 }
2483 return retval;
2484 }
2485
is_cpu_first_thread_in_core(struct thread_data * t,struct core_data * c)2486 int is_cpu_first_thread_in_core(struct thread_data *t, struct core_data *c)
2487 {
2488 return ((int)t->cpu_id == c->first_cpu || c->first_cpu < 0);
2489 }
2490
is_cpu_first_core_in_package(struct thread_data * t,struct pkg_data * p)2491 int is_cpu_first_core_in_package(struct thread_data *t, struct pkg_data *p)
2492 {
2493 return ((int)t->cpu_id == p->first_cpu || p->first_cpu < 0);
2494 }
2495
is_cpu_first_thread_in_package(struct thread_data * t,struct core_data * c,struct pkg_data * p)2496 int is_cpu_first_thread_in_package(struct thread_data *t, struct core_data *c, struct pkg_data *p)
2497 {
2498 return is_cpu_first_thread_in_core(t, c) && is_cpu_first_core_in_package(t, p);
2499 }
2500
cpu_migrate(int cpu)2501 int cpu_migrate(int cpu)
2502 {
2503 CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
2504 CPU_SET_S(cpu, cpu_affinity_setsize, cpu_affinity_set);
2505 if (sched_setaffinity(0, cpu_affinity_setsize, cpu_affinity_set) == -1)
2506 return -1;
2507 else
2508 return 0;
2509 }
2510
get_msr_fd(int cpu)2511 int get_msr_fd(int cpu)
2512 {
2513 char pathname[32];
2514 int fd;
2515
2516 fd = fd_percpu[cpu];
2517
2518 if (fd)
2519 return fd;
2520 sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", cpu);
2521 fd = open(pathname, O_RDONLY);
2522 if (fd < 0)
2523 err(-1, "%s open failed, try chown or chmod +r %s, "
2524 "or run with --no-msr, or run as root", pathname, use_android_msr_path ? "/dev/msr*" : "/dev/cpu/*/msr");
2525 fd_percpu[cpu] = fd;
2526
2527 return fd;
2528 }
2529
bic_disable_msr_access(void)2530 static void bic_disable_msr_access(void)
2531 {
2532 CLR_BIC(BIC_Mod_c6, &bic_enabled);
2533 CLR_BIC(BIC_CoreTmp, &bic_enabled);
2534 CLR_BIC(BIC_Totl_c0, &bic_enabled);
2535 CLR_BIC(BIC_Any_c0, &bic_enabled);
2536 CLR_BIC(BIC_GFX_c0, &bic_enabled);
2537 CLR_BIC(BIC_CPUGFX, &bic_enabled);
2538 CLR_BIC(BIC_PkgTmp, &bic_enabled);
2539
2540 free_sys_msr_counters();
2541 }
2542
bic_disable_perf_access(void)2543 static void bic_disable_perf_access(void)
2544 {
2545 CLR_BIC(BIC_IPC, &bic_enabled);
2546 CLR_BIC(BIC_LLC_MRPS, &bic_enabled);
2547 CLR_BIC(BIC_LLC_HIT, &bic_enabled);
2548 CLR_BIC(BIC_L2_MRPS, &bic_enabled);
2549 CLR_BIC(BIC_L2_HIT, &bic_enabled);
2550 }
2551
perf_event_open(struct perf_event_attr * hw_event,pid_t pid,int cpu,int group_fd,unsigned long flags)2552 static long perf_event_open(struct perf_event_attr *hw_event, pid_t pid, int cpu, int group_fd, unsigned long flags)
2553 {
2554 assert(!no_perf);
2555
2556 return syscall(__NR_perf_event_open, hw_event, pid, cpu, group_fd, flags);
2557 }
2558
open_perf_counter(int cpu,unsigned int type,unsigned int config,int group_fd,__u64 read_format)2559 static long open_perf_counter(int cpu, unsigned int type, unsigned int config, int group_fd, __u64 read_format)
2560 {
2561 struct perf_event_attr attr;
2562 const pid_t pid = -1;
2563 const unsigned long flags = 0;
2564
2565 assert(!no_perf);
2566
2567 memset(&attr, 0, sizeof(struct perf_event_attr));
2568
2569 attr.type = type;
2570 attr.size = sizeof(struct perf_event_attr);
2571 attr.config = config;
2572 attr.disabled = 0;
2573 attr.sample_type = PERF_SAMPLE_IDENTIFIER;
2574 attr.read_format = read_format;
2575
2576 const int fd = perf_event_open(&attr, pid, cpu, group_fd, flags);
2577
2578 return fd;
2579 }
2580
get_instr_count_fd(int cpu)2581 int get_instr_count_fd(int cpu)
2582 {
2583 if (fd_instr_count_percpu[cpu])
2584 return fd_instr_count_percpu[cpu];
2585
2586 fd_instr_count_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0);
2587
2588 return fd_instr_count_percpu[cpu];
2589 }
2590
get_msr(int cpu,off_t offset,unsigned long long * msr)2591 int get_msr(int cpu, off_t offset, unsigned long long *msr)
2592 {
2593 ssize_t retval;
2594
2595 assert(!no_msr);
2596
2597 retval = pread(get_msr_fd(cpu), msr, sizeof(*msr), offset);
2598
2599 if (retval != sizeof *msr)
2600 err(-1, "cpu%d: msr offset 0x%llx read failed", cpu, (unsigned long long)offset);
2601
2602 return 0;
2603 }
2604
add_msr_counter(int cpu,off_t offset)2605 int add_msr_counter(int cpu, off_t offset)
2606 {
2607 ssize_t retval;
2608 unsigned long long value;
2609
2610 if (no_msr)
2611 return -1;
2612
2613 if (!offset)
2614 return -1;
2615
2616 retval = pread(get_msr_fd(cpu), &value, sizeof(value), offset);
2617
2618 /* if the read failed, the probe fails */
2619 if (retval != sizeof(value))
2620 return -1;
2621
2622 if (value == 0)
2623 return 0;
2624
2625 return 1;
2626 }
2627
add_rapl_msr_counter(int cpu,const struct rapl_counter_arch_info * cai)2628 int add_rapl_msr_counter(int cpu, const struct rapl_counter_arch_info *cai)
2629 {
2630 int ret;
2631
2632 if (!(valid_rapl_msrs & cai->feature_mask))
2633 return -1;
2634
2635 ret = add_msr_counter(cpu, cai->msr);
2636 if (ret < 0)
2637 return -1;
2638
2639 switch (cai->rci_index) {
2640 case RAPL_RCI_INDEX_ENERGY_PKG:
2641 case RAPL_RCI_INDEX_ENERGY_CORES:
2642 case RAPL_RCI_INDEX_DRAM:
2643 case RAPL_RCI_INDEX_GFX:
2644 case RAPL_RCI_INDEX_ENERGY_PLATFORM:
2645 if (ret == 0)
2646 return 1;
2647 }
2648
2649 /* PKG,DRAM_PERF_STATUS MSRs, can return any value */
2650 return 1;
2651 }
2652
2653 /* Convert CPU ID to domain ID for given added perf counter. */
cpu_to_domain(const struct perf_counter_info * pc,int cpu)2654 unsigned int cpu_to_domain(const struct perf_counter_info *pc, int cpu)
2655 {
2656 switch (pc->scope) {
2657 case SCOPE_CPU:
2658 return cpu;
2659
2660 case SCOPE_CORE:
2661 return cpus[cpu].core_id;
2662
2663 case SCOPE_PACKAGE:
2664 return cpus[cpu].package_id;
2665 }
2666
2667 __builtin_unreachable();
2668 }
2669
2670 #define MAX_DEFERRED 16
2671 char *deferred_add_names[MAX_DEFERRED];
2672 char *deferred_skip_names[MAX_DEFERRED];
2673 int deferred_add_index;
2674 int deferred_skip_index;
2675 unsigned int deferred_add_consumed;
2676 unsigned int deferred_skip_consumed;
2677
2678 /*
2679 * HIDE_LIST - hide this list of counters, show the rest [default]
2680 * SHOW_LIST - show this list of counters, hide the rest
2681 */
2682 enum show_hide_mode { SHOW_LIST, HIDE_LIST } global_show_hide_mode = HIDE_LIST;
2683
help(void)2684 void help(void)
2685 {
2686 fprintf(outf,
2687 "Usage: turbostat [OPTIONS][(--interval seconds) | COMMAND ...]\n"
2688 "\n"
2689 "Turbostat forks the specified COMMAND and prints statistics\n"
2690 "when COMMAND completes.\n"
2691 "If no COMMAND is specified, turbostat wakes every 5-seconds\n"
2692 "to print statistics, until interrupted.\n"
2693 " -a, --add counter\n"
2694 " add a counter\n"
2695 " eg. --add msr0x10,u64,cpu,delta,MY_TSC\n"
2696 " eg. --add perf/cstate_pkg/c2-residency,package,delta,percent,perfPC2\n"
2697 " eg. --add pmt,name=XTAL,type=raw,domain=package0,offset=0,lsb=0,msb=63,guid=0x1a067102\n"
2698 " -c, --cpu cpu-set\n"
2699 " limit output to summary plus cpu-set:\n"
2700 " {core | package | j,k,l..m,n-p }\n"
2701 " -d, --debug\n"
2702 " displays usec, Time_Of_Day_Seconds and more debugging\n"
2703 " debug messages are printed to stderr\n"
2704 " -D, --Dump\n"
2705 " displays the raw counter values\n"
2706 " -e, --enable [all | column]\n"
2707 " shows all or the specified disabled column\n"
2708 " -f, --force\n"
2709 " force load turbostat with minimum default features on unsupported platforms.\n"
2710 " -H, --hide [column | column,column,...]\n"
2711 " hide the specified column(s)\n"
2712 " -i, --interval sec.subsec\n"
2713 " override default 5-second measurement interval\n"
2714 " -J, --Joules\n"
2715 " displays energy in Joules instead of Watts\n"
2716 " -l, --list\n"
2717 " list column headers only\n"
2718 " -M, --no-msr\n"
2719 " disable all uses of the MSR driver\n"
2720 " -P, --no-perf\n"
2721 " disable all uses of the perf API\n"
2722 " -n, --num_iterations num\n"
2723 " number of the measurement iterations\n"
2724 " -N, --header_iterations num\n"
2725 " print header every num iterations\n"
2726 " -o, --out file\n"
2727 " create or truncate \"file\" for all output\n"
2728 " -q, --quiet\n"
2729 " skip decoding system configuration header\n"
2730 " -s, --show [column | column,column,...]\n"
2731 " show only the specified column(s)\n"
2732 " -S, --Summary\n"
2733 " limits output to 1-line system summary per interval\n"
2734 " -T, --TCC temperature\n"
2735 " sets the Thermal Control Circuit temperature in\n"
2736 " degrees Celsius\n"
2737 " -h, --help\n"
2738 " print this help message\n -v, --version\n\t\tprint version information\n\nFor more help, run \"man turbostat\"\n");
2739 }
2740
2741 /*
2742 * bic_lookup
2743 * for all the strings in comma separate name_list,
2744 * set the approprate bit in return value.
2745 */
bic_lookup(cpu_set_t * ret_set,char * name_list,enum show_hide_mode mode)2746 void bic_lookup(cpu_set_t *ret_set, char *name_list, enum show_hide_mode mode)
2747 {
2748 unsigned int i;
2749
2750 while (name_list) {
2751 char *comma;
2752
2753 comma = strchr(name_list, ',');
2754
2755 if (comma)
2756 *comma = '\0';
2757
2758 for (i = 0; i < MAX_BIC; ++i) {
2759 if (!strcmp(name_list, bic[i].name)) {
2760 SET_BIC(i, ret_set);
2761 break;
2762 }
2763 if (!strcmp(name_list, "all")) {
2764 bic_set_all(ret_set);
2765 break;
2766 } else if (!strcmp(name_list, "topology")) {
2767 CPU_OR(ret_set, ret_set, &bic_group_topology);
2768 break;
2769 } else if (!strcmp(name_list, "power")) {
2770 CPU_OR(ret_set, ret_set, &bic_group_thermal_pwr);
2771 break;
2772 } else if (!strcmp(name_list, "idle")) {
2773 CPU_OR(ret_set, ret_set, &bic_group_idle);
2774 break;
2775 } else if (!strcmp(name_list, "cache")) {
2776 CPU_OR(ret_set, ret_set, &bic_group_cache);
2777 break;
2778 } else if (!strcmp(name_list, "llc")) {
2779 CPU_OR(ret_set, ret_set, &bic_group_cache);
2780 break;
2781 } else if (!strcmp(name_list, "swidle")) {
2782 CPU_OR(ret_set, ret_set, &bic_group_sw_idle);
2783 break;
2784 } else if (!strcmp(name_list, "sysfs")) { /* legacy compatibility */
2785 CPU_OR(ret_set, ret_set, &bic_group_sw_idle);
2786 break;
2787 } else if (!strcmp(name_list, "hwidle")) {
2788 CPU_OR(ret_set, ret_set, &bic_group_hw_idle);
2789 break;
2790 } else if (!strcmp(name_list, "frequency")) {
2791 CPU_OR(ret_set, ret_set, &bic_group_frequency);
2792 break;
2793 } else if (!strcmp(name_list, "other")) {
2794 CPU_OR(ret_set, ret_set, &bic_group_other);
2795 break;
2796 }
2797 }
2798 if (i == MAX_BIC) {
2799 if (mode == SHOW_LIST) {
2800 deferred_add_names[deferred_add_index++] = name_list;
2801 if (deferred_add_index >= MAX_DEFERRED) {
2802 fprintf(stderr, "More than max %d un-recognized --add options '%s'\n", MAX_DEFERRED, name_list);
2803 help();
2804 exit(1);
2805 }
2806 } else {
2807 deferred_skip_names[deferred_skip_index++] = name_list;
2808 if (debug)
2809 fprintf(stderr, "deferred \"%s\"\n", name_list);
2810 if (deferred_skip_index >= MAX_DEFERRED) {
2811 fprintf(stderr, "More than max %d un-recognized --skip options '%s'\n", MAX_DEFERRED, name_list);
2812 help();
2813 exit(1);
2814 }
2815 }
2816 }
2817
2818 name_list = comma;
2819 if (name_list)
2820 name_list++;
2821
2822 }
2823 }
2824
2825 /*
2826 * print_name()
2827 * Print column header name for raw 64-bit counter in 16 columns (at least 8-char plus a tab)
2828 * Otherwise, allow the name + tab to fit within 8-coumn tab-stop.
2829 * In both cases, left justififed, just like other turbostat columns,
2830 * to allow the column values to consume the tab.
2831 *
2832 * Yes, 32-bit counters can overflow 8-columns, and
2833 * 64-bit counters can overflow 16-columns, but that is uncommon.
2834 */
print_name(int width,int * printed,char * delim,char * name,enum counter_type type,enum counter_format format)2835 static inline int print_name(int width, int *printed, char *delim, char *name, enum counter_type type, enum counter_format format)
2836 {
2837 UNUSED(type);
2838
2839 if (format == FORMAT_RAW && width >= 64)
2840 return (sprintf(outp, "%s%-8s", ((*printed)++ ? delim : ""), name));
2841 else
2842 return (sprintf(outp, "%s%s", ((*printed)++ ? delim : ""), name));
2843 }
2844
print_hex_value(int width,int * printed,char * delim,unsigned long long value)2845 static inline int print_hex_value(int width, int *printed, char *delim, unsigned long long value)
2846 {
2847 if (width <= 32)
2848 return (sprintf(outp, "%s%08x", ((*printed)++ ? delim : ""), (unsigned int)value));
2849 else
2850 return (sprintf(outp, "%s%016llx", ((*printed)++ ? delim : ""), value));
2851 }
2852
print_decimal_value(int width,int * printed,char * delim,unsigned long long value)2853 static inline int print_decimal_value(int width, int *printed, char *delim, unsigned long long value)
2854 {
2855 UNUSED(width);
2856
2857 return (sprintf(outp, "%s%lld", ((*printed)++ ? delim : ""), value));
2858 }
2859
print_float_value(int * printed,char * delim,double value)2860 static inline int print_float_value(int *printed, char *delim, double value)
2861 {
2862 return (sprintf(outp, "%s%0.2f", ((*printed)++ ? delim : ""), value));
2863 }
2864
print_header(char * delim)2865 void print_header(char *delim)
2866 {
2867 struct msr_counter *mp;
2868 struct perf_counter_info *pp;
2869 struct pmt_counter *ppmt;
2870 int printed = 0;
2871
2872 if (DO_BIC(BIC_USEC))
2873 outp += sprintf(outp, "%susec", (printed++ ? delim : ""));
2874 if (DO_BIC(BIC_TOD))
2875 outp += sprintf(outp, "%sTime_Of_Day_Seconds", (printed++ ? delim : ""));
2876 if (DO_BIC(BIC_Package))
2877 outp += sprintf(outp, "%sPackage", (printed++ ? delim : ""));
2878 if (DO_BIC(BIC_Die))
2879 outp += sprintf(outp, "%sDie", (printed++ ? delim : ""));
2880 if (DO_BIC(BIC_L3))
2881 outp += sprintf(outp, "%sL3", (printed++ ? delim : ""));
2882 if (DO_BIC(BIC_Node))
2883 outp += sprintf(outp, "%sNode", (printed++ ? delim : ""));
2884 if (DO_BIC(BIC_Core))
2885 outp += sprintf(outp, "%sCore", (printed++ ? delim : ""));
2886 if (DO_BIC(BIC_CPU))
2887 outp += sprintf(outp, "%sCPU", (printed++ ? delim : ""));
2888 if (DO_BIC(BIC_APIC))
2889 outp += sprintf(outp, "%sAPIC", (printed++ ? delim : ""));
2890 if (DO_BIC(BIC_X2APIC))
2891 outp += sprintf(outp, "%sX2APIC", (printed++ ? delim : ""));
2892 if (DO_BIC(BIC_Avg_MHz))
2893 outp += sprintf(outp, "%sAvg_MHz", (printed++ ? delim : ""));
2894 if (DO_BIC(BIC_Busy))
2895 outp += sprintf(outp, "%sBusy%%", (printed++ ? delim : ""));
2896 if (DO_BIC(BIC_Bzy_MHz))
2897 outp += sprintf(outp, "%sBzy_MHz", (printed++ ? delim : ""));
2898 if (DO_BIC(BIC_TSC_MHz))
2899 outp += sprintf(outp, "%sTSC_MHz", (printed++ ? delim : ""));
2900
2901 if (DO_BIC(BIC_IPC))
2902 outp += sprintf(outp, "%sIPC", (printed++ ? delim : ""));
2903
2904 if (DO_BIC(BIC_IRQ)) {
2905 if (sums_need_wide_columns)
2906 outp += sprintf(outp, "%s IRQ", (printed++ ? delim : ""));
2907 else
2908 outp += sprintf(outp, "%sIRQ", (printed++ ? delim : ""));
2909 }
2910 if (DO_BIC(BIC_NMI)) {
2911 if (sums_need_wide_columns)
2912 outp += sprintf(outp, "%s NMI", (printed++ ? delim : ""));
2913 else
2914 outp += sprintf(outp, "%sNMI", (printed++ ? delim : ""));
2915 }
2916
2917 if (DO_BIC(BIC_SMI))
2918 outp += sprintf(outp, "%sSMI", (printed++ ? delim : ""));
2919
2920 if (DO_BIC(BIC_LLC_MRPS))
2921 outp += sprintf(outp, "%sLLCMRPS", (printed++ ? delim : ""));
2922
2923 if (DO_BIC(BIC_LLC_HIT))
2924 outp += sprintf(outp, "%sLLC%%hit", (printed++ ? delim : ""));
2925
2926 if (DO_BIC(BIC_L2_MRPS))
2927 outp += sprintf(outp, "%sL2MRPS", (printed++ ? delim : ""));
2928
2929 if (DO_BIC(BIC_L2_HIT))
2930 outp += sprintf(outp, "%sL2%%hit", (printed++ ? delim : ""));
2931
2932 for (mp = sys.tp; mp; mp = mp->next)
2933 outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format);
2934
2935 for (pp = sys.perf_tp; pp; pp = pp->next)
2936 outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format);
2937
2938 ppmt = sys.pmt_tp;
2939 while (ppmt) {
2940 switch (ppmt->type) {
2941 case PMT_TYPE_RAW:
2942 outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2943 break;
2944
2945 case PMT_TYPE_XTAL_TIME:
2946 case PMT_TYPE_TCORE_CLOCK:
2947 outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2948 break;
2949 }
2950
2951 ppmt = ppmt->next;
2952 }
2953
2954 if (DO_BIC(BIC_CPU_c1))
2955 outp += sprintf(outp, "%sCPU%%c1", (printed++ ? delim : ""));
2956 if (DO_BIC(BIC_CPU_c3))
2957 outp += sprintf(outp, "%sCPU%%c3", (printed++ ? delim : ""));
2958 if (DO_BIC(BIC_CPU_c6))
2959 outp += sprintf(outp, "%sCPU%%c6", (printed++ ? delim : ""));
2960 if (DO_BIC(BIC_CPU_c7))
2961 outp += sprintf(outp, "%sCPU%%c7", (printed++ ? delim : ""));
2962
2963 if (DO_BIC(BIC_Mod_c6))
2964 outp += sprintf(outp, "%sMod%%c6", (printed++ ? delim : ""));
2965
2966 if (DO_BIC(BIC_CoreTmp))
2967 outp += sprintf(outp, "%sCoreTmp", (printed++ ? delim : ""));
2968
2969 if (DO_BIC(BIC_CORE_THROT_CNT))
2970 outp += sprintf(outp, "%sCoreThr", (printed++ ? delim : ""));
2971
2972 if (valid_rapl_msrs && !rapl_joules) {
2973 if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
2974 outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : ""));
2975 } else if (valid_rapl_msrs && rapl_joules) {
2976 if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
2977 outp += sprintf(outp, "%sCor_J", (printed++ ? delim : ""));
2978 }
2979
2980 for (mp = sys.cp; mp; mp = mp->next)
2981 outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format);
2982
2983 for (pp = sys.perf_cp; pp; pp = pp->next)
2984 outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format);
2985
2986 ppmt = sys.pmt_cp;
2987 while (ppmt) {
2988 switch (ppmt->type) {
2989 case PMT_TYPE_RAW:
2990 outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2991
2992 break;
2993
2994 case PMT_TYPE_XTAL_TIME:
2995 case PMT_TYPE_TCORE_CLOCK:
2996 outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
2997 break;
2998 }
2999
3000 ppmt = ppmt->next;
3001 }
3002 if (DO_BIC(BIC_PkgTmp))
3003 outp += sprintf(outp, "%sPkgTmp", (printed++ ? delim : ""));
3004
3005 if (DO_BIC(BIC_GFX_rc6))
3006 outp += sprintf(outp, "%sGFX%%rc6", (printed++ ? delim : ""));
3007
3008 if (DO_BIC(BIC_GFXMHz))
3009 outp += sprintf(outp, "%sGFXMHz", (printed++ ? delim : ""));
3010
3011 if (DO_BIC(BIC_GFXACTMHz))
3012 outp += sprintf(outp, "%sGFXAMHz", (printed++ ? delim : ""));
3013
3014 if (DO_BIC(BIC_SAM_mc6))
3015 outp += sprintf(outp, "%sSAM%%mc6", (printed++ ? delim : ""));
3016
3017 if (DO_BIC(BIC_SAMMHz))
3018 outp += sprintf(outp, "%sSAMMHz", (printed++ ? delim : ""));
3019
3020 if (DO_BIC(BIC_SAMACTMHz))
3021 outp += sprintf(outp, "%sSAMAMHz", (printed++ ? delim : ""));
3022
3023 if (DO_BIC(BIC_Totl_c0))
3024 outp += sprintf(outp, "%sTotl%%C0", (printed++ ? delim : ""));
3025 if (DO_BIC(BIC_Any_c0))
3026 outp += sprintf(outp, "%sAny%%C0", (printed++ ? delim : ""));
3027 if (DO_BIC(BIC_GFX_c0))
3028 outp += sprintf(outp, "%sGFX%%C0", (printed++ ? delim : ""));
3029 if (DO_BIC(BIC_CPUGFX))
3030 outp += sprintf(outp, "%sCPUGFX%%", (printed++ ? delim : ""));
3031
3032 if (DO_BIC(BIC_Pkgpc2))
3033 outp += sprintf(outp, "%sPkg%%pc2", (printed++ ? delim : ""));
3034 if (DO_BIC(BIC_Pkgpc3))
3035 outp += sprintf(outp, "%sPkg%%pc3", (printed++ ? delim : ""));
3036 if (DO_BIC(BIC_Pkgpc6))
3037 outp += sprintf(outp, "%sPkg%%pc6", (printed++ ? delim : ""));
3038 if (DO_BIC(BIC_Pkgpc7))
3039 outp += sprintf(outp, "%sPkg%%pc7", (printed++ ? delim : ""));
3040 if (DO_BIC(BIC_Pkgpc8))
3041 outp += sprintf(outp, "%sPkg%%pc8", (printed++ ? delim : ""));
3042 if (DO_BIC(BIC_Pkgpc9))
3043 outp += sprintf(outp, "%sPkg%%pc9", (printed++ ? delim : ""));
3044 if (DO_BIC(BIC_Pkgpc10))
3045 outp += sprintf(outp, "%sPk%%pc10", (printed++ ? delim : ""));
3046 if (DO_BIC(BIC_Diec6))
3047 outp += sprintf(outp, "%sDie%%c6", (printed++ ? delim : ""));
3048 if (DO_BIC(BIC_CPU_LPI))
3049 outp += sprintf(outp, "%sCPU%%LPI", (printed++ ? delim : ""));
3050 if (DO_BIC(BIC_SYS_LPI))
3051 outp += sprintf(outp, "%sSYS%%LPI", (printed++ ? delim : ""));
3052
3053 if (!rapl_joules) {
3054 if (DO_BIC(BIC_PkgWatt))
3055 outp += sprintf(outp, "%sPkgWatt", (printed++ ? delim : ""));
3056 if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
3057 outp += sprintf(outp, "%sCorWatt", (printed++ ? delim : ""));
3058 if (DO_BIC(BIC_GFXWatt))
3059 outp += sprintf(outp, "%sGFXWatt", (printed++ ? delim : ""));
3060 if (DO_BIC(BIC_RAMWatt))
3061 outp += sprintf(outp, "%sRAMWatt", (printed++ ? delim : ""));
3062 if (DO_BIC(BIC_PKG__))
3063 outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
3064 if (DO_BIC(BIC_RAM__))
3065 outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
3066 } else {
3067 if (DO_BIC(BIC_Pkg_J))
3068 outp += sprintf(outp, "%sPkg_J", (printed++ ? delim : ""));
3069 if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
3070 outp += sprintf(outp, "%sCor_J", (printed++ ? delim : ""));
3071 if (DO_BIC(BIC_GFX_J))
3072 outp += sprintf(outp, "%sGFX_J", (printed++ ? delim : ""));
3073 if (DO_BIC(BIC_RAM_J))
3074 outp += sprintf(outp, "%sRAM_J", (printed++ ? delim : ""));
3075 if (DO_BIC(BIC_PKG__))
3076 outp += sprintf(outp, "%sPKG_%%", (printed++ ? delim : ""));
3077 if (DO_BIC(BIC_RAM__))
3078 outp += sprintf(outp, "%sRAM_%%", (printed++ ? delim : ""));
3079 }
3080 if (DO_BIC(BIC_UNCORE_MHZ))
3081 outp += sprintf(outp, "%sUncMHz", (printed++ ? delim : ""));
3082
3083 for (mp = sys.pp; mp; mp = mp->next)
3084 outp += print_name(mp->width, &printed, delim, mp->name, mp->type, mp->format);
3085
3086 for (pp = sys.perf_pp; pp; pp = pp->next)
3087 outp += print_name(pp->width, &printed, delim, pp->name, pp->type, pp->format);
3088
3089 ppmt = sys.pmt_pp;
3090 while (ppmt) {
3091 switch (ppmt->type) {
3092 case PMT_TYPE_RAW:
3093 outp += print_name(pmt_counter_get_width(ppmt), &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
3094 break;
3095
3096 case PMT_TYPE_XTAL_TIME:
3097 case PMT_TYPE_TCORE_CLOCK:
3098 outp += print_name(32, &printed, delim, ppmt->name, COUNTER_ITEMS, ppmt->format);
3099 break;
3100 }
3101
3102 ppmt = ppmt->next;
3103 }
3104
3105 if (DO_BIC(BIC_SysWatt))
3106 outp += sprintf(outp, "%sSysWatt", (printed++ ? delim : ""));
3107 if (DO_BIC(BIC_Sys_J))
3108 outp += sprintf(outp, "%sSys_J", (printed++ ? delim : ""));
3109
3110 outp += sprintf(outp, "\n");
3111 }
3112
3113 /*
3114 * pct(numerator, denominator)
3115 *
3116 * Return sanity checked percentage (100.0 * numerator/denominotor)
3117 *
3118 * n < 0: nan
3119 * d <= 0: nan
3120 * n/d > 1.1: nan
3121 */
pct(double numerator,double denominator)3122 double pct(double numerator, double denominator)
3123 {
3124 double retval;
3125
3126 if (numerator < 0)
3127 return nan("");
3128
3129 if (denominator <= 0)
3130 return nan("");
3131
3132 retval = 100.0 * numerator / denominator;
3133
3134 if (retval > 110.0)
3135 return nan("");
3136
3137 return retval;
3138 }
3139
dump_counters(PER_THREAD_PARAMS)3140 int dump_counters(PER_THREAD_PARAMS)
3141 {
3142 int i;
3143 struct msr_counter *mp;
3144 struct platform_counters *pplat_cnt = p == odd.packages ? &platform_counters_odd : &platform_counters_even;
3145
3146 outp += sprintf(outp, "t %p, c %p, p %p\n", t, c, p);
3147
3148 if (t) {
3149 outp += sprintf(outp, "CPU: %d flags 0x%x\n", t->cpu_id, t->flags);
3150 outp += sprintf(outp, "TSC: %016llX\n", t->tsc);
3151 outp += sprintf(outp, "aperf: %016llX\n", t->aperf);
3152 outp += sprintf(outp, "mperf: %016llX\n", t->mperf);
3153 outp += sprintf(outp, "c1: %016llX\n", t->c1);
3154
3155 if (DO_BIC(BIC_IPC))
3156 outp += sprintf(outp, "IPC: %lld\n", t->instr_count);
3157
3158 if (DO_BIC(BIC_IRQ))
3159 outp += sprintf(outp, "IRQ: %lld\n", t->irq_count);
3160 if (DO_BIC(BIC_NMI))
3161 outp += sprintf(outp, "IRQ: %lld\n", t->nmi_count);
3162 if (DO_BIC(BIC_SMI))
3163 outp += sprintf(outp, "SMI: %d\n", t->smi_count);
3164
3165 outp += sprintf(outp, "LLC refs: %lld", t->llc.references);
3166 outp += sprintf(outp, "LLC miss: %lld", t->llc.misses);
3167 outp += sprintf(outp, "LLC Hit%%: %.2f", pct((t->llc.references - t->llc.misses), t->llc.references));
3168
3169 outp += sprintf(outp, "L2 refs: %lld", t->l2.references);
3170 outp += sprintf(outp, "L2 hits: %lld", t->l2.hits);
3171 outp += sprintf(outp, "L2 Hit%%: %.2f", pct(t->l2.hits, t->l2.references));
3172
3173 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3174 outp += sprintf(outp, "tADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, t->counter[i], mp->sp->path);
3175 }
3176 }
3177
3178 if (c && is_cpu_first_thread_in_core(t, c)) {
3179 outp += sprintf(outp, "core: %d\n", cpus[t->cpu_id].core_id);
3180 outp += sprintf(outp, "c3: %016llX\n", c->c3);
3181 outp += sprintf(outp, "c6: %016llX\n", c->c6);
3182 outp += sprintf(outp, "c7: %016llX\n", c->c7);
3183 outp += sprintf(outp, "DTS: %dC\n", c->core_temp_c);
3184 outp += sprintf(outp, "cpu_throt_count: %016llX\n", c->core_throt_cnt);
3185
3186 const unsigned long long energy_value = c->core_energy.raw_value * c->core_energy.scale;
3187 const double energy_scale = c->core_energy.scale;
3188
3189 if (c->core_energy.unit == RAPL_UNIT_JOULES)
3190 outp += sprintf(outp, "Joules: %0llX (scale: %lf)\n", energy_value, energy_scale);
3191
3192 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3193 outp += sprintf(outp, "cADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, c->counter[i], mp->sp->path);
3194 }
3195 outp += sprintf(outp, "mc6_us: %016llX\n", c->mc6_us);
3196 }
3197
3198 if (p && is_cpu_first_core_in_package(t, p)) {
3199 outp += sprintf(outp, "Weighted cores: %016llX\n", p->pkg_wtd_core_c0);
3200 outp += sprintf(outp, "Any cores: %016llX\n", p->pkg_any_core_c0);
3201 outp += sprintf(outp, "Any GFX: %016llX\n", p->pkg_any_gfxe_c0);
3202 outp += sprintf(outp, "CPU + GFX: %016llX\n", p->pkg_both_core_gfxe_c0);
3203
3204 outp += sprintf(outp, "pc2: %016llX\n", p->pc2);
3205 if (DO_BIC(BIC_Pkgpc3))
3206 outp += sprintf(outp, "pc3: %016llX\n", p->pc3);
3207 if (DO_BIC(BIC_Pkgpc6))
3208 outp += sprintf(outp, "pc6: %016llX\n", p->pc6);
3209 if (DO_BIC(BIC_Pkgpc7))
3210 outp += sprintf(outp, "pc7: %016llX\n", p->pc7);
3211 outp += sprintf(outp, "pc8: %016llX\n", p->pc8);
3212 outp += sprintf(outp, "pc9: %016llX\n", p->pc9);
3213 outp += sprintf(outp, "pc10: %016llX\n", p->pc10);
3214 outp += sprintf(outp, "cpu_lpi: %016llX\n", p->cpu_lpi);
3215 outp += sprintf(outp, "sys_lpi: %016llX\n", p->sys_lpi);
3216 outp += sprintf(outp, "Joules PKG: %0llX\n", p->energy_pkg.raw_value);
3217 outp += sprintf(outp, "Joules COR: %0llX\n", p->energy_cores.raw_value);
3218 outp += sprintf(outp, "Joules GFX: %0llX\n", p->energy_gfx.raw_value);
3219 outp += sprintf(outp, "Joules RAM: %0llX\n", p->energy_dram.raw_value);
3220 outp += sprintf(outp, "Joules PSYS: %0llX\n", pplat_cnt->energy_psys.raw_value);
3221 outp += sprintf(outp, "Throttle PKG: %0llX\n", p->rapl_pkg_perf_status.raw_value);
3222 outp += sprintf(outp, "Throttle RAM: %0llX\n", p->rapl_dram_perf_status.raw_value);
3223 outp += sprintf(outp, "PTM: %dC\n", p->pkg_temp_c);
3224
3225 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3226 outp += sprintf(outp, "pADDED [%d] %8s msr0x%x: %08llX %s\n", i, mp->name, mp->msr_num, p->counter[i], mp->sp->path);
3227 }
3228 }
3229
3230 outp += sprintf(outp, "\n");
3231
3232 return 0;
3233 }
3234
rapl_counter_get_value(const struct rapl_counter * c,enum rapl_unit desired_unit,double interval)3235 double rapl_counter_get_value(const struct rapl_counter *c, enum rapl_unit desired_unit, double interval)
3236 {
3237 assert(desired_unit != RAPL_UNIT_INVALID);
3238
3239 /*
3240 * For now we don't expect anything other than joules,
3241 * so just simplify the logic.
3242 */
3243 assert(c->unit == RAPL_UNIT_JOULES);
3244
3245 const double scaled = c->raw_value * c->scale;
3246
3247 if (desired_unit == RAPL_UNIT_WATTS)
3248 return scaled / interval;
3249 return scaled;
3250 }
3251
get_perf_llc_stats(int cpu,struct llc_stats * llc)3252 void get_perf_llc_stats(int cpu, struct llc_stats *llc)
3253 {
3254 struct read_format {
3255 unsigned long long num_read;
3256 struct llc_stats llc;
3257 } r;
3258 const ssize_t expected_read_size = sizeof(r);
3259 ssize_t actual_read_size;
3260
3261 actual_read_size = read(fd_llc_percpu[cpu], &r, expected_read_size);
3262
3263 if (actual_read_size == -1)
3264 err(-1, "%s(cpu%d,) %d,,%ld", __func__, cpu, fd_llc_percpu[cpu], expected_read_size);
3265
3266 llc->references = r.llc.references;
3267 llc->misses = r.llc.misses;
3268 if (actual_read_size != expected_read_size)
3269 warn("%s: failed to read perf_data (req %zu act %zu)", __func__, expected_read_size, actual_read_size);
3270 }
3271
get_perf_l2_stats(int cpu,struct l2_stats * l2)3272 void get_perf_l2_stats(int cpu, struct l2_stats *l2)
3273 {
3274 struct read_format {
3275 unsigned long long num_read;
3276 struct l2_stats l2;
3277 } r;
3278 const ssize_t expected_read_size = sizeof(r);
3279 ssize_t actual_read_size;
3280
3281 actual_read_size = read(fd_l2_percpu[cpu], &r, expected_read_size);
3282
3283 if (actual_read_size == -1)
3284 err(-1, "%s(cpu%d,) %d,,%ld", __func__, cpu, fd_l2_percpu[cpu], expected_read_size);
3285
3286 l2->references = r.l2.references;
3287 l2->hits = r.l2.hits;
3288 if (actual_read_size != expected_read_size)
3289 warn("%s: cpu%d: failed to read(%d) perf_data (req %zu act %zu)", __func__, cpu, fd_l2_percpu[cpu], expected_read_size, actual_read_size);
3290 }
3291
3292 /*
3293 * column formatting convention & formats
3294 */
format_counters(PER_THREAD_PARAMS)3295 int format_counters(PER_THREAD_PARAMS)
3296 {
3297 static int count;
3298
3299 struct platform_counters *pplat_cnt = NULL;
3300 double interval_float, tsc;
3301 char *fmt8 = "%s%.2f";
3302
3303 int i;
3304 struct msr_counter *mp;
3305 struct perf_counter_info *pp;
3306 struct pmt_counter *ppmt;
3307 char *delim = "\t";
3308 int printed = 0;
3309
3310 if (t == average.threads) {
3311 pplat_cnt = count & 1 ? &platform_counters_odd : &platform_counters_even;
3312 ++count;
3313 }
3314
3315 /* if showing only 1st thread in core and this isn't one, bail out */
3316 if (show_core_only && !is_cpu_first_thread_in_core(t, c))
3317 return 0;
3318
3319 /* if showing only 1st thread in pkg and this isn't one, bail out */
3320 if (show_pkg_only && !is_cpu_first_core_in_package(t, p))
3321 return 0;
3322
3323 /*if not summary line and --cpu is used */
3324 if ((t != average.threads) && (cpu_subset && !CPU_ISSET_S(t->cpu_id, cpu_subset_size, cpu_subset)))
3325 return 0;
3326
3327 if (DO_BIC(BIC_USEC)) {
3328 /* on each row, print how many usec each timestamp took to gather */
3329 struct timeval tv;
3330
3331 timersub(&t->tv_end, &t->tv_begin, &tv);
3332 outp += sprintf(outp, "%5ld\t", tv.tv_sec * 1000000 + tv.tv_usec);
3333 }
3334
3335 /* Time_Of_Day_Seconds: on each row, print sec.usec last timestamp taken */
3336 if (DO_BIC(BIC_TOD))
3337 outp += sprintf(outp, "%10ld.%06ld\t", t->tv_end.tv_sec, t->tv_end.tv_usec);
3338
3339 interval_float = t->tv_delta.tv_sec + t->tv_delta.tv_usec / 1000000.0;
3340
3341 tsc = t->tsc * tsc_tweak;
3342
3343 /* topo columns, print blanks on 1st (average) line */
3344 if (t == average.threads) {
3345 if (DO_BIC(BIC_Package))
3346 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3347 if (DO_BIC(BIC_Die))
3348 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3349 if (DO_BIC(BIC_L3))
3350 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3351 if (DO_BIC(BIC_Node))
3352 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3353 if (DO_BIC(BIC_Core))
3354 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3355 if (DO_BIC(BIC_CPU))
3356 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3357 if (DO_BIC(BIC_APIC))
3358 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3359 if (DO_BIC(BIC_X2APIC))
3360 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3361 } else {
3362 if (DO_BIC(BIC_Package)) {
3363 if (p)
3364 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].package_id);
3365 else
3366 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3367 }
3368 if (DO_BIC(BIC_Die)) {
3369 if (c)
3370 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].die_id);
3371 else
3372 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3373 }
3374 if (DO_BIC(BIC_L3)) {
3375 if (c)
3376 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].l3_id);
3377 else
3378 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3379 }
3380 if (DO_BIC(BIC_Node)) {
3381 if (t)
3382 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].physical_node_id);
3383 else
3384 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3385 }
3386 if (DO_BIC(BIC_Core)) {
3387 if (c)
3388 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), cpus[t->cpu_id].core_id);
3389 else
3390 outp += sprintf(outp, "%s-", (printed++ ? delim : ""));
3391 }
3392 if (DO_BIC(BIC_CPU))
3393 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->cpu_id);
3394 if (DO_BIC(BIC_APIC))
3395 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->apic_id);
3396 if (DO_BIC(BIC_X2APIC))
3397 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->x2apic_id);
3398 }
3399
3400 if (DO_BIC(BIC_Avg_MHz))
3401 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 / units * t->aperf / interval_float);
3402
3403 if (DO_BIC(BIC_Busy))
3404 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->mperf, tsc));
3405
3406 if (DO_BIC(BIC_Bzy_MHz)) {
3407 if (has_base_hz)
3408 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), base_hz / units * t->aperf / t->mperf);
3409 else
3410 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), tsc / units * t->aperf / t->mperf / interval_float);
3411 }
3412
3413 if (DO_BIC(BIC_TSC_MHz))
3414 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), 1.0 * t->tsc / units / interval_float);
3415
3416 if (DO_BIC(BIC_IPC))
3417 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 1.0 * t->instr_count / t->aperf);
3418
3419 /* IRQ */
3420 if (DO_BIC(BIC_IRQ)) {
3421 if (sums_need_wide_columns)
3422 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->irq_count);
3423 else
3424 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->irq_count);
3425 }
3426
3427 /* NMI */
3428 if (DO_BIC(BIC_NMI)) {
3429 if (sums_need_wide_columns)
3430 outp += sprintf(outp, "%s%8lld", (printed++ ? delim : ""), t->nmi_count);
3431 else
3432 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), t->nmi_count);
3433 }
3434
3435 /* SMI */
3436 if (DO_BIC(BIC_SMI))
3437 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), t->smi_count);
3438
3439 /* LLC Stats */
3440 if (DO_BIC(BIC_LLC_MRPS))
3441 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->llc.references / interval_float / 1000000);
3442
3443 if (DO_BIC(BIC_LLC_HIT))
3444 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct((t->llc.references - t->llc.misses), t->llc.references));
3445
3446 /* L2 Stats */
3447 if (DO_BIC(BIC_L2_MRPS))
3448 outp += sprintf(outp, "%s%.0f", (printed++ ? delim : ""), t->l2.references / interval_float / 1000000);
3449
3450 if (DO_BIC(BIC_L2_HIT))
3451 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), pct(t->l2.hits, t->l2.references));
3452
3453 /* Added Thread Counters */
3454 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
3455 if (mp->format == FORMAT_RAW)
3456 outp += print_hex_value(mp->width, &printed, delim, t->counter[i]);
3457 else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3458 outp += print_decimal_value(mp->width, &printed, delim, t->counter[i]);
3459 else if (mp->format == FORMAT_PERCENT) {
3460 if (mp->type == COUNTER_USEC)
3461 outp += print_float_value(&printed, delim, t->counter[i] / interval_float / 10000);
3462 else
3463 outp += print_float_value(&printed, delim, pct(t->counter[i], tsc));
3464 }
3465 }
3466
3467 /* Added perf Thread Counters */
3468 for (i = 0, pp = sys.perf_tp; pp; ++i, pp = pp->next) {
3469 if (pp->format == FORMAT_RAW)
3470 outp += print_hex_value(pp->width, &printed, delim, t->perf_counter[i]);
3471 else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE)
3472 outp += print_decimal_value(pp->width, &printed, delim, t->perf_counter[i]);
3473 else if (pp->format == FORMAT_PERCENT) {
3474 if (pp->type == COUNTER_USEC)
3475 outp += print_float_value(&printed, delim, t->perf_counter[i] / interval_float / 10000);
3476 else
3477 outp += print_float_value(&printed, delim, pct(t->perf_counter[i], tsc));
3478 }
3479 }
3480
3481 /* Added PMT Thread Counters */
3482 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
3483 const unsigned long value_raw = t->pmt_counter[i];
3484 double value_converted;
3485 switch (ppmt->type) {
3486 case PMT_TYPE_RAW:
3487 outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, t->pmt_counter[i]);
3488 break;
3489
3490 case PMT_TYPE_XTAL_TIME:
3491 value_converted = pct(value_raw / crystal_hz, interval_float);
3492 outp += print_float_value(&printed, delim, value_converted);
3493 break;
3494
3495 case PMT_TYPE_TCORE_CLOCK:
3496 value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float);
3497 outp += print_float_value(&printed, delim, value_converted);
3498 }
3499 }
3500
3501 /* C1 */
3502 if (DO_BIC(BIC_CPU_c1))
3503 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(t->c1, tsc));
3504
3505 /* print per-core data only for 1st thread in core */
3506 if (!is_cpu_first_thread_in_core(t, c))
3507 goto done;
3508
3509 if (DO_BIC(BIC_CPU_c3))
3510 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c3, tsc));
3511 if (DO_BIC(BIC_CPU_c6))
3512 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c6, tsc));
3513 if (DO_BIC(BIC_CPU_c7))
3514 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->c7, tsc));
3515
3516 /* Mod%c6 */
3517 if (DO_BIC(BIC_Mod_c6))
3518 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(c->mc6_us, tsc));
3519
3520 if (DO_BIC(BIC_CoreTmp))
3521 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), c->core_temp_c);
3522
3523 /* Core throttle count */
3524 if (DO_BIC(BIC_CORE_THROT_CNT))
3525 outp += sprintf(outp, "%s%lld", (printed++ ? delim : ""), c->core_throt_cnt);
3526
3527 /* Added Core Counters */
3528 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3529 if (mp->format == FORMAT_RAW)
3530 outp += print_hex_value(mp->width, &printed, delim, c->counter[i]);
3531 else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3532 outp += print_decimal_value(mp->width, &printed, delim, c->counter[i]);
3533 else if (mp->format == FORMAT_PERCENT)
3534 outp += print_float_value(&printed, delim, pct(c->counter[i], tsc));
3535 }
3536
3537 /* Added perf Core counters */
3538 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3539 if (pp->format == FORMAT_RAW)
3540 outp += print_hex_value(pp->width, &printed, delim, c->perf_counter[i]);
3541 else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE)
3542 outp += print_decimal_value(pp->width, &printed, delim, c->perf_counter[i]);
3543 else if (pp->format == FORMAT_PERCENT)
3544 outp += print_float_value(&printed, delim, pct(c->perf_counter[i], tsc));
3545 }
3546
3547 /* Added PMT Core counters */
3548 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3549 const unsigned long value_raw = c->pmt_counter[i];
3550 double value_converted;
3551 switch (ppmt->type) {
3552 case PMT_TYPE_RAW:
3553 outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, c->pmt_counter[i]);
3554 break;
3555
3556 case PMT_TYPE_XTAL_TIME:
3557 value_converted = pct(value_raw / crystal_hz, interval_float);
3558 outp += print_float_value(&printed, delim, value_converted);
3559 break;
3560
3561 case PMT_TYPE_TCORE_CLOCK:
3562 value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float);
3563 outp += print_float_value(&printed, delim, value_converted);
3564 }
3565 }
3566
3567 if (DO_BIC(BIC_CorWatt) && platform->has_per_core_rapl)
3568 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_WATTS, interval_float));
3569 if (DO_BIC(BIC_Cor_J) && platform->has_per_core_rapl)
3570 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&c->core_energy, RAPL_UNIT_JOULES, interval_float));
3571
3572 /* print per-package data only for 1st core in package */
3573 if (!is_cpu_first_core_in_package(t, p))
3574 goto done;
3575
3576 /* PkgTmp */
3577 if (DO_BIC(BIC_PkgTmp))
3578 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->pkg_temp_c);
3579
3580 /* GFXrc6 */
3581 if (DO_BIC(BIC_GFX_rc6)) {
3582 if (p->gfx_rc6_ms == -1) { /* detect GFX counter reset */
3583 outp += sprintf(outp, "%s**.**", (printed++ ? delim : ""));
3584 } else {
3585 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->gfx_rc6_ms / 10.0 / interval_float);
3586 }
3587 }
3588
3589 /* GFXMHz */
3590 if (DO_BIC(BIC_GFXMHz))
3591 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_mhz);
3592
3593 /* GFXACTMHz */
3594 if (DO_BIC(BIC_GFXACTMHz))
3595 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->gfx_act_mhz);
3596
3597 /* SAMmc6 */
3598 if (DO_BIC(BIC_SAM_mc6)) {
3599 if (p->sam_mc6_ms == -1) { /* detect GFX counter reset */
3600 outp += sprintf(outp, "%s**.**", (printed++ ? delim : ""));
3601 } else {
3602 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), p->sam_mc6_ms / 10.0 / interval_float);
3603 }
3604 }
3605
3606 /* SAMMHz */
3607 if (DO_BIC(BIC_SAMMHz))
3608 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_mhz);
3609
3610 /* SAMACTMHz */
3611 if (DO_BIC(BIC_SAMACTMHz))
3612 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->sam_act_mhz);
3613
3614 /* Totl%C0, Any%C0 GFX%C0 CPUGFX% */
3615 if (DO_BIC(BIC_Totl_c0))
3616 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), 100 * p->pkg_wtd_core_c0 / tsc); /* can exceed 100% */
3617 if (DO_BIC(BIC_Any_c0))
3618 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_core_c0, tsc));
3619 if (DO_BIC(BIC_GFX_c0))
3620 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_any_gfxe_c0, tsc));
3621 if (DO_BIC(BIC_CPUGFX))
3622 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pkg_both_core_gfxe_c0, tsc));
3623
3624 if (DO_BIC(BIC_Pkgpc2))
3625 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc2, tsc));
3626 if (DO_BIC(BIC_Pkgpc3))
3627 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc3, tsc));
3628 if (DO_BIC(BIC_Pkgpc6))
3629 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc6, tsc));
3630 if (DO_BIC(BIC_Pkgpc7))
3631 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc7, tsc));
3632 if (DO_BIC(BIC_Pkgpc8))
3633 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc8, tsc));
3634 if (DO_BIC(BIC_Pkgpc9))
3635 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc9, tsc));
3636 if (DO_BIC(BIC_Pkgpc10))
3637 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->pc10, tsc));
3638
3639 if (DO_BIC(BIC_Diec6))
3640 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->die_c6 / crystal_hz, interval_float));
3641
3642 if (DO_BIC(BIC_CPU_LPI)) {
3643 if (p->cpu_lpi >= 0)
3644 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->cpu_lpi / 1000000.0, interval_float));
3645 else
3646 outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
3647 }
3648 if (DO_BIC(BIC_SYS_LPI)) {
3649 if (p->sys_lpi >= 0)
3650 outp += sprintf(outp, "%s%.2f", (printed++ ? delim : ""), pct(p->sys_lpi / 1000000.0, interval_float));
3651 else
3652 outp += sprintf(outp, "%s(neg)", (printed++ ? delim : ""));
3653 }
3654
3655 if (DO_BIC(BIC_PkgWatt))
3656 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_WATTS, interval_float));
3657 if (DO_BIC(BIC_CorWatt) && !platform->has_per_core_rapl)
3658 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_WATTS, interval_float));
3659 if (DO_BIC(BIC_GFXWatt))
3660 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_WATTS, interval_float));
3661 if (DO_BIC(BIC_RAMWatt))
3662 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_WATTS, interval_float));
3663 if (DO_BIC(BIC_Pkg_J))
3664 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_pkg, RAPL_UNIT_JOULES, interval_float));
3665 if (DO_BIC(BIC_Cor_J) && !platform->has_per_core_rapl)
3666 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_cores, RAPL_UNIT_JOULES, interval_float));
3667 if (DO_BIC(BIC_GFX_J))
3668 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_gfx, RAPL_UNIT_JOULES, interval_float));
3669 if (DO_BIC(BIC_RAM_J))
3670 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->energy_dram, RAPL_UNIT_JOULES, interval_float));
3671 if (DO_BIC(BIC_PKG__))
3672 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_pkg_perf_status, RAPL_UNIT_WATTS, interval_float));
3673 if (DO_BIC(BIC_RAM__))
3674 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&p->rapl_dram_perf_status, RAPL_UNIT_WATTS, interval_float));
3675 /* UncMHz */
3676 if (DO_BIC(BIC_UNCORE_MHZ))
3677 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), p->uncore_mhz);
3678
3679 /* Added Package Counters */
3680 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3681 if (mp->format == FORMAT_RAW)
3682 outp += print_hex_value(mp->width, &printed, delim, p->counter[i]);
3683 else if (mp->type == COUNTER_K2M)
3684 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->counter[i] / 1000);
3685 else if (mp->format == FORMAT_DELTA || mp->format == FORMAT_AVERAGE)
3686 outp += print_decimal_value(mp->width, &printed, delim, p->counter[i]);
3687 else if (mp->format == FORMAT_PERCENT)
3688 outp += print_float_value(&printed, delim, pct(p->counter[i], tsc));
3689 }
3690
3691 /* Added perf Package Counters */
3692 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3693 if (pp->format == FORMAT_RAW)
3694 outp += print_hex_value(pp->width, &printed, delim, p->perf_counter[i]);
3695 else if (pp->type == COUNTER_K2M)
3696 outp += sprintf(outp, "%s%d", (printed++ ? delim : ""), (unsigned int)p->perf_counter[i] / 1000);
3697 else if (pp->format == FORMAT_DELTA || pp->format == FORMAT_AVERAGE)
3698 outp += print_decimal_value(pp->width, &printed, delim, p->perf_counter[i]);
3699 else if (pp->format == FORMAT_PERCENT)
3700 outp += print_float_value(&printed, delim, pct(p->perf_counter[i], tsc));
3701 }
3702
3703 /* Added PMT Package Counters */
3704 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3705 const unsigned long value_raw = p->pmt_counter[i];
3706 double value_converted;
3707 switch (ppmt->type) {
3708 case PMT_TYPE_RAW:
3709 outp += print_hex_value(pmt_counter_get_width(ppmt), &printed, delim, p->pmt_counter[i]);
3710 break;
3711
3712 case PMT_TYPE_XTAL_TIME:
3713 value_converted = pct(value_raw / crystal_hz, interval_float);
3714 outp += print_float_value(&printed, delim, value_converted);
3715 break;
3716
3717 case PMT_TYPE_TCORE_CLOCK:
3718 value_converted = pct(value_raw / tcore_clock_freq_hz, interval_float);
3719 outp += print_float_value(&printed, delim, value_converted);
3720 }
3721 }
3722
3723 if (DO_BIC(BIC_SysWatt) && (t == average.threads))
3724 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_WATTS, interval_float));
3725 if (DO_BIC(BIC_Sys_J) && (t == average.threads))
3726 outp += sprintf(outp, fmt8, (printed++ ? delim : ""), rapl_counter_get_value(&pplat_cnt->energy_psys, RAPL_UNIT_JOULES, interval_float));
3727
3728 done:
3729 if (*(outp - 1) != '\n')
3730 outp += sprintf(outp, "\n");
3731
3732 return 0;
3733 }
3734
flush_output_stdout(void)3735 void flush_output_stdout(void)
3736 {
3737 FILE *filep;
3738
3739 if (outf == stderr)
3740 filep = stdout;
3741 else
3742 filep = outf;
3743
3744 fputs(output_buffer, filep);
3745 fflush(filep);
3746
3747 outp = output_buffer;
3748 }
3749
flush_output_stderr(void)3750 void flush_output_stderr(void)
3751 {
3752 fputs(output_buffer, outf);
3753 fflush(outf);
3754 outp = output_buffer;
3755 }
3756
format_all_counters(PER_THREAD_PARAMS)3757 void format_all_counters(PER_THREAD_PARAMS)
3758 {
3759 static int count;
3760
3761 if ((!count || (header_iterations && !(count % header_iterations))) || !summary_only)
3762 print_header("\t");
3763
3764 format_counters(average.threads, average.cores, average.packages);
3765
3766 count++;
3767
3768 if (summary_only)
3769 return;
3770
3771 for_all_cpus(format_counters, t, c, p);
3772 }
3773
3774 #define DELTA_WRAP32(new, old) \
3775 old = ((((unsigned long long)new << 32) - ((unsigned long long)old << 32)) >> 32);
3776
delta_package(struct pkg_data * new,struct pkg_data * old)3777 int delta_package(struct pkg_data *new, struct pkg_data *old)
3778 {
3779 int i;
3780 struct msr_counter *mp;
3781 struct perf_counter_info *pp;
3782 struct pmt_counter *ppmt;
3783
3784 if (DO_BIC(BIC_Totl_c0))
3785 old->pkg_wtd_core_c0 = new->pkg_wtd_core_c0 - old->pkg_wtd_core_c0;
3786 if (DO_BIC(BIC_Any_c0))
3787 old->pkg_any_core_c0 = new->pkg_any_core_c0 - old->pkg_any_core_c0;
3788 if (DO_BIC(BIC_GFX_c0))
3789 old->pkg_any_gfxe_c0 = new->pkg_any_gfxe_c0 - old->pkg_any_gfxe_c0;
3790 if (DO_BIC(BIC_CPUGFX))
3791 old->pkg_both_core_gfxe_c0 = new->pkg_both_core_gfxe_c0 - old->pkg_both_core_gfxe_c0;
3792
3793 old->pc2 = new->pc2 - old->pc2;
3794 if (DO_BIC(BIC_Pkgpc3))
3795 old->pc3 = new->pc3 - old->pc3;
3796 if (DO_BIC(BIC_Pkgpc6))
3797 old->pc6 = new->pc6 - old->pc6;
3798 if (DO_BIC(BIC_Pkgpc7))
3799 old->pc7 = new->pc7 - old->pc7;
3800 old->pc8 = new->pc8 - old->pc8;
3801 old->pc9 = new->pc9 - old->pc9;
3802 old->pc10 = new->pc10 - old->pc10;
3803 old->die_c6 = new->die_c6 - old->die_c6;
3804 old->cpu_lpi = new->cpu_lpi - old->cpu_lpi;
3805 old->sys_lpi = new->sys_lpi - old->sys_lpi;
3806 old->pkg_temp_c = new->pkg_temp_c;
3807
3808 /* flag an error when rc6 counter resets/wraps */
3809 if (old->gfx_rc6_ms > new->gfx_rc6_ms)
3810 old->gfx_rc6_ms = -1;
3811 else
3812 old->gfx_rc6_ms = new->gfx_rc6_ms - old->gfx_rc6_ms;
3813
3814 old->uncore_mhz = new->uncore_mhz;
3815 old->gfx_mhz = new->gfx_mhz;
3816 old->gfx_act_mhz = new->gfx_act_mhz;
3817
3818 /* flag an error when mc6 counter resets/wraps */
3819 if (old->sam_mc6_ms > new->sam_mc6_ms)
3820 old->sam_mc6_ms = -1;
3821 else
3822 old->sam_mc6_ms = new->sam_mc6_ms - old->sam_mc6_ms;
3823
3824 old->sam_mhz = new->sam_mhz;
3825 old->sam_act_mhz = new->sam_act_mhz;
3826
3827 old->energy_pkg.raw_value = new->energy_pkg.raw_value - old->energy_pkg.raw_value;
3828 old->energy_cores.raw_value = new->energy_cores.raw_value - old->energy_cores.raw_value;
3829 old->energy_gfx.raw_value = new->energy_gfx.raw_value - old->energy_gfx.raw_value;
3830 old->energy_dram.raw_value = new->energy_dram.raw_value - old->energy_dram.raw_value;
3831 old->rapl_pkg_perf_status.raw_value = new->rapl_pkg_perf_status.raw_value - old->rapl_pkg_perf_status.raw_value;
3832 old->rapl_dram_perf_status.raw_value = new->rapl_dram_perf_status.raw_value - old->rapl_dram_perf_status.raw_value;
3833
3834 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
3835 if (mp->format == FORMAT_RAW)
3836 old->counter[i] = new->counter[i];
3837 else if (mp->format == FORMAT_AVERAGE)
3838 old->counter[i] = new->counter[i];
3839 else
3840 old->counter[i] = new->counter[i] - old->counter[i];
3841 }
3842
3843 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
3844 if (pp->format == FORMAT_RAW)
3845 old->perf_counter[i] = new->perf_counter[i];
3846 else if (pp->format == FORMAT_AVERAGE)
3847 old->perf_counter[i] = new->perf_counter[i];
3848 else
3849 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3850 }
3851
3852 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
3853 if (ppmt->format == FORMAT_RAW)
3854 old->pmt_counter[i] = new->pmt_counter[i];
3855 else
3856 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3857 }
3858
3859 return 0;
3860 }
3861
delta_core(struct core_data * new,struct core_data * old)3862 void delta_core(struct core_data *new, struct core_data *old)
3863 {
3864 int i;
3865 struct msr_counter *mp;
3866 struct perf_counter_info *pp;
3867 struct pmt_counter *ppmt;
3868
3869 old->c3 = new->c3 - old->c3;
3870 old->c6 = new->c6 - old->c6;
3871 old->c7 = new->c7 - old->c7;
3872 old->core_temp_c = new->core_temp_c;
3873 old->core_throt_cnt = new->core_throt_cnt - old->core_throt_cnt;
3874 old->mc6_us = new->mc6_us - old->mc6_us;
3875
3876 DELTA_WRAP32(new->core_energy.raw_value, old->core_energy.raw_value);
3877
3878 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
3879 if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE)
3880 old->counter[i] = new->counter[i];
3881 else
3882 old->counter[i] = new->counter[i] - old->counter[i];
3883 }
3884
3885 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
3886 if (pp->format == FORMAT_RAW)
3887 old->perf_counter[i] = new->perf_counter[i];
3888 else
3889 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
3890 }
3891
3892 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
3893 if (ppmt->format == FORMAT_RAW)
3894 old->pmt_counter[i] = new->pmt_counter[i];
3895 else
3896 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
3897 }
3898 }
3899
soft_c1_residency_display(int bic)3900 int soft_c1_residency_display(int bic)
3901 {
3902 if (!DO_BIC(BIC_CPU_c1) || platform->has_msr_core_c1_res)
3903 return 0;
3904
3905 return DO_BIC_READ(bic);
3906 }
3907
3908 /*
3909 * old = new - old
3910 */
delta_thread(struct thread_data * new,struct thread_data * old,struct core_data * core_delta)3911 int delta_thread(struct thread_data *new, struct thread_data *old, struct core_data *core_delta)
3912 {
3913 int i;
3914 struct msr_counter *mp;
3915 struct perf_counter_info *pp;
3916 struct pmt_counter *ppmt;
3917
3918 /* we run cpuid just the 1st time, copy the results */
3919 if (DO_BIC(BIC_APIC))
3920 new->apic_id = old->apic_id;
3921 if (DO_BIC(BIC_X2APIC))
3922 new->x2apic_id = old->x2apic_id;
3923
3924 /*
3925 * the timestamps from start of measurement interval are in "old"
3926 * the timestamp from end of measurement interval are in "new"
3927 * over-write old w/ new so we can print end of interval values
3928 */
3929
3930 timersub(&new->tv_begin, &old->tv_begin, &old->tv_delta);
3931 old->tv_begin = new->tv_begin;
3932 old->tv_end = new->tv_end;
3933
3934 old->tsc = new->tsc - old->tsc;
3935
3936 /* check for TSC < 1 Mcycles over interval */
3937 if (old->tsc < (1000 * 1000))
3938 errx(-3, "Insanely slow TSC rate, TSC stops in idle?\n"
3939 "You can disable all c-states by booting with \"idle=poll\"\nor just the deep ones with \"processor.max_cstate=1\"");
3940
3941 old->c1 = new->c1 - old->c1;
3942
3943 if (DO_BIC(BIC_Avg_MHz) || DO_BIC(BIC_Busy) || DO_BIC(BIC_Bzy_MHz) || DO_BIC(BIC_IPC)
3944 || soft_c1_residency_display(BIC_Avg_MHz)) {
3945 if ((new->aperf > old->aperf) && (new->mperf > old->mperf)) {
3946 old->aperf = new->aperf - old->aperf;
3947 old->mperf = new->mperf - old->mperf;
3948 } else {
3949 return -1;
3950 }
3951 }
3952
3953 if (platform->has_msr_core_c1_res) {
3954 /*
3955 * Some models have a dedicated C1 residency MSR,
3956 * which should be more accurate than the derivation below.
3957 */
3958 } else {
3959 /*
3960 * As counter collection is not atomic,
3961 * it is possible for mperf's non-halted cycles + idle states
3962 * to exceed TSC's all cycles: show c1 = 0% in that case.
3963 */
3964 if ((old->mperf + core_delta->c3 + core_delta->c6 + core_delta->c7) > (old->tsc * tsc_tweak))
3965 old->c1 = 0;
3966 else {
3967 /* normal case, derive c1 */
3968 old->c1 = (old->tsc * tsc_tweak) - old->mperf - core_delta->c3 - core_delta->c6 - core_delta->c7;
3969 }
3970 }
3971
3972 if (old->mperf == 0) {
3973 if (debug > 1)
3974 fprintf(outf, "cpu%d MPERF 0!\n", old->cpu_id);
3975 old->mperf = 1; /* divide by 0 protection */
3976 }
3977
3978 if (DO_BIC(BIC_IPC))
3979 old->instr_count = new->instr_count - old->instr_count;
3980
3981 if (DO_BIC(BIC_IRQ))
3982 old->irq_count = new->irq_count - old->irq_count;
3983
3984 if (DO_BIC(BIC_NMI))
3985 old->nmi_count = new->nmi_count - old->nmi_count;
3986
3987 if (DO_BIC(BIC_SMI))
3988 old->smi_count = new->smi_count - old->smi_count;
3989
3990 if (DO_BIC(BIC_LLC_MRPS) || DO_BIC(BIC_LLC_HIT))
3991 old->llc.references = new->llc.references - old->llc.references;
3992
3993 if (DO_BIC(BIC_LLC_HIT))
3994 old->llc.misses = new->llc.misses - old->llc.misses;
3995
3996 if (DO_BIC(BIC_L2_MRPS) || DO_BIC(BIC_L2_HIT))
3997 old->l2.references = new->l2.references - old->l2.references;
3998
3999 if (DO_BIC(BIC_L2_HIT))
4000 old->l2.hits = new->l2.hits - old->l2.hits;
4001
4002 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4003 if (mp->format == FORMAT_RAW || mp->format == FORMAT_AVERAGE)
4004 old->counter[i] = new->counter[i];
4005 else
4006 old->counter[i] = new->counter[i] - old->counter[i];
4007 }
4008
4009 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4010 if (pp->format == FORMAT_RAW)
4011 old->perf_counter[i] = new->perf_counter[i];
4012 else
4013 old->perf_counter[i] = new->perf_counter[i] - old->perf_counter[i];
4014 }
4015
4016 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4017 if (ppmt->format == FORMAT_RAW)
4018 old->pmt_counter[i] = new->pmt_counter[i];
4019 else
4020 old->pmt_counter[i] = new->pmt_counter[i] - old->pmt_counter[i];
4021 }
4022
4023 return 0;
4024 }
4025
delta_cpu(struct thread_data * t,struct core_data * c,struct pkg_data * p,struct thread_data * t2,struct core_data * c2,struct pkg_data * p2)4026 int delta_cpu(struct thread_data *t, struct core_data *c, struct pkg_data *p, struct thread_data *t2, struct core_data *c2, struct pkg_data *p2)
4027 {
4028 int retval = 0;
4029
4030 /* calculate core delta only for 1st thread in core */
4031 if (is_cpu_first_thread_in_core(t, c))
4032 delta_core(c, c2);
4033
4034 /* always calculate thread delta */
4035 retval = delta_thread(t, t2, c2); /* c2 is core delta */
4036
4037 /* calculate package delta only for 1st core in package */
4038 if (is_cpu_first_core_in_package(t, p))
4039 retval |= delta_package(p, p2);
4040
4041 return retval;
4042 }
4043
delta_platform(struct platform_counters * new,struct platform_counters * old)4044 void delta_platform(struct platform_counters *new, struct platform_counters *old)
4045 {
4046 old->energy_psys.raw_value = new->energy_psys.raw_value - old->energy_psys.raw_value;
4047 }
4048
rapl_counter_clear(struct rapl_counter * c)4049 void rapl_counter_clear(struct rapl_counter *c)
4050 {
4051 c->raw_value = 0;
4052 c->scale = 0.0;
4053 c->unit = RAPL_UNIT_INVALID;
4054 }
4055
clear_counters(PER_THREAD_PARAMS)4056 void clear_counters(PER_THREAD_PARAMS)
4057 {
4058 int i;
4059 struct msr_counter *mp;
4060
4061 t->tv_begin.tv_sec = 0;
4062 t->tv_begin.tv_usec = 0;
4063 t->tv_end.tv_sec = 0;
4064 t->tv_end.tv_usec = 0;
4065 t->tv_delta.tv_sec = 0;
4066 t->tv_delta.tv_usec = 0;
4067
4068 t->tsc = 0;
4069 t->aperf = 0;
4070 t->mperf = 0;
4071 t->c1 = 0;
4072
4073 t->instr_count = 0;
4074
4075 t->irq_count = 0;
4076 t->nmi_count = 0;
4077 t->smi_count = 0;
4078
4079 t->llc.references = 0;
4080 t->llc.misses = 0;
4081
4082 t->l2.references = 0;
4083 t->l2.hits = 0;
4084
4085 c->c3 = 0;
4086 c->c6 = 0;
4087 c->c7 = 0;
4088 c->mc6_us = 0;
4089 c->core_temp_c = 0;
4090 rapl_counter_clear(&c->core_energy);
4091 c->core_throt_cnt = 0;
4092
4093 p->pkg_wtd_core_c0 = 0;
4094 p->pkg_any_core_c0 = 0;
4095 p->pkg_any_gfxe_c0 = 0;
4096 p->pkg_both_core_gfxe_c0 = 0;
4097
4098 p->pc2 = 0;
4099 if (DO_BIC(BIC_Pkgpc3))
4100 p->pc3 = 0;
4101 if (DO_BIC(BIC_Pkgpc6))
4102 p->pc6 = 0;
4103 if (DO_BIC(BIC_Pkgpc7))
4104 p->pc7 = 0;
4105 p->pc8 = 0;
4106 p->pc9 = 0;
4107 p->pc10 = 0;
4108 p->die_c6 = 0;
4109 p->cpu_lpi = 0;
4110 p->sys_lpi = 0;
4111
4112 rapl_counter_clear(&p->energy_pkg);
4113 rapl_counter_clear(&p->energy_dram);
4114 rapl_counter_clear(&p->energy_cores);
4115 rapl_counter_clear(&p->energy_gfx);
4116 rapl_counter_clear(&p->rapl_pkg_perf_status);
4117 rapl_counter_clear(&p->rapl_dram_perf_status);
4118 p->pkg_temp_c = 0;
4119
4120 p->gfx_rc6_ms = 0;
4121 p->uncore_mhz = 0;
4122 p->gfx_mhz = 0;
4123 p->gfx_act_mhz = 0;
4124 p->sam_mc6_ms = 0;
4125 p->sam_mhz = 0;
4126 p->sam_act_mhz = 0;
4127 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next)
4128 t->counter[i] = 0;
4129
4130 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next)
4131 c->counter[i] = 0;
4132
4133 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next)
4134 p->counter[i] = 0;
4135
4136 memset(&t->perf_counter[0], 0, sizeof(t->perf_counter));
4137 memset(&c->perf_counter[0], 0, sizeof(c->perf_counter));
4138 memset(&p->perf_counter[0], 0, sizeof(p->perf_counter));
4139
4140 memset(&t->pmt_counter[0], 0, ARRAY_SIZE(t->pmt_counter));
4141 memset(&c->pmt_counter[0], 0, ARRAY_SIZE(c->pmt_counter));
4142 memset(&p->pmt_counter[0], 0, ARRAY_SIZE(p->pmt_counter));
4143 }
4144
rapl_counter_accumulate(struct rapl_counter * dst,const struct rapl_counter * src)4145 void rapl_counter_accumulate(struct rapl_counter *dst, const struct rapl_counter *src)
4146 {
4147 /* Copy unit and scale from src if dst is not initialized */
4148 if (dst->unit == RAPL_UNIT_INVALID) {
4149 dst->unit = src->unit;
4150 dst->scale = src->scale;
4151 }
4152
4153 assert(dst->unit == src->unit);
4154 assert(dst->scale == src->scale);
4155
4156 dst->raw_value += src->raw_value;
4157 }
4158
sum_counters(PER_THREAD_PARAMS)4159 int sum_counters(PER_THREAD_PARAMS)
4160 {
4161 int i;
4162 struct msr_counter *mp;
4163 struct perf_counter_info *pp;
4164 struct pmt_counter *ppmt;
4165
4166 /* copy un-changing apic_id's */
4167 if (DO_BIC(BIC_APIC))
4168 average.threads->apic_id = t->apic_id;
4169 if (DO_BIC(BIC_X2APIC))
4170 average.threads->x2apic_id = t->x2apic_id;
4171
4172 /* remember first tv_begin */
4173 if (average.threads->tv_begin.tv_sec == 0)
4174 average.threads->tv_begin = procsysfs_tv_begin;
4175
4176 /* remember last tv_end */
4177 average.threads->tv_end = t->tv_end;
4178
4179 average.threads->tsc += t->tsc;
4180 average.threads->aperf += t->aperf;
4181 average.threads->mperf += t->mperf;
4182 average.threads->c1 += t->c1;
4183
4184 average.threads->instr_count += t->instr_count;
4185
4186 average.threads->irq_count += t->irq_count;
4187 average.threads->nmi_count += t->nmi_count;
4188 average.threads->smi_count += t->smi_count;
4189
4190 average.threads->llc.references += t->llc.references;
4191 average.threads->llc.misses += t->llc.misses;
4192
4193 average.threads->l2.references += t->l2.references;
4194 average.threads->l2.hits += t->l2.hits;
4195
4196 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4197 if (mp->format == FORMAT_RAW)
4198 continue;
4199 average.threads->counter[i] += t->counter[i];
4200 }
4201
4202 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4203 if (pp->format == FORMAT_RAW)
4204 continue;
4205 average.threads->perf_counter[i] += t->perf_counter[i];
4206 }
4207
4208 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4209 average.threads->pmt_counter[i] += t->pmt_counter[i];
4210 }
4211
4212 /* sum per-core values only for 1st thread in core */
4213 if (!is_cpu_first_thread_in_core(t, c))
4214 return 0;
4215
4216 average.cores->c3 += c->c3;
4217 average.cores->c6 += c->c6;
4218 average.cores->c7 += c->c7;
4219 average.cores->mc6_us += c->mc6_us;
4220
4221 average.cores->core_temp_c = MAX(average.cores->core_temp_c, c->core_temp_c);
4222 average.cores->core_throt_cnt = MAX(average.cores->core_throt_cnt, c->core_throt_cnt);
4223
4224 rapl_counter_accumulate(&average.cores->core_energy, &c->core_energy);
4225
4226 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
4227 if (mp->format == FORMAT_RAW)
4228 continue;
4229 average.cores->counter[i] += c->counter[i];
4230 }
4231
4232 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
4233 if (pp->format == FORMAT_RAW)
4234 continue;
4235 average.cores->perf_counter[i] += c->perf_counter[i];
4236 }
4237
4238 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
4239 average.cores->pmt_counter[i] += c->pmt_counter[i];
4240 }
4241
4242 /* sum per-pkg values only for 1st core in pkg */
4243 if (!is_cpu_first_core_in_package(t, p))
4244 return 0;
4245
4246 if (DO_BIC(BIC_Totl_c0))
4247 average.packages->pkg_wtd_core_c0 += p->pkg_wtd_core_c0;
4248 if (DO_BIC(BIC_Any_c0))
4249 average.packages->pkg_any_core_c0 += p->pkg_any_core_c0;
4250 if (DO_BIC(BIC_GFX_c0))
4251 average.packages->pkg_any_gfxe_c0 += p->pkg_any_gfxe_c0;
4252 if (DO_BIC(BIC_CPUGFX))
4253 average.packages->pkg_both_core_gfxe_c0 += p->pkg_both_core_gfxe_c0;
4254
4255 average.packages->pc2 += p->pc2;
4256 if (DO_BIC(BIC_Pkgpc3))
4257 average.packages->pc3 += p->pc3;
4258 if (DO_BIC(BIC_Pkgpc6))
4259 average.packages->pc6 += p->pc6;
4260 if (DO_BIC(BIC_Pkgpc7))
4261 average.packages->pc7 += p->pc7;
4262 average.packages->pc8 += p->pc8;
4263 average.packages->pc9 += p->pc9;
4264 average.packages->pc10 += p->pc10;
4265 average.packages->die_c6 += p->die_c6;
4266
4267 average.packages->cpu_lpi = p->cpu_lpi;
4268 average.packages->sys_lpi = p->sys_lpi;
4269
4270 rapl_counter_accumulate(&average.packages->energy_pkg, &p->energy_pkg);
4271 rapl_counter_accumulate(&average.packages->energy_dram, &p->energy_dram);
4272 rapl_counter_accumulate(&average.packages->energy_cores, &p->energy_cores);
4273 rapl_counter_accumulate(&average.packages->energy_gfx, &p->energy_gfx);
4274
4275 average.packages->gfx_rc6_ms = p->gfx_rc6_ms;
4276 average.packages->uncore_mhz = p->uncore_mhz;
4277 average.packages->gfx_mhz = p->gfx_mhz;
4278 average.packages->gfx_act_mhz = p->gfx_act_mhz;
4279 average.packages->sam_mc6_ms = p->sam_mc6_ms;
4280 average.packages->sam_mhz = p->sam_mhz;
4281 average.packages->sam_act_mhz = p->sam_act_mhz;
4282
4283 average.packages->pkg_temp_c = MAX(average.packages->pkg_temp_c, p->pkg_temp_c);
4284
4285 rapl_counter_accumulate(&average.packages->rapl_pkg_perf_status, &p->rapl_pkg_perf_status);
4286 rapl_counter_accumulate(&average.packages->rapl_dram_perf_status, &p->rapl_dram_perf_status);
4287
4288 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
4289 if ((mp->format == FORMAT_RAW) && (topo.num_packages == 0))
4290 average.packages->counter[i] = p->counter[i];
4291 else
4292 average.packages->counter[i] += p->counter[i];
4293 }
4294
4295 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
4296 if ((pp->format == FORMAT_RAW) && (topo.num_packages == 0))
4297 average.packages->perf_counter[i] = p->perf_counter[i];
4298 else
4299 average.packages->perf_counter[i] += p->perf_counter[i];
4300 }
4301
4302 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
4303 average.packages->pmt_counter[i] += p->pmt_counter[i];
4304 }
4305
4306 return 0;
4307 }
4308
4309 /*
4310 * sum the counters for all cpus in the system
4311 * compute the weighted average
4312 */
compute_average(PER_THREAD_PARAMS)4313 void compute_average(PER_THREAD_PARAMS)
4314 {
4315 int i;
4316 struct msr_counter *mp;
4317 struct perf_counter_info *pp;
4318 struct pmt_counter *ppmt;
4319
4320 clear_counters(average.threads, average.cores, average.packages);
4321
4322 for_all_cpus(sum_counters, t, c, p);
4323
4324 /* Use the global time delta for the average. */
4325 average.threads->tv_delta = tv_delta;
4326
4327 average.threads->tsc /= topo.allowed_cpus;
4328 average.threads->aperf /= topo.allowed_cpus;
4329 average.threads->mperf /= topo.allowed_cpus;
4330 average.threads->instr_count /= topo.allowed_cpus;
4331 average.threads->c1 /= topo.allowed_cpus;
4332
4333 if (average.threads->irq_count > 9999999)
4334 sums_need_wide_columns = 1;
4335 if (average.threads->nmi_count > 9999999)
4336 sums_need_wide_columns = 1;
4337
4338 average.cores->c3 /= topo.allowed_cores;
4339 average.cores->c6 /= topo.allowed_cores;
4340 average.cores->c7 /= topo.allowed_cores;
4341 average.cores->mc6_us /= topo.allowed_cores;
4342
4343 if (DO_BIC(BIC_Totl_c0))
4344 average.packages->pkg_wtd_core_c0 /= topo.allowed_packages;
4345 if (DO_BIC(BIC_Any_c0))
4346 average.packages->pkg_any_core_c0 /= topo.allowed_packages;
4347 if (DO_BIC(BIC_GFX_c0))
4348 average.packages->pkg_any_gfxe_c0 /= topo.allowed_packages;
4349 if (DO_BIC(BIC_CPUGFX))
4350 average.packages->pkg_both_core_gfxe_c0 /= topo.allowed_packages;
4351
4352 average.packages->pc2 /= topo.allowed_packages;
4353 if (DO_BIC(BIC_Pkgpc3))
4354 average.packages->pc3 /= topo.allowed_packages;
4355 if (DO_BIC(BIC_Pkgpc6))
4356 average.packages->pc6 /= topo.allowed_packages;
4357 if (DO_BIC(BIC_Pkgpc7))
4358 average.packages->pc7 /= topo.allowed_packages;
4359
4360 average.packages->pc8 /= topo.allowed_packages;
4361 average.packages->pc9 /= topo.allowed_packages;
4362 average.packages->pc10 /= topo.allowed_packages;
4363 average.packages->die_c6 /= topo.allowed_packages;
4364
4365 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
4366 if (mp->format == FORMAT_RAW)
4367 continue;
4368 if (mp->type == COUNTER_ITEMS) {
4369 if (average.threads->counter[i] > 9999999)
4370 sums_need_wide_columns = 1;
4371 continue;
4372 }
4373 average.threads->counter[i] /= topo.allowed_cpus;
4374 }
4375 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
4376 if (mp->format == FORMAT_RAW)
4377 continue;
4378 if (mp->type == COUNTER_ITEMS) {
4379 if (average.cores->counter[i] > 9999999)
4380 sums_need_wide_columns = 1;
4381 }
4382 average.cores->counter[i] /= topo.allowed_cores;
4383 }
4384 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
4385 if (mp->format == FORMAT_RAW)
4386 continue;
4387 if (mp->type == COUNTER_ITEMS) {
4388 if (average.packages->counter[i] > 9999999)
4389 sums_need_wide_columns = 1;
4390 }
4391 average.packages->counter[i] /= topo.allowed_packages;
4392 }
4393
4394 for (i = 0, pp = sys.perf_tp; pp; i++, pp = pp->next) {
4395 if (pp->format == FORMAT_RAW)
4396 continue;
4397 if (pp->type == COUNTER_ITEMS) {
4398 if (average.threads->perf_counter[i] > 9999999)
4399 sums_need_wide_columns = 1;
4400 continue;
4401 }
4402 average.threads->perf_counter[i] /= topo.allowed_cpus;
4403 }
4404 for (i = 0, pp = sys.perf_cp; pp; i++, pp = pp->next) {
4405 if (pp->format == FORMAT_RAW)
4406 continue;
4407 if (pp->type == COUNTER_ITEMS) {
4408 if (average.cores->perf_counter[i] > 9999999)
4409 sums_need_wide_columns = 1;
4410 }
4411 average.cores->perf_counter[i] /= topo.allowed_cores;
4412 }
4413 for (i = 0, pp = sys.perf_pp; pp; i++, pp = pp->next) {
4414 if (pp->format == FORMAT_RAW)
4415 continue;
4416 if (pp->type == COUNTER_ITEMS) {
4417 if (average.packages->perf_counter[i] > 9999999)
4418 sums_need_wide_columns = 1;
4419 }
4420 average.packages->perf_counter[i] /= topo.allowed_packages;
4421 }
4422
4423 for (i = 0, ppmt = sys.pmt_tp; ppmt; i++, ppmt = ppmt->next) {
4424 average.threads->pmt_counter[i] /= topo.allowed_cpus;
4425 }
4426 for (i = 0, ppmt = sys.pmt_cp; ppmt; i++, ppmt = ppmt->next) {
4427 average.cores->pmt_counter[i] /= topo.allowed_cores;
4428 }
4429 for (i = 0, ppmt = sys.pmt_pp; ppmt; i++, ppmt = ppmt->next) {
4430 average.packages->pmt_counter[i] /= topo.allowed_packages;
4431 }
4432 }
4433
rdtsc(void)4434 static unsigned long long rdtsc(void)
4435 {
4436 unsigned int low, high;
4437
4438 asm volatile ("rdtsc":"=a" (low), "=d"(high));
4439
4440 return low | ((unsigned long long)high) << 32;
4441 }
4442
4443 /*
4444 * Open a file, and exit on failure
4445 */
fopen_or_die(const char * path,const char * mode)4446 FILE *fopen_or_die(const char *path, const char *mode)
4447 {
4448 FILE *filep = fopen(path, mode);
4449
4450 if (!filep)
4451 err(1, "%s: open failed", path);
4452 return filep;
4453 }
4454
4455 /*
4456 * snapshot_sysfs_counter()
4457 *
4458 * return snapshot of given counter
4459 */
snapshot_sysfs_counter(char * path)4460 unsigned long long snapshot_sysfs_counter(char *path)
4461 {
4462 FILE *fp;
4463 int retval;
4464 unsigned long long counter;
4465
4466 fp = fopen_or_die(path, "r");
4467
4468 retval = fscanf(fp, "%lld", &counter);
4469 if (retval != 1)
4470 err(1, "snapshot_sysfs_counter(%s)", path);
4471
4472 fclose(fp);
4473
4474 return counter;
4475 }
4476
get_mp(int cpu,struct msr_counter * mp,unsigned long long * counterp,char * counter_path)4477 int get_mp(int cpu, struct msr_counter *mp, unsigned long long *counterp, char *counter_path)
4478 {
4479 if (mp->msr_num != 0) {
4480 assert(!no_msr);
4481 if (get_msr(cpu, mp->msr_num, counterp))
4482 return -1;
4483 } else {
4484 char path[128 + PATH_BYTES];
4485
4486 if (mp->flags & SYSFS_PERCPU) {
4487 sprintf(path, "/sys/devices/system/cpu/cpu%d/%s", cpu, mp->sp->path);
4488
4489 *counterp = snapshot_sysfs_counter(path);
4490 } else {
4491 *counterp = snapshot_sysfs_counter(counter_path);
4492 }
4493 }
4494
4495 return 0;
4496 }
4497
get_legacy_uncore_mhz(int package)4498 unsigned long long get_legacy_uncore_mhz(int package)
4499 {
4500 char path[128];
4501 int die;
4502 static int warn_once;
4503
4504 /*
4505 * for this package, use the first die_id that exists
4506 */
4507 for (die = 0; die <= topo.max_die_id; ++die) {
4508
4509 sprintf(path, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d/current_freq_khz", package, die);
4510
4511 if (access(path, R_OK) == 0)
4512 return (snapshot_sysfs_counter(path) / 1000);
4513 }
4514 if (!warn_once) {
4515 warnx("BUG: %s: No %s", __func__, path);
4516 warn_once = 1;
4517 }
4518
4519 return 0;
4520 }
4521
get_epb(int cpu)4522 int get_epb(int cpu)
4523 {
4524 char path[128 + PATH_BYTES];
4525 unsigned long long msr;
4526 int ret, epb = -1;
4527 FILE *fp;
4528
4529 sprintf(path, "/sys/devices/system/cpu/cpu%d/power/energy_perf_bias", cpu);
4530
4531 fp = fopen(path, "r");
4532 if (!fp)
4533 goto msr_fallback;
4534
4535 ret = fscanf(fp, "%d", &epb);
4536 if (ret != 1)
4537 err(1, "%s(%s)", __func__, path);
4538
4539 fclose(fp);
4540
4541 return epb;
4542
4543 msr_fallback:
4544 if (no_msr)
4545 return -1;
4546
4547 get_msr(cpu, MSR_IA32_ENERGY_PERF_BIAS, &msr);
4548
4549 return msr & 0xf;
4550 }
4551
get_apic_id(struct thread_data * t)4552 void get_apic_id(struct thread_data *t)
4553 {
4554 unsigned int eax, ebx, ecx, edx;
4555
4556 if (DO_BIC(BIC_APIC)) {
4557 eax = ebx = ecx = edx = 0;
4558 __cpuid(1, eax, ebx, ecx, edx);
4559
4560 t->apic_id = (ebx >> 24) & 0xff;
4561 }
4562
4563 if (!DO_BIC(BIC_X2APIC))
4564 return;
4565
4566 if (authentic_amd || hygon_genuine) {
4567 unsigned int topology_extensions;
4568
4569 if (max_extended_level < 0x8000001e)
4570 return;
4571
4572 eax = ebx = ecx = edx = 0;
4573 __cpuid(0x80000001, eax, ebx, ecx, edx);
4574 topology_extensions = ecx & (1 << 22);
4575
4576 if (topology_extensions == 0)
4577 return;
4578
4579 eax = ebx = ecx = edx = 0;
4580 __cpuid(0x8000001e, eax, ebx, ecx, edx);
4581
4582 t->x2apic_id = eax;
4583 return;
4584 }
4585
4586 if (!genuine_intel)
4587 return;
4588
4589 if (max_level < 0xb)
4590 return;
4591
4592 ecx = 0;
4593 __cpuid(0xb, eax, ebx, ecx, edx);
4594 t->x2apic_id = edx;
4595
4596 if (debug && (t->apic_id != (t->x2apic_id & 0xff)))
4597 fprintf(outf, "cpu%d: BIOS BUG: apic 0x%x x2apic 0x%x\n", t->cpu_id, t->apic_id, t->x2apic_id);
4598 }
4599
get_core_throt_cnt(int cpu,unsigned long long * cnt)4600 int get_core_throt_cnt(int cpu, unsigned long long *cnt)
4601 {
4602 char path[128 + PATH_BYTES];
4603 unsigned long long tmp;
4604 FILE *fp;
4605 int ret;
4606
4607 sprintf(path, "/sys/devices/system/cpu/cpu%d/thermal_throttle/core_throttle_count", cpu);
4608 fp = fopen(path, "r");
4609 if (!fp)
4610 return -1;
4611 ret = fscanf(fp, "%lld", &tmp);
4612 fclose(fp);
4613 if (ret != 1)
4614 return -1;
4615 *cnt = tmp;
4616
4617 return 0;
4618 }
4619
read_perf_counter_info(const char * const path,const char * const parse_format,void * value_ptr)4620 static int read_perf_counter_info(const char *const path, const char *const parse_format, void *value_ptr)
4621 {
4622 int fdmt;
4623 int bytes_read;
4624 char buf[64];
4625 int ret = -1;
4626
4627 fdmt = open(path, O_RDONLY, 0);
4628 if (fdmt == -1) {
4629 if (debug)
4630 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4631 ret = -1;
4632 goto cleanup_and_exit;
4633 }
4634
4635 bytes_read = read(fdmt, buf, sizeof(buf) - 1);
4636 if (bytes_read <= 0 || bytes_read >= (int)sizeof(buf)) {
4637 if (debug)
4638 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4639 ret = -1;
4640 goto cleanup_and_exit;
4641 }
4642
4643 buf[bytes_read] = '\0';
4644
4645 if (sscanf(buf, parse_format, value_ptr) != 1) {
4646 if (debug)
4647 fprintf(stderr, "Failed to parse perf counter info %s\n", path);
4648 ret = -1;
4649 goto cleanup_and_exit;
4650 }
4651
4652 ret = 0;
4653
4654 cleanup_and_exit:
4655 close(fdmt);
4656 return ret;
4657 }
4658
read_perf_counter_info_n(const char * const path,const char * const parse_format)4659 static unsigned int read_perf_counter_info_n(const char *const path, const char *const parse_format)
4660 {
4661 unsigned int v;
4662 int status;
4663
4664 status = read_perf_counter_info(path, parse_format, &v);
4665 if (status)
4666 v = -1;
4667
4668 return v;
4669 }
4670
read_perf_type(const char * subsys)4671 static unsigned int read_perf_type(const char *subsys)
4672 {
4673 const char *const path_format = "/sys/bus/event_source/devices/%s/type";
4674 const char *const format = "%u";
4675 char path[128];
4676
4677 snprintf(path, sizeof(path), path_format, subsys);
4678
4679 return read_perf_counter_info_n(path, format);
4680 }
4681
read_perf_config(const char * subsys,const char * event_name)4682 static unsigned int read_perf_config(const char *subsys, const char *event_name)
4683 {
4684 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s";
4685 FILE *fconfig = NULL;
4686 char path[128];
4687 char config_str[64];
4688 unsigned int config;
4689 unsigned int umask;
4690 bool has_config = false;
4691 bool has_umask = false;
4692 unsigned int ret = -1;
4693
4694 snprintf(path, sizeof(path), path_format, subsys, event_name);
4695
4696 fconfig = fopen(path, "r");
4697 if (!fconfig)
4698 return -1;
4699
4700 if (fgets(config_str, ARRAY_SIZE(config_str), fconfig) != config_str)
4701 goto cleanup_and_exit;
4702
4703 for (char *pconfig_str = &config_str[0]; pconfig_str;) {
4704 if (sscanf(pconfig_str, "event=%x", &config) == 1) {
4705 has_config = true;
4706 goto next;
4707 }
4708
4709 if (sscanf(pconfig_str, "umask=%x", &umask) == 1) {
4710 has_umask = true;
4711 goto next;
4712 }
4713
4714 next:
4715 pconfig_str = strchr(pconfig_str, ',');
4716 if (pconfig_str) {
4717 *pconfig_str = '\0';
4718 ++pconfig_str;
4719 }
4720 }
4721
4722 if (!has_umask)
4723 umask = 0;
4724
4725 if (has_config)
4726 ret = (umask << 8) | config;
4727
4728 cleanup_and_exit:
4729 fclose(fconfig);
4730 return ret;
4731 }
4732
read_perf_rapl_unit(const char * subsys,const char * event_name)4733 static unsigned int read_perf_rapl_unit(const char *subsys, const char *event_name)
4734 {
4735 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.unit";
4736 const char *const format = "%s";
4737 char path[128];
4738 char unit_buffer[16];
4739
4740 snprintf(path, sizeof(path), path_format, subsys, event_name);
4741
4742 read_perf_counter_info(path, format, &unit_buffer);
4743 if (strcmp("Joules", unit_buffer) == 0)
4744 return RAPL_UNIT_JOULES;
4745
4746 return RAPL_UNIT_INVALID;
4747 }
4748
read_perf_scale(const char * subsys,const char * event_name)4749 static double read_perf_scale(const char *subsys, const char *event_name)
4750 {
4751 const char *const path_format = "/sys/bus/event_source/devices/%s/events/%s.scale";
4752 const char *const format = "%lf";
4753 char path[128];
4754 double scale;
4755
4756 snprintf(path, sizeof(path), path_format, subsys, event_name);
4757
4758 if (read_perf_counter_info(path, format, &scale))
4759 return 0.0;
4760
4761 return scale;
4762 }
4763
rapl_counter_info_count_perf(const struct rapl_counter_info_t * rci)4764 size_t rapl_counter_info_count_perf(const struct rapl_counter_info_t *rci)
4765 {
4766 size_t ret = 0;
4767
4768 for (int i = 0; i < NUM_RAPL_COUNTERS; ++i)
4769 if (rci->source[i] == COUNTER_SOURCE_PERF)
4770 ++ret;
4771
4772 return ret;
4773 }
4774
cstate_counter_info_count_perf(const struct cstate_counter_info_t * cci)4775 static size_t cstate_counter_info_count_perf(const struct cstate_counter_info_t *cci)
4776 {
4777 size_t ret = 0;
4778
4779 for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i)
4780 if (cci->source[i] == COUNTER_SOURCE_PERF)
4781 ++ret;
4782
4783 return ret;
4784 }
4785
write_rapl_counter(struct rapl_counter * rc,struct rapl_counter_info_t * rci,unsigned int idx)4786 void write_rapl_counter(struct rapl_counter *rc, struct rapl_counter_info_t *rci, unsigned int idx)
4787 {
4788 if (rci->source[idx] == COUNTER_SOURCE_NONE)
4789 return;
4790
4791 rc->raw_value = rci->data[idx];
4792 rc->unit = rci->unit[idx];
4793 rc->scale = rci->scale[idx];
4794 }
4795
get_rapl_counters(int cpu,unsigned int domain,struct core_data * c,struct pkg_data * p)4796 int get_rapl_counters(int cpu, unsigned int domain, struct core_data *c, struct pkg_data *p)
4797 {
4798 struct platform_counters *pplat_cnt = p == odd.packages ? &platform_counters_odd : &platform_counters_even;
4799 unsigned long long perf_data[NUM_RAPL_COUNTERS + 1];
4800 struct rapl_counter_info_t *rci;
4801
4802 if (debug >= 2)
4803 fprintf(stderr, "%s: cpu%d domain%d\n", __func__, cpu, domain);
4804
4805 assert(rapl_counter_info_perdomain);
4806 assert(domain < rapl_counter_info_perdomain_size);
4807
4808 rci = &rapl_counter_info_perdomain[domain];
4809
4810 /*
4811 * If we have any perf counters to read, read them all now, in bulk
4812 */
4813 if (rci->fd_perf != -1) {
4814 size_t num_perf_counters = rapl_counter_info_count_perf(rci);
4815 const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
4816 const ssize_t actual_read_size = read(rci->fd_perf, &perf_data[0], sizeof(perf_data));
4817
4818 if (actual_read_size != expected_read_size)
4819 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size);
4820 }
4821
4822 for (unsigned int i = 0, pi = 1; i < NUM_RAPL_COUNTERS; ++i) {
4823 switch (rci->source[i]) {
4824 case COUNTER_SOURCE_NONE:
4825 rci->data[i] = 0;
4826 break;
4827
4828 case COUNTER_SOURCE_PERF:
4829 assert(pi < ARRAY_SIZE(perf_data));
4830 assert(rci->fd_perf != -1);
4831
4832 if (debug >= 2)
4833 fprintf(stderr, "Reading rapl counter via perf at %u (%llu %e %lf)\n",
4834 i, perf_data[pi], rci->scale[i], perf_data[pi] * rci->scale[i]);
4835
4836 rci->data[i] = perf_data[pi];
4837
4838 ++pi;
4839 break;
4840
4841 case COUNTER_SOURCE_MSR:
4842 if (debug >= 2)
4843 fprintf(stderr, "Reading rapl counter via msr at %u\n", i);
4844
4845 assert(!no_msr);
4846 if (rci->flags[i] & RAPL_COUNTER_FLAG_USE_MSR_SUM) {
4847 if (get_msr_sum(cpu, rci->msr[i], &rci->data[i]))
4848 return -13 - i;
4849 } else {
4850 if (get_msr(cpu, rci->msr[i], &rci->data[i]))
4851 return -13 - i;
4852 }
4853
4854 rci->data[i] &= rci->msr_mask[i];
4855 if (rci->msr_shift[i] >= 0)
4856 rci->data[i] >>= abs(rci->msr_shift[i]);
4857 else
4858 rci->data[i] <<= abs(rci->msr_shift[i]);
4859
4860 break;
4861 }
4862 }
4863
4864 BUILD_BUG_ON(NUM_RAPL_COUNTERS != 8);
4865 write_rapl_counter(&p->energy_pkg, rci, RAPL_RCI_INDEX_ENERGY_PKG);
4866 write_rapl_counter(&p->energy_cores, rci, RAPL_RCI_INDEX_ENERGY_CORES);
4867 write_rapl_counter(&p->energy_dram, rci, RAPL_RCI_INDEX_DRAM);
4868 write_rapl_counter(&p->energy_gfx, rci, RAPL_RCI_INDEX_GFX);
4869 write_rapl_counter(&p->rapl_pkg_perf_status, rci, RAPL_RCI_INDEX_PKG_PERF_STATUS);
4870 write_rapl_counter(&p->rapl_dram_perf_status, rci, RAPL_RCI_INDEX_DRAM_PERF_STATUS);
4871 write_rapl_counter(&c->core_energy, rci, RAPL_RCI_INDEX_CORE_ENERGY);
4872 write_rapl_counter(&pplat_cnt->energy_psys, rci, RAPL_RCI_INDEX_ENERGY_PLATFORM);
4873
4874 return 0;
4875 }
4876
find_sysfs_path_by_id(struct sysfs_path * sp,int id)4877 char *find_sysfs_path_by_id(struct sysfs_path *sp, int id)
4878 {
4879 while (sp) {
4880 if (sp->id == id)
4881 return (sp->path);
4882 sp = sp->next;
4883 }
4884 if (debug)
4885 warnx("%s: id%d not found", __func__, id);
4886 return NULL;
4887 }
4888
get_cstate_counters(unsigned int cpu,PER_THREAD_PARAMS)4889 int get_cstate_counters(unsigned int cpu, PER_THREAD_PARAMS)
4890 {
4891 /*
4892 * Overcommit memory a little bit here,
4893 * but skip calculating exact sizes for the buffers.
4894 */
4895 unsigned long long perf_data[NUM_CSTATE_COUNTERS];
4896 unsigned long long perf_data_core[NUM_CSTATE_COUNTERS + 1];
4897 unsigned long long perf_data_pkg[NUM_CSTATE_COUNTERS + 1];
4898
4899 struct cstate_counter_info_t *cci;
4900
4901 if (debug >= 2)
4902 fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
4903
4904 assert(ccstate_counter_info);
4905 assert(cpu <= ccstate_counter_info_size);
4906
4907 ZERO_ARRAY(perf_data);
4908 ZERO_ARRAY(perf_data_core);
4909 ZERO_ARRAY(perf_data_pkg);
4910
4911 cci = &ccstate_counter_info[cpu];
4912
4913 /*
4914 * If we have any perf counters to read, read them all now, in bulk
4915 */
4916 const size_t num_perf_counters = cstate_counter_info_count_perf(cci);
4917 ssize_t expected_read_size = num_perf_counters * sizeof(unsigned long long);
4918 ssize_t actual_read_size_core = 0, actual_read_size_pkg = 0;
4919
4920 if (cci->fd_perf_core != -1) {
4921 /* Each descriptor read begins with number of counters read. */
4922 expected_read_size += sizeof(unsigned long long);
4923
4924 actual_read_size_core = read(cci->fd_perf_core, &perf_data_core[0], sizeof(perf_data_core));
4925
4926 if (actual_read_size_core <= 0)
4927 err(-1, "%s: read perf %s: %ld", __func__, "core", actual_read_size_core);
4928 }
4929
4930 if (cci->fd_perf_pkg != -1) {
4931 /* Each descriptor read begins with number of counters read. */
4932 expected_read_size += sizeof(unsigned long long);
4933
4934 actual_read_size_pkg = read(cci->fd_perf_pkg, &perf_data_pkg[0], sizeof(perf_data_pkg));
4935
4936 if (actual_read_size_pkg <= 0)
4937 err(-1, "%s: read perf %s: %ld", __func__, "pkg", actual_read_size_pkg);
4938 }
4939
4940 const ssize_t actual_read_size_total = actual_read_size_core + actual_read_size_pkg;
4941
4942 if (actual_read_size_total != expected_read_size)
4943 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size_total);
4944
4945 /*
4946 * Copy ccstate and pcstate data into unified buffer.
4947 *
4948 * Skip first element from core and pkg buffers.
4949 * Kernel puts there how many counters were read.
4950 */
4951 const size_t num_core_counters = perf_data_core[0];
4952 const size_t num_pkg_counters = perf_data_pkg[0];
4953
4954 assert(num_perf_counters == num_core_counters + num_pkg_counters);
4955
4956 /* Copy ccstate perf data */
4957 memcpy(&perf_data[0], &perf_data_core[1], num_core_counters * sizeof(unsigned long long));
4958
4959 /* Copy pcstate perf data */
4960 memcpy(&perf_data[num_core_counters], &perf_data_pkg[1], num_pkg_counters * sizeof(unsigned long long));
4961
4962 for (unsigned int i = 0, pi = 0; i < NUM_CSTATE_COUNTERS; ++i) {
4963 switch (cci->source[i]) {
4964 case COUNTER_SOURCE_NONE:
4965 break;
4966
4967 case COUNTER_SOURCE_PERF:
4968 assert(pi < ARRAY_SIZE(perf_data));
4969 assert(cci->fd_perf_core != -1 || cci->fd_perf_pkg != -1);
4970
4971 if (debug >= 2)
4972 fprintf(stderr, "cstate via %s %u: %llu\n", "perf", i, perf_data[pi]);
4973
4974 cci->data[i] = perf_data[pi];
4975
4976 ++pi;
4977 break;
4978
4979 case COUNTER_SOURCE_MSR:
4980 assert(!no_msr);
4981 if (get_msr(cpu, cci->msr[i], &cci->data[i]))
4982 return -13 - i;
4983
4984 if (debug >= 2)
4985 fprintf(stderr, "cstate via %s0x%llx %u: %llu\n", "msr", cci->msr[i], i, cci->data[i]);
4986
4987 break;
4988 }
4989 }
4990
4991 /*
4992 * Helper to write the data only if the source of
4993 * the counter for the current cpu is not none.
4994 *
4995 * Otherwise we would overwrite core data with 0 (default value),
4996 * when invoked for the thread sibling.
4997 */
4998 #define PERF_COUNTER_WRITE_DATA(out_counter, index) do { \
4999 if (cci->source[index] != COUNTER_SOURCE_NONE) \
5000 out_counter = cci->data[index]; \
5001 } while (0)
5002
5003 BUILD_BUG_ON(NUM_CSTATE_COUNTERS != 11);
5004
5005 PERF_COUNTER_WRITE_DATA(t->c1, CCSTATE_RCI_INDEX_C1_RESIDENCY);
5006 PERF_COUNTER_WRITE_DATA(c->c3, CCSTATE_RCI_INDEX_C3_RESIDENCY);
5007 PERF_COUNTER_WRITE_DATA(c->c6, CCSTATE_RCI_INDEX_C6_RESIDENCY);
5008 PERF_COUNTER_WRITE_DATA(c->c7, CCSTATE_RCI_INDEX_C7_RESIDENCY);
5009
5010 PERF_COUNTER_WRITE_DATA(p->pc2, PCSTATE_RCI_INDEX_C2_RESIDENCY);
5011 PERF_COUNTER_WRITE_DATA(p->pc3, PCSTATE_RCI_INDEX_C3_RESIDENCY);
5012 PERF_COUNTER_WRITE_DATA(p->pc6, PCSTATE_RCI_INDEX_C6_RESIDENCY);
5013 PERF_COUNTER_WRITE_DATA(p->pc7, PCSTATE_RCI_INDEX_C7_RESIDENCY);
5014 PERF_COUNTER_WRITE_DATA(p->pc8, PCSTATE_RCI_INDEX_C8_RESIDENCY);
5015 PERF_COUNTER_WRITE_DATA(p->pc9, PCSTATE_RCI_INDEX_C9_RESIDENCY);
5016 PERF_COUNTER_WRITE_DATA(p->pc10, PCSTATE_RCI_INDEX_C10_RESIDENCY);
5017
5018 #undef PERF_COUNTER_WRITE_DATA
5019
5020 return 0;
5021 }
5022
msr_counter_info_count_perf(const struct msr_counter_info_t * mci)5023 size_t msr_counter_info_count_perf(const struct msr_counter_info_t *mci)
5024 {
5025 size_t ret = 0;
5026
5027 for (int i = 0; i < NUM_MSR_COUNTERS; ++i)
5028 if (mci->source[i] == COUNTER_SOURCE_PERF)
5029 ++ret;
5030
5031 return ret;
5032 }
5033
get_smi_aperf_mperf(unsigned int cpu,struct thread_data * t)5034 int get_smi_aperf_mperf(unsigned int cpu, struct thread_data *t)
5035 {
5036 unsigned long long perf_data[NUM_MSR_COUNTERS + 1];
5037
5038 struct msr_counter_info_t *mci;
5039
5040 if (debug >= 2)
5041 fprintf(stderr, "%s: cpu%d\n", __func__, cpu);
5042
5043 assert(msr_counter_info);
5044 assert(cpu <= msr_counter_info_size);
5045
5046 mci = &msr_counter_info[cpu];
5047
5048 ZERO_ARRAY(perf_data);
5049 ZERO_ARRAY(mci->data);
5050
5051 if (mci->fd_perf != -1) {
5052 const size_t num_perf_counters = msr_counter_info_count_perf(mci);
5053 const ssize_t expected_read_size = (num_perf_counters + 1) * sizeof(unsigned long long);
5054 const ssize_t actual_read_size = read(mci->fd_perf, &perf_data[0], sizeof(perf_data));
5055
5056 if (actual_read_size != expected_read_size)
5057 err(-1, "%s: failed to read perf_data (%zu %zu)", __func__, expected_read_size, actual_read_size);
5058 }
5059
5060 for (unsigned int i = 0, pi = 1; i < NUM_MSR_COUNTERS; ++i) {
5061 switch (mci->source[i]) {
5062 case COUNTER_SOURCE_NONE:
5063 break;
5064
5065 case COUNTER_SOURCE_PERF:
5066 assert(pi < ARRAY_SIZE(perf_data));
5067 assert(mci->fd_perf != -1);
5068
5069 if (debug >= 2)
5070 fprintf(stderr, "Reading msr counter via perf at %u: %llu\n", i, perf_data[pi]);
5071
5072 mci->data[i] = perf_data[pi];
5073
5074 ++pi;
5075 break;
5076
5077 case COUNTER_SOURCE_MSR:
5078 assert(!no_msr);
5079
5080 if (get_msr(cpu, mci->msr[i], &mci->data[i]))
5081 return -2 - i;
5082
5083 mci->data[i] &= mci->msr_mask[i];
5084
5085 if (debug >= 2)
5086 fprintf(stderr, "Reading msr counter via msr at %u: %llu\n", i, mci->data[i]);
5087
5088 break;
5089 }
5090 }
5091
5092 BUILD_BUG_ON(NUM_MSR_COUNTERS != 3);
5093 t->aperf = mci->data[MSR_RCI_INDEX_APERF];
5094 t->mperf = mci->data[MSR_RCI_INDEX_MPERF];
5095 t->smi_count = mci->data[MSR_RCI_INDEX_SMI];
5096
5097 return 0;
5098 }
5099
perf_counter_info_read_values(struct perf_counter_info * pp,int cpu,unsigned long long * out,size_t out_size)5100 int perf_counter_info_read_values(struct perf_counter_info *pp, int cpu, unsigned long long *out, size_t out_size)
5101 {
5102 unsigned int domain;
5103 unsigned long long value;
5104 int fd_counter;
5105
5106 for (size_t i = 0; pp; ++i, pp = pp->next) {
5107 domain = cpu_to_domain(pp, cpu);
5108 assert(domain < pp->num_domains);
5109
5110 fd_counter = pp->fd_perf_per_domain[domain];
5111
5112 if (fd_counter == -1)
5113 continue;
5114
5115 if (read(fd_counter, &value, sizeof(value)) != sizeof(value))
5116 return 1;
5117
5118 assert(i < out_size);
5119 out[i] = value * pp->scale;
5120 }
5121
5122 return 0;
5123 }
5124
pmt_gen_value_mask(unsigned int lsb,unsigned int msb)5125 unsigned long pmt_gen_value_mask(unsigned int lsb, unsigned int msb)
5126 {
5127 unsigned long mask;
5128
5129 if (msb == 63)
5130 mask = 0xffffffffffffffff;
5131 else
5132 mask = ((1 << (msb + 1)) - 1);
5133
5134 mask -= (1 << lsb) - 1;
5135
5136 return mask;
5137 }
5138
pmt_read_counter(struct pmt_counter * ppmt,unsigned int domain_id)5139 unsigned long pmt_read_counter(struct pmt_counter *ppmt, unsigned int domain_id)
5140 {
5141 if (domain_id >= ppmt->num_domains)
5142 return 0;
5143
5144 const unsigned long *pmmio = ppmt->domains[domain_id].pcounter;
5145 const unsigned long value = pmmio ? *pmmio : 0;
5146 const unsigned long value_mask = pmt_gen_value_mask(ppmt->lsb, ppmt->msb);
5147 const unsigned long value_shift = ppmt->lsb;
5148
5149 return (value & value_mask) >> value_shift;
5150 }
5151
5152 /* Rapl domain enumeration helpers */
get_rapl_num_domains(void)5153 static inline int get_rapl_num_domains(void)
5154 {
5155 if (!platform->has_per_core_rapl)
5156 return topo.num_packages;
5157
5158 return topo.num_cores;
5159 }
5160
get_rapl_domain_id(int cpu)5161 static inline int get_rapl_domain_id(int cpu)
5162 {
5163 if (!platform->has_per_core_rapl)
5164 return cpus[cpu].package_id;
5165
5166 return GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id);
5167 }
5168
5169 /*
5170 * get_counters(...)
5171 * migrate to cpu
5172 * acquire and record local counters for that cpu
5173 */
get_counters(PER_THREAD_PARAMS)5174 int get_counters(PER_THREAD_PARAMS)
5175 {
5176 int cpu = t->cpu_id;
5177 unsigned long long msr;
5178 struct msr_counter *mp;
5179 struct pmt_counter *pp;
5180 int i;
5181 int status;
5182
5183 if (cpu_migrate(cpu)) {
5184 fprintf(outf, "%s: Could not migrate to CPU %d\n", __func__, cpu);
5185 return -1;
5186 }
5187
5188 gettimeofday(&t->tv_begin, (struct timezone *)NULL);
5189
5190 if (first_counter_read)
5191 get_apic_id(t);
5192
5193 t->tsc = rdtsc(); /* we are running on local CPU of interest */
5194
5195 get_smi_aperf_mperf(cpu, t);
5196
5197 if (DO_BIC(BIC_LLC_MRPS) || DO_BIC(BIC_LLC_HIT))
5198 get_perf_llc_stats(cpu, &t->llc);
5199
5200 if (DO_BIC(BIC_L2_MRPS) || DO_BIC(BIC_L2_HIT))
5201 get_perf_l2_stats(cpu, &t->l2);
5202
5203 if (DO_BIC(BIC_IPC))
5204 if (read(get_instr_count_fd(cpu), &t->instr_count, sizeof(long long)) != sizeof(long long))
5205 return -4;
5206
5207 if (DO_BIC(BIC_IRQ))
5208 t->irq_count = irqs_per_cpu[cpu];
5209 if (DO_BIC(BIC_NMI))
5210 t->nmi_count = nmi_per_cpu[cpu];
5211
5212 get_cstate_counters(cpu, t, c, p);
5213
5214 for (i = 0, mp = sys.tp; mp; i++, mp = mp->next) {
5215 if (get_mp(cpu, mp, &t->counter[i], mp->sp->path))
5216 return -10;
5217 }
5218
5219 if (perf_counter_info_read_values(sys.perf_tp, cpu, t->perf_counter, MAX_ADDED_THREAD_COUNTERS))
5220 return -10;
5221
5222 for (i = 0, pp = sys.pmt_tp; pp; i++, pp = pp->next)
5223 t->pmt_counter[i] = pmt_read_counter(pp, t->cpu_id);
5224
5225 /* collect core counters only for 1st thread in core */
5226 if (!is_cpu_first_thread_in_core(t, c))
5227 goto done;
5228
5229 if (platform->has_per_core_rapl) {
5230 status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p);
5231 if (status != 0)
5232 return status;
5233 }
5234
5235 if (DO_BIC(BIC_CPU_c7) && t->is_atom) {
5236 /*
5237 * For Atom CPUs that has core cstate deeper than c6,
5238 * MSR_CORE_C6_RESIDENCY returns residency of cc6 and deeper.
5239 * Minus CC7 (and deeper cstates) residency to get
5240 * accturate cc6 residency.
5241 */
5242 c->c6 -= c->c7;
5243 }
5244
5245 if (DO_BIC(BIC_Mod_c6))
5246 if (get_msr(cpu, MSR_MODULE_C6_RES_MS, &c->mc6_us))
5247 return -8;
5248
5249 if (DO_BIC(BIC_CoreTmp)) {
5250 if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
5251 return -9;
5252 c->core_temp_c = tj_max - ((msr >> 16) & 0x7F);
5253 }
5254
5255 if (DO_BIC(BIC_CORE_THROT_CNT))
5256 get_core_throt_cnt(cpu, &c->core_throt_cnt);
5257
5258 for (i = 0, mp = sys.cp; mp; i++, mp = mp->next) {
5259 if (get_mp(cpu, mp, &c->counter[i], mp->sp->path))
5260 return -10;
5261 }
5262
5263 if (perf_counter_info_read_values(sys.perf_cp, cpu, c->perf_counter, MAX_ADDED_CORE_COUNTERS))
5264 return -10;
5265
5266 for (i = 0, pp = sys.pmt_cp; pp; i++, pp = pp->next)
5267 c->pmt_counter[i] = pmt_read_counter(pp, cpus[t->cpu_id].core_id);
5268
5269 /* collect package counters only for 1st core in package */
5270 if (!is_cpu_first_core_in_package(t, p))
5271 goto done;
5272
5273 if (DO_BIC(BIC_Totl_c0)) {
5274 if (get_msr(cpu, MSR_PKG_WEIGHTED_CORE_C0_RES, &p->pkg_wtd_core_c0))
5275 return -10;
5276 }
5277 if (DO_BIC(BIC_Any_c0)) {
5278 if (get_msr(cpu, MSR_PKG_ANY_CORE_C0_RES, &p->pkg_any_core_c0))
5279 return -11;
5280 }
5281 if (DO_BIC(BIC_GFX_c0)) {
5282 if (get_msr(cpu, MSR_PKG_ANY_GFXE_C0_RES, &p->pkg_any_gfxe_c0))
5283 return -12;
5284 }
5285 if (DO_BIC(BIC_CPUGFX)) {
5286 if (get_msr(cpu, MSR_PKG_BOTH_CORE_GFXE_C0_RES, &p->pkg_both_core_gfxe_c0))
5287 return -13;
5288 }
5289
5290 if (DO_BIC(BIC_CPU_LPI))
5291 p->cpu_lpi = cpuidle_cur_cpu_lpi_us;
5292 if (DO_BIC(BIC_SYS_LPI))
5293 p->sys_lpi = cpuidle_cur_sys_lpi_us;
5294
5295 if (!platform->has_per_core_rapl) {
5296 status = get_rapl_counters(cpu, get_rapl_domain_id(cpu), c, p);
5297 if (status != 0)
5298 return status;
5299 }
5300
5301 if (DO_BIC(BIC_PkgTmp)) {
5302 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
5303 return -17;
5304 p->pkg_temp_c = tj_max - ((msr >> 16) & 0x7F);
5305 }
5306
5307 if (DO_BIC(BIC_UNCORE_MHZ))
5308 p->uncore_mhz = get_legacy_uncore_mhz(cpus[t->cpu_id].package_id);
5309
5310 if (DO_BIC(BIC_GFX_rc6))
5311 p->gfx_rc6_ms = gfx_info[GFX_rc6].val_ull;
5312
5313 if (DO_BIC(BIC_GFXMHz))
5314 p->gfx_mhz = gfx_info[GFX_MHz].val;
5315
5316 if (DO_BIC(BIC_GFXACTMHz))
5317 p->gfx_act_mhz = gfx_info[GFX_ACTMHz].val;
5318
5319 if (DO_BIC(BIC_SAM_mc6))
5320 p->sam_mc6_ms = gfx_info[SAM_mc6].val_ull;
5321
5322 if (DO_BIC(BIC_SAMMHz))
5323 p->sam_mhz = gfx_info[SAM_MHz].val;
5324
5325 if (DO_BIC(BIC_SAMACTMHz))
5326 p->sam_act_mhz = gfx_info[SAM_ACTMHz].val;
5327
5328 for (i = 0, mp = sys.pp; mp; i++, mp = mp->next) {
5329 char *path = NULL;
5330
5331 if (mp->msr_num == 0) {
5332 path = find_sysfs_path_by_id(mp->sp, cpus[t->cpu_id].package_id);
5333 if (path == NULL) {
5334 warnx("%s: package_id %d not found", __func__, cpus[t->cpu_id].package_id);
5335 return -10;
5336 }
5337 }
5338 if (get_mp(cpu, mp, &p->counter[i], path))
5339 return -10;
5340 }
5341
5342 if (perf_counter_info_read_values(sys.perf_pp, cpu, p->perf_counter, MAX_ADDED_PACKAGE_COUNTERS))
5343 return -10;
5344
5345 for (i = 0, pp = sys.pmt_pp; pp; i++, pp = pp->next)
5346 p->pmt_counter[i] = pmt_read_counter(pp, cpus[t->cpu_id].package_id);
5347
5348 done:
5349 gettimeofday(&t->tv_end, (struct timezone *)NULL);
5350
5351 return 0;
5352 }
5353
5354 int pkg_cstate_limit = PCLUKN;
5355 char *pkg_cstate_limit_strings[] = { "unknown", "reserved", "pc0", "pc1", "pc2",
5356 "pc3", "pc4", "pc6", "pc6n", "pc6r", "pc7", "pc7s", "pc8", "pc9", "pc10", "unlimited"
5357 };
5358
5359 int nhm_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCL__3, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5360 PCLRSV, PCLRSV
5361 };
5362
5363 int snb_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCL__7, PCL_7S, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5364 PCLRSV, PCLRSV
5365 };
5366
5367 int hsw_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5368 PCLRSV, PCLRSV
5369 };
5370
5371 int slv_pkg_cstate_limits[16] = { PCL__0, PCL__1, PCLRSV, PCLRSV, PCL__4, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5372 PCL__6, PCL__7
5373 };
5374
5375 int amt_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__2, PCLRSV, PCLRSV, PCLRSV, PCL__6, PCL__7, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5376 PCLRSV, PCLRSV
5377 };
5378
5379 int phi_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5380 PCLRSV, PCLRSV
5381 };
5382
5383 int glm_pkg_cstate_limits[16] = { PCLUNL, PCL__1, PCL__3, PCL__6, PCL__7, PCL_7S, PCL__8, PCL__9, PCL_10, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5384 PCLRSV, PCLRSV
5385 };
5386
5387 int skx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL_6N, PCL_6R, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5388 PCLRSV, PCLRSV
5389 };
5390
5391 int icx_pkg_cstate_limits[16] = { PCL__0, PCL__2, PCL__6, PCL__6, PCLRSV, PCLRSV, PCLRSV, PCLUNL, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV, PCLRSV,
5392 PCLRSV, PCLRSV
5393 };
5394
probe_cst_limit(void)5395 void probe_cst_limit(void)
5396 {
5397 unsigned long long msr;
5398 int *pkg_cstate_limits;
5399
5400 if (!platform->has_nhm_msrs || no_msr)
5401 return;
5402
5403 switch (platform->cst_limit) {
5404 case CST_LIMIT_NHM:
5405 pkg_cstate_limits = nhm_pkg_cstate_limits;
5406 break;
5407 case CST_LIMIT_SNB:
5408 pkg_cstate_limits = snb_pkg_cstate_limits;
5409 break;
5410 case CST_LIMIT_HSW:
5411 pkg_cstate_limits = hsw_pkg_cstate_limits;
5412 break;
5413 case CST_LIMIT_SKX:
5414 pkg_cstate_limits = skx_pkg_cstate_limits;
5415 break;
5416 case CST_LIMIT_ICX:
5417 pkg_cstate_limits = icx_pkg_cstate_limits;
5418 break;
5419 case CST_LIMIT_SLV:
5420 pkg_cstate_limits = slv_pkg_cstate_limits;
5421 break;
5422 case CST_LIMIT_AMT:
5423 pkg_cstate_limits = amt_pkg_cstate_limits;
5424 break;
5425 case CST_LIMIT_KNL:
5426 pkg_cstate_limits = phi_pkg_cstate_limits;
5427 break;
5428 case CST_LIMIT_GMT:
5429 pkg_cstate_limits = glm_pkg_cstate_limits;
5430 break;
5431 default:
5432 return;
5433 }
5434
5435 get_msr(master_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
5436 pkg_cstate_limit = pkg_cstate_limits[msr & 0xF];
5437 }
5438
dump_platform_info(void)5439 static void dump_platform_info(void)
5440 {
5441 unsigned long long msr;
5442 unsigned int ratio;
5443
5444 if (!platform->has_nhm_msrs || no_msr)
5445 return;
5446
5447 get_msr(master_cpu, MSR_PLATFORM_INFO, &msr);
5448
5449 fprintf(outf, "cpu%d: MSR_PLATFORM_INFO: 0x%08llx\n", master_cpu, msr);
5450
5451 ratio = (msr >> 40) & 0xFF;
5452 fprintf(outf, "%d * %.1f = %.1f MHz max efficiency frequency\n", ratio, bclk, ratio * bclk);
5453
5454 ratio = (msr >> 8) & 0xFF;
5455 fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
5456 }
5457
dump_power_ctl(void)5458 static void dump_power_ctl(void)
5459 {
5460 unsigned long long msr;
5461
5462 if (!platform->has_nhm_msrs || no_msr)
5463 return;
5464
5465 get_msr(master_cpu, MSR_IA32_POWER_CTL, &msr);
5466 fprintf(outf, "cpu%d: MSR_IA32_POWER_CTL: 0x%08llx (C1E auto-promotion: %sabled)\n", master_cpu, msr, msr & 0x2 ? "EN" : "DIS");
5467
5468 /* C-state Pre-wake Disable (CSTATE_PREWAKE_DISABLE) */
5469 if (platform->has_cst_prewake_bit)
5470 fprintf(outf, "C-state Pre-wake: %sabled\n", msr & 0x40000000 ? "DIS" : "EN");
5471
5472 return;
5473 }
5474
dump_turbo_ratio_limit2(void)5475 static void dump_turbo_ratio_limit2(void)
5476 {
5477 unsigned long long msr;
5478 unsigned int ratio;
5479
5480 get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT2, &msr);
5481
5482 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT2: 0x%08llx\n", master_cpu, msr);
5483
5484 ratio = (msr >> 8) & 0xFF;
5485 if (ratio)
5486 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 18 active cores\n", ratio, bclk, ratio * bclk);
5487
5488 ratio = (msr >> 0) & 0xFF;
5489 if (ratio)
5490 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 17 active cores\n", ratio, bclk, ratio * bclk);
5491 return;
5492 }
5493
dump_turbo_ratio_limit1(void)5494 static void dump_turbo_ratio_limit1(void)
5495 {
5496 unsigned long long msr;
5497 unsigned int ratio;
5498
5499 get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT1, &msr);
5500
5501 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", master_cpu, msr);
5502
5503 ratio = (msr >> 56) & 0xFF;
5504 if (ratio)
5505 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 16 active cores\n", ratio, bclk, ratio * bclk);
5506
5507 ratio = (msr >> 48) & 0xFF;
5508 if (ratio)
5509 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 15 active cores\n", ratio, bclk, ratio * bclk);
5510
5511 ratio = (msr >> 40) & 0xFF;
5512 if (ratio)
5513 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 14 active cores\n", ratio, bclk, ratio * bclk);
5514
5515 ratio = (msr >> 32) & 0xFF;
5516 if (ratio)
5517 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 13 active cores\n", ratio, bclk, ratio * bclk);
5518
5519 ratio = (msr >> 24) & 0xFF;
5520 if (ratio)
5521 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 12 active cores\n", ratio, bclk, ratio * bclk);
5522
5523 ratio = (msr >> 16) & 0xFF;
5524 if (ratio)
5525 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 11 active cores\n", ratio, bclk, ratio * bclk);
5526
5527 ratio = (msr >> 8) & 0xFF;
5528 if (ratio)
5529 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 10 active cores\n", ratio, bclk, ratio * bclk);
5530
5531 ratio = (msr >> 0) & 0xFF;
5532 if (ratio)
5533 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 9 active cores\n", ratio, bclk, ratio * bclk);
5534 return;
5535 }
5536
dump_turbo_ratio_limits(int trl_msr_offset)5537 static void dump_turbo_ratio_limits(int trl_msr_offset)
5538 {
5539 unsigned long long msr, core_counts;
5540 int shift;
5541
5542 get_msr(master_cpu, trl_msr_offset, &msr);
5543 fprintf(outf, "cpu%d: MSR_%sTURBO_RATIO_LIMIT: 0x%08llx\n", master_cpu, trl_msr_offset == MSR_SECONDARY_TURBO_RATIO_LIMIT ? "SECONDARY_" : "", msr);
5544
5545 if (platform->trl_msrs & TRL_CORECOUNT) {
5546 get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT1, &core_counts);
5547 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT1: 0x%08llx\n", master_cpu, core_counts);
5548 } else {
5549 core_counts = 0x0807060504030201;
5550 }
5551
5552 for (shift = 56; shift >= 0; shift -= 8) {
5553 unsigned int ratio, group_size;
5554
5555 ratio = (msr >> shift) & 0xFF;
5556 group_size = (core_counts >> shift) & 0xFF;
5557 if (ratio)
5558 fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio, bclk, ratio * bclk, group_size);
5559 }
5560
5561 return;
5562 }
5563
dump_atom_turbo_ratio_limits(void)5564 static void dump_atom_turbo_ratio_limits(void)
5565 {
5566 unsigned long long msr;
5567 unsigned int ratio;
5568
5569 get_msr(master_cpu, MSR_ATOM_CORE_RATIOS, &msr);
5570 fprintf(outf, "cpu%d: MSR_ATOM_CORE_RATIOS: 0x%08llx\n", master_cpu, msr & 0xFFFFFFFF);
5571
5572 ratio = (msr >> 0) & 0x3F;
5573 if (ratio)
5574 fprintf(outf, "%d * %.1f = %.1f MHz minimum operating frequency\n", ratio, bclk, ratio * bclk);
5575
5576 ratio = (msr >> 8) & 0x3F;
5577 if (ratio)
5578 fprintf(outf, "%d * %.1f = %.1f MHz low frequency mode (LFM)\n", ratio, bclk, ratio * bclk);
5579
5580 ratio = (msr >> 16) & 0x3F;
5581 if (ratio)
5582 fprintf(outf, "%d * %.1f = %.1f MHz base frequency\n", ratio, bclk, ratio * bclk);
5583
5584 get_msr(master_cpu, MSR_ATOM_CORE_TURBO_RATIOS, &msr);
5585 fprintf(outf, "cpu%d: MSR_ATOM_CORE_TURBO_RATIOS: 0x%08llx\n", master_cpu, msr & 0xFFFFFFFF);
5586
5587 ratio = (msr >> 24) & 0x3F;
5588 if (ratio)
5589 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 4 active cores\n", ratio, bclk, ratio * bclk);
5590
5591 ratio = (msr >> 16) & 0x3F;
5592 if (ratio)
5593 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 3 active cores\n", ratio, bclk, ratio * bclk);
5594
5595 ratio = (msr >> 8) & 0x3F;
5596 if (ratio)
5597 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 2 active cores\n", ratio, bclk, ratio * bclk);
5598
5599 ratio = (msr >> 0) & 0x3F;
5600 if (ratio)
5601 fprintf(outf, "%d * %.1f = %.1f MHz max turbo 1 active core\n", ratio, bclk, ratio * bclk);
5602 }
5603
dump_knl_turbo_ratio_limits(void)5604 static void dump_knl_turbo_ratio_limits(void)
5605 {
5606 const unsigned int buckets_no = 7;
5607
5608 unsigned long long msr;
5609 int delta_cores, delta_ratio;
5610 int i, b_nr;
5611 unsigned int cores[buckets_no];
5612 unsigned int ratio[buckets_no];
5613
5614 get_msr(master_cpu, MSR_TURBO_RATIO_LIMIT, &msr);
5615
5616 fprintf(outf, "cpu%d: MSR_TURBO_RATIO_LIMIT: 0x%08llx\n", master_cpu, msr);
5617
5618 /*
5619 * Turbo encoding in KNL is as follows:
5620 * [0] -- Reserved
5621 * [7:1] -- Base value of number of active cores of bucket 1.
5622 * [15:8] -- Base value of freq ratio of bucket 1.
5623 * [20:16] -- +ve delta of number of active cores of bucket 2.
5624 * i.e. active cores of bucket 2 =
5625 * active cores of bucket 1 + delta
5626 * [23:21] -- Negative delta of freq ratio of bucket 2.
5627 * i.e. freq ratio of bucket 2 =
5628 * freq ratio of bucket 1 - delta
5629 * [28:24]-- +ve delta of number of active cores of bucket 3.
5630 * [31:29]-- -ve delta of freq ratio of bucket 3.
5631 * [36:32]-- +ve delta of number of active cores of bucket 4.
5632 * [39:37]-- -ve delta of freq ratio of bucket 4.
5633 * [44:40]-- +ve delta of number of active cores of bucket 5.
5634 * [47:45]-- -ve delta of freq ratio of bucket 5.
5635 * [52:48]-- +ve delta of number of active cores of bucket 6.
5636 * [55:53]-- -ve delta of freq ratio of bucket 6.
5637 * [60:56]-- +ve delta of number of active cores of bucket 7.
5638 * [63:61]-- -ve delta of freq ratio of bucket 7.
5639 */
5640
5641 b_nr = 0;
5642 cores[b_nr] = (msr & 0xFF) >> 1;
5643 ratio[b_nr] = (msr >> 8) & 0xFF;
5644
5645 for (i = 16; i < 64; i += 8) {
5646 delta_cores = (msr >> i) & 0x1F;
5647 delta_ratio = (msr >> (i + 5)) & 0x7;
5648
5649 cores[b_nr + 1] = cores[b_nr] + delta_cores;
5650 ratio[b_nr + 1] = ratio[b_nr] - delta_ratio;
5651 b_nr++;
5652 }
5653
5654 for (i = buckets_no - 1; i >= 0; i--)
5655 if (i > 0 ? ratio[i] != ratio[i - 1] : 1)
5656 fprintf(outf, "%d * %.1f = %.1f MHz max turbo %d active cores\n", ratio[i], bclk, ratio[i] * bclk, cores[i]);
5657 }
5658
dump_cst_cfg(void)5659 static void dump_cst_cfg(void)
5660 {
5661 unsigned long long msr;
5662
5663 if (!platform->has_nhm_msrs || no_msr)
5664 return;
5665
5666 get_msr(master_cpu, MSR_PKG_CST_CONFIG_CONTROL, &msr);
5667
5668 fprintf(outf, "cpu%d: MSR_PKG_CST_CONFIG_CONTROL: 0x%08llx", master_cpu, msr);
5669
5670 fprintf(outf, " (%s%s%s%s%slocked, pkg-cstate-limit=%d (%s)",
5671 (msr & SNB_C3_AUTO_UNDEMOTE) ? "UNdemote-C3, " : "",
5672 (msr & SNB_C1_AUTO_UNDEMOTE) ? "UNdemote-C1, " : "",
5673 (msr & NHM_C3_AUTO_DEMOTE) ? "demote-C3, " : "",
5674 (msr & NHM_C1_AUTO_DEMOTE) ? "demote-C1, " : "",
5675 (msr & (1 << 15)) ? "" : "UN", (unsigned int)msr & 0xF, pkg_cstate_limit_strings[pkg_cstate_limit]);
5676
5677 #define AUTOMATIC_CSTATE_CONVERSION (1UL << 16)
5678 if (platform->has_cst_auto_convension) {
5679 fprintf(outf, ", automatic c-state conversion=%s", (msr & AUTOMATIC_CSTATE_CONVERSION) ? "on" : "off");
5680 }
5681
5682 fprintf(outf, ")\n");
5683
5684 return;
5685 }
5686
dump_config_tdp(void)5687 static void dump_config_tdp(void)
5688 {
5689 unsigned long long msr;
5690
5691 get_msr(master_cpu, MSR_CONFIG_TDP_NOMINAL, &msr);
5692 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_NOMINAL: 0x%08llx", master_cpu, msr);
5693 fprintf(outf, " (base_ratio=%d)\n", (unsigned int)msr & 0xFF);
5694
5695 get_msr(master_cpu, MSR_CONFIG_TDP_LEVEL_1, &msr);
5696 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_1: 0x%08llx (", master_cpu, msr);
5697 if (msr) {
5698 fprintf(outf, "PKG_MIN_PWR_LVL1=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5699 fprintf(outf, "PKG_MAX_PWR_LVL1=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5700 fprintf(outf, "LVL1_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5701 fprintf(outf, "PKG_TDP_LVL1=%d", (unsigned int)(msr) & 0x7FFF);
5702 }
5703 fprintf(outf, ")\n");
5704
5705 get_msr(master_cpu, MSR_CONFIG_TDP_LEVEL_2, &msr);
5706 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_LEVEL_2: 0x%08llx (", master_cpu, msr);
5707 if (msr) {
5708 fprintf(outf, "PKG_MIN_PWR_LVL2=%d ", (unsigned int)(msr >> 48) & 0x7FFF);
5709 fprintf(outf, "PKG_MAX_PWR_LVL2=%d ", (unsigned int)(msr >> 32) & 0x7FFF);
5710 fprintf(outf, "LVL2_RATIO=%d ", (unsigned int)(msr >> 16) & 0xFF);
5711 fprintf(outf, "PKG_TDP_LVL2=%d", (unsigned int)(msr) & 0x7FFF);
5712 }
5713 fprintf(outf, ")\n");
5714
5715 get_msr(master_cpu, MSR_CONFIG_TDP_CONTROL, &msr);
5716 fprintf(outf, "cpu%d: MSR_CONFIG_TDP_CONTROL: 0x%08llx (", master_cpu, msr);
5717 if ((msr) & 0x3)
5718 fprintf(outf, "TDP_LEVEL=%d ", (unsigned int)(msr) & 0x3);
5719 fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5720 fprintf(outf, ")\n");
5721
5722 get_msr(master_cpu, MSR_TURBO_ACTIVATION_RATIO, &msr);
5723 fprintf(outf, "cpu%d: MSR_TURBO_ACTIVATION_RATIO: 0x%08llx (", master_cpu, msr);
5724 fprintf(outf, "MAX_NON_TURBO_RATIO=%d", (unsigned int)(msr) & 0xFF);
5725 fprintf(outf, " lock=%d", (unsigned int)(msr >> 31) & 1);
5726 fprintf(outf, ")\n");
5727 }
5728
5729 unsigned int irtl_time_units[] = { 1, 32, 1024, 32768, 1048576, 33554432, 0, 0 };
5730
print_irtl(void)5731 void print_irtl(void)
5732 {
5733 unsigned long long msr;
5734
5735 if (!platform->has_irtl_msrs || no_msr)
5736 return;
5737
5738 if (platform->supported_cstates & PC3) {
5739 get_msr(master_cpu, MSR_PKGC3_IRTL, &msr);
5740 fprintf(outf, "cpu%d: MSR_PKGC3_IRTL: 0x%08llx (", master_cpu, msr);
5741 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5742 }
5743
5744 if (platform->supported_cstates & PC6) {
5745 get_msr(master_cpu, MSR_PKGC6_IRTL, &msr);
5746 fprintf(outf, "cpu%d: MSR_PKGC6_IRTL: 0x%08llx (", master_cpu, msr);
5747 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5748 }
5749
5750 if (platform->supported_cstates & PC7) {
5751 get_msr(master_cpu, MSR_PKGC7_IRTL, &msr);
5752 fprintf(outf, "cpu%d: MSR_PKGC7_IRTL: 0x%08llx (", master_cpu, msr);
5753 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5754 }
5755
5756 if (platform->supported_cstates & PC8) {
5757 get_msr(master_cpu, MSR_PKGC8_IRTL, &msr);
5758 fprintf(outf, "cpu%d: MSR_PKGC8_IRTL: 0x%08llx (", master_cpu, msr);
5759 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5760 }
5761
5762 if (platform->supported_cstates & PC9) {
5763 get_msr(master_cpu, MSR_PKGC9_IRTL, &msr);
5764 fprintf(outf, "cpu%d: MSR_PKGC9_IRTL: 0x%08llx (", master_cpu, msr);
5765 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5766 }
5767
5768 if (platform->supported_cstates & PC10) {
5769 get_msr(master_cpu, MSR_PKGC10_IRTL, &msr);
5770 fprintf(outf, "cpu%d: MSR_PKGC10_IRTL: 0x%08llx (", master_cpu, msr);
5771 fprintf(outf, "%svalid, %lld ns)\n", msr & (1 << 15) ? "" : "NOT", (msr & 0x3FF) * irtl_time_units[(msr >> 10) & 0x3]);
5772 }
5773 }
5774
free_fd_percpu(void)5775 void free_fd_percpu(void)
5776 {
5777 int i;
5778
5779 if (!fd_percpu)
5780 return;
5781
5782 for (i = 0; i < topo.max_cpu_num + 1; ++i) {
5783 if (fd_percpu[i] != 0)
5784 close(fd_percpu[i]);
5785 }
5786
5787 free(fd_percpu);
5788 fd_percpu = NULL;
5789 }
5790
free_fd_instr_count_percpu(void)5791 void free_fd_instr_count_percpu(void)
5792 {
5793 if (!fd_instr_count_percpu)
5794 return;
5795
5796 for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5797 if (fd_instr_count_percpu[i] != 0)
5798 close(fd_instr_count_percpu[i]);
5799 }
5800
5801 free(fd_instr_count_percpu);
5802 fd_instr_count_percpu = NULL;
5803 }
5804
free_fd_llc_percpu(void)5805 void free_fd_llc_percpu(void)
5806 {
5807 if (!fd_llc_percpu)
5808 return;
5809
5810 for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5811 if (fd_llc_percpu[i] != 0)
5812 close(fd_llc_percpu[i]);
5813 }
5814
5815 free(fd_llc_percpu);
5816 fd_llc_percpu = NULL;
5817
5818 BIC_NOT_PRESENT(BIC_LLC_MRPS);
5819 BIC_NOT_PRESENT(BIC_LLC_HIT);
5820 }
5821
free_fd_l2_percpu(void)5822 void free_fd_l2_percpu(void)
5823 {
5824 if (!fd_l2_percpu)
5825 return;
5826
5827 for (int i = 0; i < topo.max_cpu_num + 1; ++i) {
5828 if (fd_l2_percpu[i] != 0)
5829 close(fd_l2_percpu[i]);
5830 }
5831
5832 free(fd_l2_percpu);
5833 fd_l2_percpu = NULL;
5834
5835 BIC_NOT_PRESENT(BIC_L2_MRPS);
5836 BIC_NOT_PRESENT(BIC_L2_HIT);
5837 }
5838
free_fd_cstate(void)5839 void free_fd_cstate(void)
5840 {
5841 if (!ccstate_counter_info)
5842 return;
5843
5844 const int counter_info_num = ccstate_counter_info_size;
5845
5846 for (int counter_id = 0; counter_id < counter_info_num; ++counter_id) {
5847 if (ccstate_counter_info[counter_id].fd_perf_core != -1)
5848 close(ccstate_counter_info[counter_id].fd_perf_core);
5849
5850 if (ccstate_counter_info[counter_id].fd_perf_pkg != -1)
5851 close(ccstate_counter_info[counter_id].fd_perf_pkg);
5852 }
5853
5854 free(ccstate_counter_info);
5855 ccstate_counter_info = NULL;
5856 ccstate_counter_info_size = 0;
5857 }
5858
free_fd_msr(void)5859 void free_fd_msr(void)
5860 {
5861 if (!msr_counter_info)
5862 return;
5863
5864 for (int cpu = 0; cpu < topo.max_cpu_num; ++cpu) {
5865 if (msr_counter_info[cpu].fd_perf != -1)
5866 close(msr_counter_info[cpu].fd_perf);
5867 }
5868
5869 free(msr_counter_info);
5870 msr_counter_info = NULL;
5871 msr_counter_info_size = 0;
5872 }
5873
free_fd_rapl_percpu(void)5874 void free_fd_rapl_percpu(void)
5875 {
5876 if (!rapl_counter_info_perdomain)
5877 return;
5878
5879 const int num_domains = rapl_counter_info_perdomain_size;
5880
5881 for (int domain_id = 0; domain_id < num_domains; ++domain_id) {
5882 if (rapl_counter_info_perdomain[domain_id].fd_perf != -1)
5883 close(rapl_counter_info_perdomain[domain_id].fd_perf);
5884 }
5885
5886 free(rapl_counter_info_perdomain);
5887 rapl_counter_info_perdomain = NULL;
5888 rapl_counter_info_perdomain_size = 0;
5889 }
5890
free_fd_added_perf_counters_(struct perf_counter_info * pp)5891 void free_fd_added_perf_counters_(struct perf_counter_info *pp)
5892 {
5893 if (!pp)
5894 return;
5895
5896 if (!pp->fd_perf_per_domain)
5897 return;
5898
5899 while (pp) {
5900 for (size_t domain = 0; domain < pp->num_domains; ++domain) {
5901 if (pp->fd_perf_per_domain[domain] != -1) {
5902 close(pp->fd_perf_per_domain[domain]);
5903 pp->fd_perf_per_domain[domain] = -1;
5904 }
5905 }
5906
5907 free(pp->fd_perf_per_domain);
5908 pp->fd_perf_per_domain = NULL;
5909
5910 pp = pp->next;
5911 }
5912 }
5913
free_fd_added_perf_counters(void)5914 void free_fd_added_perf_counters(void)
5915 {
5916 free_fd_added_perf_counters_(sys.perf_tp);
5917 free_fd_added_perf_counters_(sys.perf_cp);
5918 free_fd_added_perf_counters_(sys.perf_pp);
5919 }
5920
free_all_buffers(void)5921 void free_all_buffers(void)
5922 {
5923 int i;
5924
5925 CPU_FREE(cpu_present_set);
5926 cpu_present_set = NULL;
5927 cpu_present_setsize = 0;
5928
5929 CPU_FREE(cpu_effective_set);
5930 cpu_effective_set = NULL;
5931 cpu_effective_setsize = 0;
5932
5933 CPU_FREE(cpu_allowed_set);
5934 cpu_allowed_set = NULL;
5935 cpu_allowed_setsize = 0;
5936
5937 CPU_FREE(cpu_affinity_set);
5938 cpu_affinity_set = NULL;
5939 cpu_affinity_setsize = 0;
5940
5941 if (perf_pcore_set) {
5942 CPU_FREE(perf_pcore_set);
5943 perf_pcore_set = NULL;
5944 }
5945
5946 if (perf_ecore_set) {
5947 CPU_FREE(perf_ecore_set);
5948 perf_ecore_set = NULL;
5949 }
5950
5951 if (perf_lcore_set) {
5952 CPU_FREE(perf_lcore_set);
5953 perf_lcore_set = NULL;
5954 }
5955
5956 free(even.threads);
5957 free(even.cores);
5958 free(even.packages);
5959
5960 even.threads = NULL;
5961 even.cores = NULL;
5962 even.packages = NULL;
5963
5964 free(odd.threads);
5965 free(odd.cores);
5966 free(odd.packages);
5967
5968 odd.threads = NULL;
5969 odd.cores = NULL;
5970 odd.packages = NULL;
5971
5972 free(output_buffer);
5973 output_buffer = NULL;
5974 outp = NULL;
5975
5976 free_fd_percpu();
5977 free_fd_instr_count_percpu();
5978 free_fd_llc_percpu();
5979 free_fd_l2_percpu();
5980 free_fd_msr();
5981 free_fd_rapl_percpu();
5982 free_fd_cstate();
5983 free_fd_added_perf_counters();
5984
5985 free(irq_column_2_cpu);
5986 free(irqs_per_cpu);
5987 free(nmi_per_cpu);
5988
5989 for (i = 0; i <= topo.max_cpu_num; ++i) {
5990 if (cpus[i].put_ids)
5991 CPU_FREE(cpus[i].put_ids);
5992 }
5993 free(cpus);
5994 }
5995
5996 /*
5997 * Parse a file containing a single int.
5998 * Return 0 if file can not be opened
5999 * Exit if file can be opened, but can not be parsed
6000 */
parse_int_file(const char * fmt,...)6001 int parse_int_file(const char *fmt, ...)
6002 {
6003 va_list args;
6004 char path[PATH_MAX];
6005 FILE *filep;
6006 int value;
6007
6008 va_start(args, fmt);
6009 vsnprintf(path, sizeof(path), fmt, args);
6010 va_end(args);
6011 filep = fopen(path, "r");
6012 if (!filep)
6013 return 0;
6014 if (fscanf(filep, "%d", &value) != 1)
6015 err(1, "%s: failed to parse number from file", path);
6016 fclose(filep);
6017 return value;
6018 }
6019
6020 /*
6021 * cpu_is_first_core_in_package(cpu)
6022 * return 1 if given CPU is 1st core in package
6023 */
cpu_is_first_core_in_package(int cpu)6024 int cpu_is_first_core_in_package(int cpu)
6025 {
6026 return cpu == parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_siblings_list", cpu);
6027 }
6028
get_package_id(int cpu)6029 int get_package_id(int cpu)
6030 {
6031 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/physical_package_id", cpu);
6032 }
6033
get_die_id(int cpu)6034 int get_die_id(int cpu)
6035 {
6036 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/die_id", cpu);
6037 }
6038
get_l3_id(int cpu)6039 int get_l3_id(int cpu)
6040 {
6041 return parse_int_file("/sys/devices/system/cpu/cpu%d/cache/index3/id", cpu);
6042 }
6043
get_core_id(int cpu)6044 int get_core_id(int cpu)
6045 {
6046 return parse_int_file("/sys/devices/system/cpu/cpu%d/topology/core_id", cpu);
6047 }
6048
set_node_data(void)6049 void set_node_data(void)
6050 {
6051 int pkg, node, lnode, cpu, cpux;
6052 int cpu_count;
6053
6054 /* initialize logical_node_id */
6055 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu)
6056 cpus[cpu].logical_node_id = -1;
6057
6058 cpu_count = 0;
6059 for (pkg = 0; pkg < topo.num_packages; pkg++) {
6060 lnode = 0;
6061 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
6062 if (cpus[cpu].package_id != pkg)
6063 continue;
6064 /* find a cpu with an unset logical_node_id */
6065 if (cpus[cpu].logical_node_id != -1)
6066 continue;
6067 cpus[cpu].logical_node_id = lnode;
6068 node = cpus[cpu].physical_node_id;
6069 cpu_count++;
6070 /*
6071 * find all matching cpus on this pkg and set
6072 * the logical_node_id
6073 */
6074 for (cpux = cpu; cpux <= topo.max_cpu_num; cpux++) {
6075 if ((cpus[cpux].package_id == pkg) && (cpus[cpux].physical_node_id == node)) {
6076 cpus[cpux].logical_node_id = lnode;
6077 cpu_count++;
6078 }
6079 }
6080 lnode++;
6081 if (lnode > topo.nodes_per_pkg)
6082 topo.nodes_per_pkg = lnode;
6083 }
6084 if (cpu_count >= topo.max_cpu_num)
6085 break;
6086 }
6087 }
6088
get_physical_node_id(struct cpu_topology * thiscpu)6089 int get_physical_node_id(struct cpu_topology *thiscpu)
6090 {
6091 char path[80];
6092 FILE *filep;
6093 int i;
6094 int cpu = thiscpu->cpu_id;
6095
6096 for (i = 0; i <= topo.max_cpu_num; i++) {
6097 sprintf(path, "/sys/devices/system/cpu/cpu%d/node%i/cpulist", cpu, i);
6098 filep = fopen(path, "r");
6099 if (!filep)
6100 continue;
6101 fclose(filep);
6102 return i;
6103 }
6104 return -1;
6105 }
6106
parse_cpu_str(char * cpu_str,cpu_set_t * cpu_set,int cpu_set_size)6107 static int parse_cpu_str(char *cpu_str, cpu_set_t *cpu_set, int cpu_set_size)
6108 {
6109 unsigned int start, end;
6110 char *next = cpu_str;
6111
6112 while (next && *next) {
6113
6114 if (*next == '-') /* no negative cpu numbers */
6115 return 1;
6116
6117 if (*next == '\0' || *next == '\n')
6118 break;
6119
6120 start = strtoul(next, &next, 10);
6121
6122 if (start >= CPU_SUBSET_MAXCPUS)
6123 return 1;
6124 CPU_SET_S(start, cpu_set_size, cpu_set);
6125
6126 if (*next == '\0' || *next == '\n')
6127 break;
6128
6129 if (*next == ',') {
6130 next += 1;
6131 continue;
6132 }
6133
6134 if (*next == '-') {
6135 next += 1; /* start range */
6136 } else if (*next == '.') {
6137 next += 1;
6138 if (*next == '.')
6139 next += 1; /* start range */
6140 else
6141 return 1;
6142 }
6143
6144 end = strtoul(next, &next, 10);
6145 if (end <= start)
6146 return 1;
6147
6148 while (++start <= end) {
6149 if (start >= CPU_SUBSET_MAXCPUS)
6150 return 1;
6151 CPU_SET_S(start, cpu_set_size, cpu_set);
6152 }
6153
6154 if (*next == ',')
6155 next += 1;
6156 else if (*next != '\0' && *next != '\n')
6157 return 1;
6158 }
6159
6160 return 0;
6161 }
6162
set_thread_siblings(struct cpu_topology * thiscpu)6163 int set_thread_siblings(struct cpu_topology *thiscpu)
6164 {
6165 char path[80], character;
6166 FILE *filep;
6167 unsigned long map;
6168 int so, shift, sib_core;
6169 int cpu = thiscpu->cpu_id;
6170 int offset = topo.max_cpu_num + 1;
6171 size_t size;
6172 int thread_id = 0;
6173
6174 thiscpu->put_ids = CPU_ALLOC((topo.max_cpu_num + 1));
6175 if (thiscpu->ht_id < 0)
6176 thiscpu->ht_id = thread_id++;
6177 if (!thiscpu->put_ids)
6178 return -1;
6179
6180 size = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
6181 CPU_ZERO_S(size, thiscpu->put_ids);
6182
6183 sprintf(path, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", cpu);
6184 filep = fopen(path, "r");
6185
6186 if (!filep) {
6187 warnx("%s: open failed", path);
6188 return -1;
6189 }
6190 do {
6191 offset -= BITMASK_SIZE;
6192 if (fscanf(filep, "%lx%c", &map, &character) != 2)
6193 err(1, "%s: failed to parse file", path);
6194 for (shift = 0; shift < BITMASK_SIZE; shift++) {
6195 if ((map >> shift) & 0x1) {
6196 so = shift + offset;
6197 sib_core = get_core_id(so);
6198 if (sib_core == thiscpu->core_id) {
6199 CPU_SET_S(so, size, thiscpu->put_ids);
6200 if ((so != cpu) && (cpus[so].ht_id < 0)) {
6201 cpus[so].ht_id = thread_id;
6202 cpus[cpu].ht_sibling_cpu_id[thread_id] = so;
6203 if (debug)
6204 fprintf(stderr, "%s: cpu%d.ht_sibling_cpu_id[%d] = %d\n", __func__, cpu, thread_id, so);
6205 thread_id += 1;
6206 }
6207 }
6208 }
6209 }
6210 } while (character == ',');
6211 fclose(filep);
6212
6213 return CPU_COUNT_S(size, thiscpu->put_ids);
6214 }
6215
6216 /*
6217 * run func(thread, core, package) in topology order
6218 * skip non-present cpus
6219 */
6220
for_all_cpus_2(int (func)(struct thread_data *,struct core_data *,struct pkg_data *,struct thread_data *,struct core_data *,struct pkg_data *),struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base,struct thread_data * thread_base2,struct core_data * core_base2,struct pkg_data * pkg_base2)6221 int for_all_cpus_2(int (func) (struct thread_data *, struct core_data *,
6222 struct pkg_data *, struct thread_data *, struct core_data *,
6223 struct pkg_data *), struct thread_data *thread_base,
6224 struct core_data *core_base, struct pkg_data *pkg_base,
6225 struct thread_data *thread_base2, struct core_data *core_base2, struct pkg_data *pkg_base2)
6226 {
6227 int cpu, retval;
6228
6229 retval = 0;
6230
6231 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
6232 struct thread_data *t, *t2;
6233 struct core_data *c, *c2;
6234 struct pkg_data *p, *p2;
6235
6236 if (cpu_is_not_allowed(cpu))
6237 continue;
6238
6239 if (cpus[cpu].ht_id > 0) /* skip HT sibling */
6240 continue;
6241
6242 t = &thread_base[cpu];
6243 t2 = &thread_base2[cpu];
6244 c = &core_base[GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id)];
6245 c2 = &core_base2[GLOBAL_CORE_ID(cpus[cpu].core_id, cpus[cpu].package_id)];
6246 p = &pkg_base[cpus[cpu].package_id];
6247 p2 = &pkg_base2[cpus[cpu].package_id];
6248
6249 retval |= func(t, c, p, t2, c2, p2);
6250
6251 /* Handle HT sibling now */
6252 int i;
6253
6254 for (i = MAX_HT_ID; i > 0; --i) { /* ht_id 0 is self */
6255 if (cpus[cpu].ht_sibling_cpu_id[i] <= 0)
6256 continue;
6257 t = &thread_base[cpus[cpu].ht_sibling_cpu_id[i]];
6258 t2 = &thread_base2[cpus[cpu].ht_sibling_cpu_id[i]];
6259
6260 retval |= func(t, c, p, t2, c2, p2);
6261 }
6262 }
6263 return retval;
6264 }
6265
6266 /*
6267 * run func(cpu) on every cpu in /proc/stat
6268 * return max_cpu number
6269 */
for_all_proc_cpus(int (func)(int))6270 int for_all_proc_cpus(int (func) (int))
6271 {
6272 FILE *fp;
6273 int cpu_num;
6274 int retval;
6275
6276 fp = fopen_or_die(proc_stat, "r");
6277
6278 retval = fscanf(fp, "cpu %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n");
6279 if (retval != 0)
6280 err(1, "%s: failed to parse format", proc_stat);
6281
6282 while (1) {
6283 retval = fscanf(fp, "cpu%u %*d %*d %*d %*d %*d %*d %*d %*d %*d %*d\n", &cpu_num);
6284 if (retval != 1)
6285 break;
6286
6287 retval = func(cpu_num);
6288 if (retval) {
6289 fclose(fp);
6290 return (retval);
6291 }
6292 }
6293 fclose(fp);
6294 return 0;
6295 }
6296
6297 #define PATH_EFFECTIVE_CPUS "/sys/fs/cgroup/cpuset.cpus.effective"
6298
6299 static char cpu_effective_str[1024];
6300
update_effective_str(bool startup)6301 static int update_effective_str(bool startup)
6302 {
6303 FILE *fp;
6304 char *pos;
6305 char buf[1024];
6306 int ret;
6307
6308 if (cpu_effective_str[0] == '\0' && !startup)
6309 return 0;
6310
6311 fp = fopen(PATH_EFFECTIVE_CPUS, "r");
6312 if (!fp)
6313 return 0;
6314
6315 pos = fgets(buf, 1024, fp);
6316 if (!pos)
6317 err(1, "%s: file read failed", PATH_EFFECTIVE_CPUS);
6318
6319 fclose(fp);
6320
6321 ret = strncmp(cpu_effective_str, buf, 1024);
6322 if (!ret)
6323 return 0;
6324
6325 strncpy(cpu_effective_str, buf, 1024);
6326 return 1;
6327 }
6328
update_effective_set(bool startup)6329 static void update_effective_set(bool startup)
6330 {
6331 update_effective_str(startup);
6332
6333 if (parse_cpu_str(cpu_effective_str, cpu_effective_set, cpu_effective_setsize))
6334 err(1, "%s: cpu str malformat %s", PATH_EFFECTIVE_CPUS, cpu_effective_str);
6335 }
6336
6337 void linux_perf_init(void);
6338 void msr_perf_init(void);
6339 void rapl_perf_init(void);
6340 void cstate_perf_init(void);
6341 void perf_llc_init(void);
6342 void perf_l2_init(void);
6343 void added_perf_counters_init(void);
6344 void pmt_init(void);
6345
re_initialize(void)6346 void re_initialize(void)
6347 {
6348 free_all_buffers();
6349 setup_all_buffers(false);
6350 linux_perf_init();
6351 msr_perf_init();
6352 rapl_perf_init();
6353 cstate_perf_init();
6354 perf_llc_init();
6355 perf_l2_init();
6356 added_perf_counters_init();
6357 pmt_init();
6358 fprintf(outf, "turbostat: re-initialized with num_cpus %d, allowed_cpus %d\n", topo.num_cpus, topo.allowed_cpus);
6359 }
6360
set_max_cpu_num(void)6361 void set_max_cpu_num(void)
6362 {
6363 FILE *filep;
6364 int current_cpu;
6365 unsigned long dummy;
6366 char pathname[64];
6367
6368 current_cpu = sched_getcpu();
6369 if (current_cpu < 0)
6370 err(1, "cannot find calling cpu ID");
6371 sprintf(pathname, "/sys/devices/system/cpu/cpu%d/topology/thread_siblings", current_cpu);
6372
6373 filep = fopen_or_die(pathname, "r");
6374 topo.max_cpu_num = 0;
6375 while (fscanf(filep, "%lx,", &dummy) == 1)
6376 topo.max_cpu_num += BITMASK_SIZE;
6377 fclose(filep);
6378 topo.max_cpu_num--; /* 0 based */
6379 }
6380
6381 /*
6382 * count_cpus()
6383 * remember the last one seen, it will be the max
6384 */
count_cpus(int cpu)6385 int count_cpus(int cpu)
6386 {
6387 UNUSED(cpu);
6388
6389 topo.num_cpus++;
6390 return 0;
6391 }
6392
mark_cpu_present(int cpu)6393 int mark_cpu_present(int cpu)
6394 {
6395 CPU_SET_S(cpu, cpu_present_setsize, cpu_present_set);
6396 return 0;
6397 }
6398
clear_ht_id(int cpu)6399 int clear_ht_id(int cpu)
6400 {
6401 int i;
6402
6403 cpus[cpu].ht_id = -1;
6404 for (i = 0; i <= MAX_HT_ID; ++i)
6405 cpus[cpu].ht_sibling_cpu_id[i] = -1;
6406 return 0;
6407 }
6408
set_my_cpu_type(void)6409 int set_my_cpu_type(void)
6410 {
6411 unsigned int eax, ebx, ecx, edx;
6412 unsigned int max_level;
6413
6414 __cpuid(0, max_level, ebx, ecx, edx);
6415
6416 if (max_level < CPUID_LEAF_MODEL_ID)
6417 return 0;
6418
6419 __cpuid(CPUID_LEAF_MODEL_ID, eax, ebx, ecx, edx);
6420
6421 return (eax >> CPUID_LEAF_MODEL_ID_CORE_TYPE_SHIFT);
6422 }
6423
set_cpu_hybrid_type(int cpu)6424 int set_cpu_hybrid_type(int cpu)
6425 {
6426 if (cpu_migrate(cpu))
6427 return -1;
6428
6429 int type = set_my_cpu_type();
6430
6431 cpus[cpu].type = type;
6432 return 0;
6433 }
6434
6435 /*
6436 * snapshot_proc_interrupts()
6437 *
6438 * read and record summary of /proc/interrupts
6439 *
6440 * return 1 if config change requires a restart, else return 0
6441 */
snapshot_proc_interrupts(void)6442 int snapshot_proc_interrupts(void)
6443 {
6444 static FILE *fp;
6445 int column, retval;
6446
6447 if (fp == NULL)
6448 fp = fopen_or_die("/proc/interrupts", "r");
6449 else
6450 rewind(fp);
6451
6452 /* read 1st line of /proc/interrupts to get cpu* name for each column */
6453 for (column = 0; column < topo.num_cpus; ++column) {
6454 int cpu_number;
6455
6456 retval = fscanf(fp, " CPU%d", &cpu_number);
6457 if (retval != 1)
6458 break;
6459
6460 if (cpu_number > topo.max_cpu_num) {
6461 warn("/proc/interrupts: cpu%d: > %d", cpu_number, topo.max_cpu_num);
6462 return 1;
6463 }
6464
6465 irq_column_2_cpu[column] = cpu_number;
6466 irqs_per_cpu[cpu_number] = 0;
6467 nmi_per_cpu[cpu_number] = 0;
6468 }
6469
6470 /* read /proc/interrupt count lines and sum up irqs per cpu */
6471 while (1) {
6472 int column;
6473 char buf[64];
6474 int this_row_is_nmi = 0;
6475
6476 retval = fscanf(fp, " %s:", buf); /* irq# "N:" */
6477 if (retval != 1)
6478 break;
6479
6480 if (strncmp(buf, "NMI", strlen("NMI")) == 0)
6481 this_row_is_nmi = 1;
6482
6483 /* read the count per cpu */
6484 for (column = 0; column < topo.num_cpus; ++column) {
6485
6486 int cpu_number, irq_count;
6487
6488 retval = fscanf(fp, " %d", &irq_count);
6489
6490 if (retval != 1)
6491 break;
6492
6493 cpu_number = irq_column_2_cpu[column];
6494 irqs_per_cpu[cpu_number] += irq_count;
6495 if (this_row_is_nmi)
6496 nmi_per_cpu[cpu_number] += irq_count;
6497 }
6498 while (getc(fp) != '\n') ; /* flush interrupt description */
6499
6500 }
6501 return 0;
6502 }
6503
6504 /*
6505 * snapshot_graphics()
6506 *
6507 * record snapshot of specified graphics sysfs knob
6508 *
6509 * return 1 if config change requires a restart, else return 0
6510 */
snapshot_graphics(int idx)6511 int snapshot_graphics(int idx)
6512 {
6513 int retval;
6514
6515 rewind(gfx_info[idx].fp);
6516 fflush(gfx_info[idx].fp);
6517
6518 switch (idx) {
6519 case GFX_rc6:
6520 case SAM_mc6:
6521 retval = fscanf(gfx_info[idx].fp, "%lld", &gfx_info[idx].val_ull);
6522 if (retval != 1)
6523 err(1, "rc6");
6524 return 0;
6525 case GFX_MHz:
6526 case GFX_ACTMHz:
6527 case SAM_MHz:
6528 case SAM_ACTMHz:
6529 retval = fscanf(gfx_info[idx].fp, "%d", &gfx_info[idx].val);
6530 if (retval != 1)
6531 err(1, "MHz");
6532 return 0;
6533 default:
6534 return -EINVAL;
6535 }
6536 }
6537
6538 /*
6539 * snapshot_cpu_lpi()
6540 *
6541 * record snapshot of
6542 * /sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us
6543 */
snapshot_cpu_lpi_us(void)6544 int snapshot_cpu_lpi_us(void)
6545 {
6546 FILE *fp;
6547 int retval;
6548
6549 fp = fopen_or_die("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", "r");
6550
6551 retval = fscanf(fp, "%lld", &cpuidle_cur_cpu_lpi_us);
6552 if (retval != 1) {
6553 fprintf(stderr, "Disabling Low Power Idle CPU output\n");
6554 BIC_NOT_PRESENT(BIC_CPU_LPI);
6555 fclose(fp);
6556 return -1;
6557 }
6558
6559 fclose(fp);
6560
6561 return 0;
6562 }
6563
6564 /*
6565 * snapshot_sys_lpi()
6566 *
6567 * record snapshot of sys_lpi_file
6568 */
snapshot_sys_lpi_us(void)6569 int snapshot_sys_lpi_us(void)
6570 {
6571 FILE *fp;
6572 int retval;
6573
6574 fp = fopen_or_die(sys_lpi_file, "r");
6575
6576 retval = fscanf(fp, "%lld", &cpuidle_cur_sys_lpi_us);
6577 if (retval != 1) {
6578 fprintf(stderr, "Disabling Low Power Idle System output\n");
6579 BIC_NOT_PRESENT(BIC_SYS_LPI);
6580 fclose(fp);
6581 return -1;
6582 }
6583 fclose(fp);
6584
6585 return 0;
6586 }
6587
6588 /*
6589 * snapshot /proc and /sys files
6590 *
6591 * return 1 if configuration restart needed, else return 0
6592 */
snapshot_proc_sysfs_files(void)6593 int snapshot_proc_sysfs_files(void)
6594 {
6595 gettimeofday(&procsysfs_tv_begin, (struct timezone *)NULL);
6596
6597 if (DO_BIC(BIC_IRQ) || DO_BIC(BIC_NMI))
6598 if (snapshot_proc_interrupts())
6599 return 1;
6600
6601 if (DO_BIC(BIC_GFX_rc6))
6602 snapshot_graphics(GFX_rc6);
6603
6604 if (DO_BIC(BIC_GFXMHz))
6605 snapshot_graphics(GFX_MHz);
6606
6607 if (DO_BIC(BIC_GFXACTMHz))
6608 snapshot_graphics(GFX_ACTMHz);
6609
6610 if (DO_BIC(BIC_SAM_mc6))
6611 snapshot_graphics(SAM_mc6);
6612
6613 if (DO_BIC(BIC_SAMMHz))
6614 snapshot_graphics(SAM_MHz);
6615
6616 if (DO_BIC(BIC_SAMACTMHz))
6617 snapshot_graphics(SAM_ACTMHz);
6618
6619 if (DO_BIC(BIC_CPU_LPI))
6620 snapshot_cpu_lpi_us();
6621
6622 if (DO_BIC(BIC_SYS_LPI))
6623 snapshot_sys_lpi_us();
6624
6625 return 0;
6626 }
6627
6628 int exit_requested;
6629
signal_handler(int signal)6630 static void signal_handler(int signal)
6631 {
6632 switch (signal) {
6633 case SIGINT:
6634 exit_requested = 1;
6635 if (debug)
6636 fprintf(stderr, " SIGINT\n");
6637 break;
6638 case SIGUSR1:
6639 if (debug > 1)
6640 fprintf(stderr, "SIGUSR1\n");
6641 break;
6642 }
6643 }
6644
setup_signal_handler(void)6645 void setup_signal_handler(void)
6646 {
6647 struct sigaction sa;
6648
6649 memset(&sa, 0, sizeof(sa));
6650
6651 sa.sa_handler = &signal_handler;
6652
6653 if (sigaction(SIGINT, &sa, NULL) < 0)
6654 err(1, "sigaction SIGINT");
6655 if (sigaction(SIGUSR1, &sa, NULL) < 0)
6656 err(1, "sigaction SIGUSR1");
6657 }
6658
do_sleep(void)6659 void do_sleep(void)
6660 {
6661 struct timeval tout;
6662 struct timespec rest;
6663 fd_set readfds;
6664 int retval;
6665
6666 FD_ZERO(&readfds);
6667 FD_SET(0, &readfds);
6668
6669 if (ignore_stdin) {
6670 nanosleep(&interval_ts, NULL);
6671 return;
6672 }
6673
6674 tout = interval_tv;
6675 retval = select(1, &readfds, NULL, NULL, &tout);
6676
6677 if (retval == 1) {
6678 switch (getc(stdin)) {
6679 case 'q':
6680 exit_requested = 1;
6681 break;
6682 case EOF:
6683 /*
6684 * 'stdin' is a pipe closed on the other end. There
6685 * won't be any further input.
6686 */
6687 ignore_stdin = 1;
6688 /* Sleep the rest of the time */
6689 rest.tv_sec = (tout.tv_sec + tout.tv_usec / 1000000);
6690 rest.tv_nsec = (tout.tv_usec % 1000000) * 1000;
6691 nanosleep(&rest, NULL);
6692 }
6693 }
6694 }
6695
get_msr_sum(int cpu,off_t offset,unsigned long long * msr)6696 int get_msr_sum(int cpu, off_t offset, unsigned long long *msr)
6697 {
6698 int ret, idx;
6699 unsigned long long msr_cur, msr_last;
6700
6701 assert(!no_msr);
6702
6703 if (!per_cpu_msr_sum)
6704 return 1;
6705
6706 idx = offset_to_idx(offset);
6707 if (idx < 0)
6708 return idx;
6709 /* get_msr_sum() = sum + (get_msr() - last) */
6710 ret = get_msr(cpu, offset, &msr_cur);
6711 if (ret)
6712 return ret;
6713 msr_last = per_cpu_msr_sum[cpu].entries[idx].last;
6714 DELTA_WRAP32(msr_cur, msr_last);
6715 *msr = msr_last + per_cpu_msr_sum[cpu].entries[idx].sum;
6716
6717 return 0;
6718 }
6719
6720 timer_t timerid;
6721
6722 /* Timer callback, update the sum of MSRs periodically. */
update_msr_sum(PER_THREAD_PARAMS)6723 static int update_msr_sum(PER_THREAD_PARAMS)
6724 {
6725 int i, ret;
6726 int cpu = t->cpu_id;
6727
6728 UNUSED(c);
6729 UNUSED(p);
6730
6731 assert(!no_msr);
6732
6733 for (i = IDX_PKG_ENERGY; i < IDX_COUNT; i++) {
6734 unsigned long long msr_cur, msr_last;
6735 off_t offset;
6736
6737 if (!idx_valid(i))
6738 continue;
6739 offset = idx_to_offset(i);
6740 if (offset < 0)
6741 continue;
6742 ret = get_msr(cpu, offset, &msr_cur);
6743 if (ret) {
6744 fprintf(outf, "Can not update msr(0x%llx)\n", (unsigned long long)offset);
6745 continue;
6746 }
6747
6748 msr_last = per_cpu_msr_sum[cpu].entries[i].last;
6749 per_cpu_msr_sum[cpu].entries[i].last = msr_cur & 0xffffffff;
6750
6751 DELTA_WRAP32(msr_cur, msr_last);
6752 per_cpu_msr_sum[cpu].entries[i].sum += msr_last;
6753 }
6754 return 0;
6755 }
6756
msr_record_handler(union sigval v)6757 static void msr_record_handler(union sigval v)
6758 {
6759 UNUSED(v);
6760
6761 for_all_cpus(update_msr_sum, EVEN_COUNTERS);
6762 }
6763
msr_sum_record(void)6764 void msr_sum_record(void)
6765 {
6766 struct itimerspec its;
6767 struct sigevent sev;
6768
6769 per_cpu_msr_sum = calloc(topo.max_cpu_num + 1, sizeof(struct msr_sum_array));
6770 if (!per_cpu_msr_sum) {
6771 fprintf(outf, "Can not allocate memory for long time MSR.\n");
6772 return;
6773 }
6774 /*
6775 * Signal handler might be restricted, so use thread notifier instead.
6776 */
6777 memset(&sev, 0, sizeof(struct sigevent));
6778 sev.sigev_notify = SIGEV_THREAD;
6779 sev.sigev_notify_function = msr_record_handler;
6780
6781 sev.sigev_value.sival_ptr = &timerid;
6782 if (timer_create(CLOCK_REALTIME, &sev, &timerid) == -1) {
6783 fprintf(outf, "Can not create timer.\n");
6784 goto release_msr;
6785 }
6786
6787 its.it_value.tv_sec = 0;
6788 its.it_value.tv_nsec = 1;
6789 /*
6790 * A wraparound time has been calculated early.
6791 * Some sources state that the peak power for a
6792 * microprocessor is usually 1.5 times the TDP rating,
6793 * use 2 * TDP for safety.
6794 */
6795 its.it_interval.tv_sec = rapl_joule_counter_range / 2;
6796 its.it_interval.tv_nsec = 0;
6797
6798 if (timer_settime(timerid, 0, &its, NULL) == -1) {
6799 fprintf(outf, "Can not set timer.\n");
6800 goto release_timer;
6801 }
6802 return;
6803
6804 release_timer:
6805 timer_delete(timerid);
6806 release_msr:
6807 free(per_cpu_msr_sum);
6808 per_cpu_msr_sum = NULL;
6809 }
6810
6811 /*
6812 * set_my_sched_priority(pri)
6813 * return previous priority on success
6814 * return value < -20 on failure
6815 */
set_my_sched_priority(int priority)6816 int set_my_sched_priority(int priority)
6817 {
6818 int retval;
6819 int original_priority;
6820
6821 errno = 0;
6822 original_priority = getpriority(PRIO_PROCESS, 0);
6823 if (errno && (original_priority == -1))
6824 return -21;
6825
6826 retval = setpriority(PRIO_PROCESS, 0, priority);
6827 if (retval)
6828 return -21;
6829
6830 errno = 0;
6831 retval = getpriority(PRIO_PROCESS, 0);
6832 if (retval != priority)
6833 return -21;
6834
6835 return original_priority;
6836 }
6837
turbostat_loop()6838 void turbostat_loop()
6839 {
6840 int retval;
6841 int restarted = 0;
6842 unsigned int done_iters = 0;
6843
6844 setup_signal_handler();
6845
6846 /*
6847 * elevate own priority for interval mode
6848 *
6849 * ignore on error - we probably don't have permission to set it, but
6850 * it's not a big deal
6851 */
6852 set_my_sched_priority(-20);
6853
6854 restart:
6855 restarted++;
6856
6857 snapshot_proc_sysfs_files();
6858 retval = for_all_cpus(get_counters, EVEN_COUNTERS);
6859 first_counter_read = 0;
6860 if (retval < -1) {
6861 exit(retval);
6862 } else if (retval == -1) {
6863 if (restarted > 10) {
6864 exit(retval);
6865 }
6866 re_initialize();
6867 goto restart;
6868 }
6869 restarted = 0;
6870 done_iters = 0;
6871 gettimeofday(&tv_even, (struct timezone *)NULL);
6872
6873 while (1) {
6874 if (for_all_proc_cpus(cpu_is_not_present)) {
6875 re_initialize();
6876 goto restart;
6877 }
6878 if (update_effective_str(false)) {
6879 re_initialize();
6880 goto restart;
6881 }
6882 do_sleep();
6883 if (snapshot_proc_sysfs_files())
6884 goto restart;
6885 retval = for_all_cpus(get_counters, ODD_COUNTERS);
6886 if (retval < -1) {
6887 exit(retval);
6888 } else if (retval == -1) {
6889 re_initialize();
6890 goto restart;
6891 }
6892 gettimeofday(&tv_odd, (struct timezone *)NULL);
6893 timersub(&tv_odd, &tv_even, &tv_delta);
6894 if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS)) {
6895 re_initialize();
6896 goto restart;
6897 }
6898 delta_platform(&platform_counters_odd, &platform_counters_even);
6899 compute_average(EVEN_COUNTERS);
6900 format_all_counters(EVEN_COUNTERS);
6901 flush_output_stdout();
6902 if (exit_requested)
6903 break;
6904 if (num_iterations && ++done_iters >= num_iterations)
6905 break;
6906 do_sleep();
6907 if (snapshot_proc_sysfs_files())
6908 goto restart;
6909 retval = for_all_cpus(get_counters, EVEN_COUNTERS);
6910 if (retval < -1) {
6911 exit(retval);
6912 } else if (retval == -1) {
6913 re_initialize();
6914 goto restart;
6915 }
6916 gettimeofday(&tv_even, (struct timezone *)NULL);
6917 timersub(&tv_even, &tv_odd, &tv_delta);
6918 if (for_all_cpus_2(delta_cpu, EVEN_COUNTERS, ODD_COUNTERS)) {
6919 re_initialize();
6920 goto restart;
6921 }
6922 delta_platform(&platform_counters_even, &platform_counters_odd);
6923 compute_average(ODD_COUNTERS);
6924 format_all_counters(ODD_COUNTERS);
6925 flush_output_stdout();
6926 if (exit_requested)
6927 break;
6928 if (num_iterations && ++done_iters >= num_iterations)
6929 break;
6930 }
6931 }
6932
probe_dev_msr(void)6933 int probe_dev_msr(void)
6934 {
6935 struct stat sb;
6936 char pathname[32];
6937
6938 sprintf(pathname, "/dev/msr%d", master_cpu);
6939 return !stat(pathname, &sb);
6940 }
6941
probe_dev_cpu_msr(void)6942 int probe_dev_cpu_msr(void)
6943 {
6944 struct stat sb;
6945 char pathname[32];
6946
6947 sprintf(pathname, "/dev/cpu/%d/msr", master_cpu);
6948 return !stat(pathname, &sb);
6949 }
6950
probe_msr_driver(void)6951 int probe_msr_driver(void)
6952 {
6953 if (probe_dev_msr()) {
6954 use_android_msr_path = 1;
6955 return 1;
6956 }
6957 return probe_dev_cpu_msr();
6958 }
6959
check_msr_driver(void)6960 void check_msr_driver(void)
6961 {
6962 if (probe_msr_driver())
6963 return;
6964
6965 if (system("/sbin/modprobe msr > /dev/null 2>&1"))
6966 no_msr = 1;
6967
6968 if (!probe_msr_driver())
6969 no_msr = 1;
6970 }
6971
6972 /*
6973 * check for CAP_SYS_RAWIO
6974 * return 0 on success
6975 * return 1 on fail
6976 */
check_for_cap_sys_rawio(void)6977 int check_for_cap_sys_rawio(void)
6978 {
6979 cap_t caps;
6980 cap_flag_value_t cap_flag_value;
6981 int ret = 0;
6982
6983 caps = cap_get_proc();
6984 if (caps == NULL) {
6985 /*
6986 * CONFIG_MULTIUSER=n kernels have no cap_get_proc()
6987 * Allow them to continue and attempt to access MSRs
6988 */
6989 if (errno == ENOSYS)
6990 return 0;
6991
6992 return 1;
6993 }
6994
6995 if (cap_get_flag(caps, CAP_SYS_RAWIO, CAP_EFFECTIVE, &cap_flag_value)) {
6996 ret = 1;
6997 goto free_and_exit;
6998 }
6999
7000 if (cap_flag_value != CAP_SET) {
7001 ret = 1;
7002 goto free_and_exit;
7003 }
7004
7005 free_and_exit:
7006 if (cap_free(caps) == -1)
7007 err(-6, "cap_free");
7008
7009 return ret;
7010 }
7011
check_msr_permission(void)7012 void check_msr_permission(void)
7013 {
7014 int failed = 0;
7015 char pathname[32];
7016
7017 if (no_msr)
7018 return;
7019
7020 /* check for CAP_SYS_RAWIO */
7021 failed += check_for_cap_sys_rawio();
7022
7023 /* test file permissions */
7024 sprintf(pathname, use_android_msr_path ? "/dev/msr%d" : "/dev/cpu/%d/msr", master_cpu);
7025 if (euidaccess(pathname, R_OK)) {
7026 failed++;
7027 }
7028
7029 if (failed) {
7030 warnx("Failed to access %s. Some of the counters may not be available\n"
7031 "\tRun as root to enable them or use %s to disable the access explicitly", pathname, "--no-msr");
7032 no_msr = 1;
7033 }
7034 }
7035
probe_bclk(void)7036 void probe_bclk(void)
7037 {
7038 unsigned long long msr;
7039 unsigned int base_ratio;
7040
7041 if (!platform->has_nhm_msrs || no_msr)
7042 return;
7043
7044 if (platform->bclk_freq == BCLK_100MHZ)
7045 bclk = 100.00;
7046 else if (platform->bclk_freq == BCLK_133MHZ)
7047 bclk = 133.33;
7048 else if (platform->bclk_freq == BCLK_SLV)
7049 bclk = slm_bclk();
7050 else
7051 return;
7052
7053 get_msr(master_cpu, MSR_PLATFORM_INFO, &msr);
7054 base_ratio = (msr >> 8) & 0xFF;
7055
7056 base_hz = base_ratio * bclk * 1000000;
7057 has_base_hz = 1;
7058
7059 if (platform->enable_tsc_tweak)
7060 tsc_tweak = base_hz / tsc_hz;
7061 }
7062
remove_underbar(char * s)7063 static void remove_underbar(char *s)
7064 {
7065 char *to = s;
7066
7067 while (*s) {
7068 if (*s != '_')
7069 *to++ = *s;
7070 s++;
7071 }
7072
7073 *to = 0;
7074 }
7075
dump_turbo_ratio_info(void)7076 static void dump_turbo_ratio_info(void)
7077 {
7078 if (!has_turbo)
7079 return;
7080
7081 if (!platform->has_nhm_msrs || no_msr)
7082 return;
7083
7084 if (platform->trl_msrs & TRL_LIMIT2)
7085 dump_turbo_ratio_limit2();
7086
7087 if (platform->trl_msrs & TRL_LIMIT1)
7088 dump_turbo_ratio_limit1();
7089
7090 if (platform->trl_msrs & TRL_BASE) {
7091 dump_turbo_ratio_limits(MSR_TURBO_RATIO_LIMIT);
7092
7093 if (is_hybrid)
7094 dump_turbo_ratio_limits(MSR_SECONDARY_TURBO_RATIO_LIMIT);
7095 }
7096
7097 if (platform->trl_msrs & TRL_ATOM)
7098 dump_atom_turbo_ratio_limits();
7099
7100 if (platform->trl_msrs & TRL_KNL)
7101 dump_knl_turbo_ratio_limits();
7102
7103 if (platform->has_config_tdp)
7104 dump_config_tdp();
7105 }
7106
read_sysfs_int(char * path)7107 static int read_sysfs_int(char *path)
7108 {
7109 FILE *input;
7110 int retval = -1;
7111
7112 input = fopen(path, "r");
7113 if (input == NULL) {
7114 if (debug)
7115 fprintf(outf, "NSFOD %s\n", path);
7116 return (-1);
7117 }
7118 if (fscanf(input, "%d", &retval) != 1)
7119 err(1, "%s: failed to read int from file", path);
7120 fclose(input);
7121
7122 return (retval);
7123 }
7124
dump_sysfs_file(char * path)7125 static void dump_sysfs_file(char *path)
7126 {
7127 FILE *input;
7128 char cpuidle_buf[64];
7129
7130 input = fopen(path, "r");
7131 if (input == NULL) {
7132 if (debug)
7133 fprintf(outf, "NSFOD %s\n", path);
7134 return;
7135 }
7136 if (!fgets(cpuidle_buf, sizeof(cpuidle_buf), input))
7137 err(1, "%s: failed to read file", path);
7138 fclose(input);
7139
7140 fprintf(outf, "%s: %s", strrchr(path, '/') + 1, cpuidle_buf);
7141 }
7142
probe_intel_uncore_frequency_legacy(void)7143 static void probe_intel_uncore_frequency_legacy(void)
7144 {
7145 int i, j;
7146 char path[256];
7147
7148 for (i = 0; i < topo.num_packages; ++i) {
7149 for (j = 0; j <= topo.max_die_id; ++j) {
7150 int k, l;
7151 char path_base[128];
7152
7153 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/package_%02d_die_%02d", i, j);
7154
7155 sprintf(path, "%s/current_freq_khz", path_base);
7156 if (access(path, R_OK))
7157 continue;
7158
7159 BIC_PRESENT(BIC_UNCORE_MHZ);
7160
7161 if (quiet)
7162 return;
7163
7164 sprintf(path, "%s/min_freq_khz", path_base);
7165 k = read_sysfs_int(path);
7166 sprintf(path, "%s/max_freq_khz", path_base);
7167 l = read_sysfs_int(path);
7168 fprintf(outf, "Uncore Frequency package%d die%d: %d - %d MHz ", i, j, k / 1000, l / 1000);
7169
7170 sprintf(path, "%s/initial_min_freq_khz", path_base);
7171 k = read_sysfs_int(path);
7172 sprintf(path, "%s/initial_max_freq_khz", path_base);
7173 l = read_sysfs_int(path);
7174 fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
7175
7176 sprintf(path, "%s/current_freq_khz", path_base);
7177 k = read_sysfs_int(path);
7178 fprintf(outf, " %d MHz\n", k / 1000);
7179 }
7180 }
7181 }
7182
probe_intel_uncore_frequency_cluster(void)7183 static void probe_intel_uncore_frequency_cluster(void)
7184 {
7185 int i, uncore_max_id;
7186 char path[256];
7187 char path_base[128];
7188
7189 if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00/current_freq_khz", R_OK))
7190 return;
7191
7192 for (uncore_max_id = 0;; ++uncore_max_id) {
7193
7194 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", uncore_max_id);
7195
7196 /* uncore## start at 00 and skips no numbers, so stop upon first missing */
7197 if (access(path_base, R_OK)) {
7198 uncore_max_id -= 1;
7199 break;
7200 }
7201 }
7202 for (i = uncore_max_id; i >= 0; --i) {
7203 int k, l;
7204 int unc_pkg_id, domain_id, cluster_id;
7205 char name_buf[16];
7206
7207 sprintf(path_base, "/sys/devices/system/cpu/intel_uncore_frequency/uncore%02d", i);
7208
7209 if (access(path_base, R_OK))
7210 err(1, "%s: %s", __func__, path_base);
7211
7212 sprintf(path, "%s/package_id", path_base);
7213 unc_pkg_id = read_sysfs_int(path);
7214
7215 sprintf(path, "%s/domain_id", path_base);
7216 domain_id = read_sysfs_int(path);
7217
7218 sprintf(path, "%s/fabric_cluster_id", path_base);
7219 cluster_id = read_sysfs_int(path);
7220
7221 sprintf(path, "%s/current_freq_khz", path_base);
7222 sprintf(name_buf, "UMHz%d.%d", domain_id, cluster_id);
7223
7224 /*
7225 * Once add_couter() is called, that counter is always read
7226 * and reported -- So it is effectively (enabled & present).
7227 * Only call add_counter() here if legacy BIC_UNCORE_MHZ (UncMHz)
7228 * is (enabled). Since we are in this routine, we
7229 * know we will not probe and set (present) the legacy counter.
7230 *
7231 * This allows "--show/--hide UncMHz" to be effective for
7232 * the clustered MHz counters, as a group.
7233 */
7234 if BIC_IS_ENABLED
7235 (BIC_UNCORE_MHZ)
7236 add_counter(0, path, name_buf, 0, SCOPE_PACKAGE, COUNTER_K2M, FORMAT_AVERAGE, 0, unc_pkg_id);
7237
7238 if (quiet)
7239 continue;
7240
7241 sprintf(path, "%s/min_freq_khz", path_base);
7242 k = read_sysfs_int(path);
7243 sprintf(path, "%s/max_freq_khz", path_base);
7244 l = read_sysfs_int(path);
7245 fprintf(outf, "Uncore Frequency package%d domain%d cluster%d: %d - %d MHz ", unc_pkg_id, domain_id, cluster_id, k / 1000, l / 1000);
7246
7247 sprintf(path, "%s/initial_min_freq_khz", path_base);
7248 k = read_sysfs_int(path);
7249 sprintf(path, "%s/initial_max_freq_khz", path_base);
7250 l = read_sysfs_int(path);
7251 fprintf(outf, "(%d - %d MHz)", k / 1000, l / 1000);
7252
7253 sprintf(path, "%s/current_freq_khz", path_base);
7254 k = read_sysfs_int(path);
7255 fprintf(outf, " %d MHz\n", k / 1000);
7256 }
7257 }
7258
probe_intel_uncore_frequency(void)7259 static void probe_intel_uncore_frequency(void)
7260 {
7261 if (!genuine_intel)
7262 return;
7263
7264 if (access("/sys/devices/system/cpu/intel_uncore_frequency/uncore00", R_OK) == 0)
7265 probe_intel_uncore_frequency_cluster();
7266 else
7267 probe_intel_uncore_frequency_legacy();
7268 }
7269
set_graphics_fp(char * path,int idx)7270 static void set_graphics_fp(char *path, int idx)
7271 {
7272 if (!access(path, R_OK))
7273 gfx_info[idx].fp = fopen_or_die(path, "r");
7274 }
7275
7276 /* Enlarge this if there are /sys/class/drm/card2 ... */
7277 #define GFX_MAX_CARDS 2
7278
probe_graphics(void)7279 static void probe_graphics(void)
7280 {
7281 char path[PATH_MAX];
7282 int i;
7283
7284 /* Xe graphics sysfs knobs */
7285 if (!access("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", R_OK)) {
7286 FILE *fp;
7287 char buf[8];
7288 bool gt0_is_gt;
7289
7290 fp = fopen("/sys/class/drm/card0/device/tile0/gt0/gtidle/name", "r");
7291 if (!fp)
7292 goto next;
7293
7294 if (!fread(buf, sizeof(char), 7, fp)) {
7295 fclose(fp);
7296 goto next;
7297 }
7298 fclose(fp);
7299
7300 if (!strncmp(buf, "gt0-rc", strlen("gt0-rc")))
7301 gt0_is_gt = true;
7302 else if (!strncmp(buf, "gt0-mc", strlen("gt0-mc")))
7303 gt0_is_gt = false;
7304 else
7305 goto next;
7306
7307 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/gtidle/idle_residency_ms", gt0_is_gt ? GFX_rc6 : SAM_mc6);
7308
7309 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/cur_freq", gt0_is_gt ? GFX_MHz : SAM_MHz);
7310
7311 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt0/freq0/act_freq", gt0_is_gt ? GFX_ACTMHz : SAM_ACTMHz);
7312
7313 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/gtidle/idle_residency_ms", gt0_is_gt ? SAM_mc6 : GFX_rc6);
7314
7315 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/cur_freq", gt0_is_gt ? SAM_MHz : GFX_MHz);
7316
7317 set_graphics_fp("/sys/class/drm/card0/device/tile0/gt1/freq0/act_freq", gt0_is_gt ? SAM_ACTMHz : GFX_ACTMHz);
7318
7319 goto end;
7320 }
7321
7322 next:
7323 /* New i915 graphics sysfs knobs */
7324 for (i = 0; i < GFX_MAX_CARDS; i++) {
7325 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i);
7326 if (!access(path, R_OK))
7327 break;
7328 }
7329
7330 if (i == GFX_MAX_CARDS)
7331 goto legacy_i915;
7332
7333 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rc6_residency_ms", i);
7334 set_graphics_fp(path, GFX_rc6);
7335
7336 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rps_cur_freq_mhz", i);
7337 set_graphics_fp(path, GFX_MHz);
7338
7339 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt0/rps_act_freq_mhz", i);
7340 set_graphics_fp(path, GFX_ACTMHz);
7341
7342 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rc6_residency_ms", i);
7343 set_graphics_fp(path, SAM_mc6);
7344
7345 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rps_cur_freq_mhz", i);
7346 set_graphics_fp(path, SAM_MHz);
7347
7348 snprintf(path, PATH_MAX, "/sys/class/drm/card%d/gt/gt1/rps_act_freq_mhz", i);
7349 set_graphics_fp(path, SAM_ACTMHz);
7350
7351 goto end;
7352
7353 legacy_i915:
7354 /* Fall back to traditional i915 graphics sysfs knobs */
7355 set_graphics_fp("/sys/class/drm/card0/power/rc6_residency_ms", GFX_rc6);
7356
7357 set_graphics_fp("/sys/class/drm/card0/gt_cur_freq_mhz", GFX_MHz);
7358 if (!gfx_info[GFX_MHz].fp)
7359 set_graphics_fp("/sys/class/graphics/fb0/device/drm/card0/gt_cur_freq_mhz", GFX_MHz);
7360
7361 set_graphics_fp("/sys/class/drm/card0/gt_act_freq_mhz", GFX_ACTMHz);
7362 if (!gfx_info[GFX_ACTMHz].fp)
7363 set_graphics_fp("/sys/class/graphics/fb0/device/drm/card0/gt_act_freq_mhz", GFX_ACTMHz);
7364
7365 end:
7366 if (gfx_info[GFX_rc6].fp)
7367 BIC_PRESENT(BIC_GFX_rc6);
7368 if (gfx_info[GFX_MHz].fp)
7369 BIC_PRESENT(BIC_GFXMHz);
7370 if (gfx_info[GFX_ACTMHz].fp)
7371 BIC_PRESENT(BIC_GFXACTMHz);
7372 if (gfx_info[SAM_mc6].fp)
7373 BIC_PRESENT(BIC_SAM_mc6);
7374 if (gfx_info[SAM_MHz].fp)
7375 BIC_PRESENT(BIC_SAMMHz);
7376 if (gfx_info[SAM_ACTMHz].fp)
7377 BIC_PRESENT(BIC_SAMACTMHz);
7378 }
7379
dump_sysfs_cstate_config(void)7380 static void dump_sysfs_cstate_config(void)
7381 {
7382 char path[64];
7383 char name_buf[16];
7384 char desc[64];
7385 FILE *input;
7386 int state;
7387 char *sp;
7388
7389 if (access("/sys/devices/system/cpu/cpuidle", R_OK)) {
7390 fprintf(outf, "cpuidle not loaded\n");
7391 return;
7392 }
7393
7394 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_driver");
7395 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor");
7396 dump_sysfs_file("/sys/devices/system/cpu/cpuidle/current_governor_ro");
7397
7398 for (state = 0; state < 10; ++state) {
7399
7400 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state);
7401 input = fopen(path, "r");
7402 if (input == NULL)
7403 continue;
7404 if (!fgets(name_buf, sizeof(name_buf), input))
7405 err(1, "%s: failed to read file", path);
7406
7407 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
7408 sp = strchr(name_buf, '-');
7409 if (!sp)
7410 sp = strchrnul(name_buf, '\n');
7411 *sp = '\0';
7412 fclose(input);
7413
7414 remove_underbar(name_buf);
7415
7416 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/desc", master_cpu, state);
7417 input = fopen(path, "r");
7418 if (input == NULL)
7419 continue;
7420 if (!fgets(desc, sizeof(desc), input))
7421 err(1, "%s: failed to read file", path);
7422
7423 fprintf(outf, "cpu%d: %s: %s", master_cpu, name_buf, desc);
7424 fclose(input);
7425 }
7426 }
7427
dump_sysfs_pstate_config(void)7428 static void dump_sysfs_pstate_config(void)
7429 {
7430 char path[64];
7431 char driver_buf[64];
7432 char governor_buf[64];
7433 FILE *input;
7434 int turbo;
7435
7436 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_driver", master_cpu);
7437 input = fopen(path, "r");
7438 if (input == NULL) {
7439 fprintf(outf, "NSFOD %s\n", path);
7440 return;
7441 }
7442 if (!fgets(driver_buf, sizeof(driver_buf), input))
7443 err(1, "%s: failed to read file", path);
7444 fclose(input);
7445
7446 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpufreq/scaling_governor", master_cpu);
7447 input = fopen(path, "r");
7448 if (input == NULL) {
7449 fprintf(outf, "NSFOD %s\n", path);
7450 return;
7451 }
7452 if (!fgets(governor_buf, sizeof(governor_buf), input))
7453 err(1, "%s: failed to read file", path);
7454 fclose(input);
7455
7456 fprintf(outf, "cpu%d: cpufreq driver: %s", master_cpu, driver_buf);
7457 fprintf(outf, "cpu%d: cpufreq governor: %s", master_cpu, governor_buf);
7458
7459 sprintf(path, "/sys/devices/system/cpu/cpufreq/boost");
7460 input = fopen(path, "r");
7461 if (input != NULL) {
7462 if (fscanf(input, "%d", &turbo) != 1)
7463 err(1, "%s: failed to parse number from file", path);
7464 fprintf(outf, "cpufreq boost: %d\n", turbo);
7465 fclose(input);
7466 }
7467
7468 sprintf(path, "/sys/devices/system/cpu/intel_pstate/no_turbo");
7469 input = fopen(path, "r");
7470 if (input != NULL) {
7471 if (fscanf(input, "%d", &turbo) != 1)
7472 err(1, "%s: failed to parse number from file", path);
7473 fprintf(outf, "cpufreq intel_pstate no_turbo: %d\n", turbo);
7474 fclose(input);
7475 }
7476 }
7477
7478 /*
7479 * print_epb()
7480 * Decode the ENERGY_PERF_BIAS MSR
7481 */
print_epb(PER_THREAD_PARAMS)7482 int print_epb(PER_THREAD_PARAMS)
7483 {
7484 char *epb_string;
7485 int cpu, epb;
7486
7487 UNUSED(c);
7488 UNUSED(p);
7489
7490 if (!has_epb)
7491 return 0;
7492
7493 cpu = t->cpu_id;
7494
7495 /* EPB is per-package */
7496 if (!is_cpu_first_thread_in_package(t, c, p))
7497 return 0;
7498
7499 if (cpu_migrate(cpu)) {
7500 fprintf(outf, "print_epb: Could not migrate to CPU %d\n", cpu);
7501 return -1;
7502 }
7503
7504 epb = get_epb(cpu);
7505 if (epb < 0)
7506 return 0;
7507
7508 switch (epb) {
7509 case ENERGY_PERF_BIAS_PERFORMANCE:
7510 epb_string = "performance";
7511 break;
7512 case ENERGY_PERF_BIAS_NORMAL:
7513 epb_string = "balanced";
7514 break;
7515 case ENERGY_PERF_BIAS_POWERSAVE:
7516 epb_string = "powersave";
7517 break;
7518 default:
7519 epb_string = "custom";
7520 break;
7521 }
7522 fprintf(outf, "cpu%d: EPB: %d (%s)\n", cpu, epb, epb_string);
7523
7524 return 0;
7525 }
7526
7527 /*
7528 * print_hwp()
7529 * Decode the MSR_HWP_CAPABILITIES
7530 */
print_hwp(PER_THREAD_PARAMS)7531 int print_hwp(PER_THREAD_PARAMS)
7532 {
7533 unsigned long long msr;
7534 int cpu;
7535
7536 UNUSED(c);
7537 UNUSED(p);
7538
7539 if (no_msr)
7540 return 0;
7541
7542 if (!has_hwp)
7543 return 0;
7544
7545 cpu = t->cpu_id;
7546
7547 /* MSR_HWP_CAPABILITIES is per-package */
7548 if (!is_cpu_first_thread_in_package(t, c, p))
7549 return 0;
7550
7551 if (cpu_migrate(cpu)) {
7552 fprintf(outf, "print_hwp: Could not migrate to CPU %d\n", cpu);
7553 return -1;
7554 }
7555
7556 if (get_msr(cpu, MSR_PM_ENABLE, &msr))
7557 return 0;
7558
7559 fprintf(outf, "cpu%d: MSR_PM_ENABLE: 0x%08llx (%sHWP)\n", cpu, msr, (msr & (1 << 0)) ? "" : "No-");
7560
7561 /* MSR_PM_ENABLE[1] == 1 if HWP is enabled and MSRs visible */
7562 if ((msr & (1 << 0)) == 0)
7563 return 0;
7564
7565 if (get_msr(cpu, MSR_HWP_CAPABILITIES, &msr))
7566 return 0;
7567
7568 fprintf(outf, "cpu%d: MSR_HWP_CAPABILITIES: 0x%08llx "
7569 "(high %d guar %d eff %d low %d)\n",
7570 cpu, msr,
7571 (unsigned int)HWP_HIGHEST_PERF(msr),
7572 (unsigned int)HWP_GUARANTEED_PERF(msr), (unsigned int)HWP_MOSTEFFICIENT_PERF(msr), (unsigned int)HWP_LOWEST_PERF(msr));
7573
7574 if (get_msr(cpu, MSR_HWP_REQUEST, &msr))
7575 return 0;
7576
7577 fprintf(outf, "cpu%d: MSR_HWP_REQUEST: 0x%08llx "
7578 "(min %d max %d des %d epp 0x%x window 0x%x pkg 0x%x)\n",
7579 cpu, msr,
7580 (unsigned int)(((msr) >> 0) & 0xff),
7581 (unsigned int)(((msr) >> 8) & 0xff),
7582 (unsigned int)(((msr) >> 16) & 0xff),
7583 (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3), (unsigned int)(((msr) >> 42) & 0x1));
7584
7585 if (has_hwp_pkg) {
7586 if (get_msr(cpu, MSR_HWP_REQUEST_PKG, &msr))
7587 return 0;
7588
7589 fprintf(outf, "cpu%d: MSR_HWP_REQUEST_PKG: 0x%08llx "
7590 "(min %d max %d des %d epp 0x%x window 0x%x)\n",
7591 cpu, msr,
7592 (unsigned int)(((msr) >> 0) & 0xff),
7593 (unsigned int)(((msr) >> 8) & 0xff),
7594 (unsigned int)(((msr) >> 16) & 0xff), (unsigned int)(((msr) >> 24) & 0xff), (unsigned int)(((msr) >> 32) & 0xff3));
7595 }
7596 if (has_hwp_notify) {
7597 if (get_msr(cpu, MSR_HWP_INTERRUPT, &msr))
7598 return 0;
7599
7600 fprintf(outf, "cpu%d: MSR_HWP_INTERRUPT: 0x%08llx "
7601 "(%s_Guaranteed_Perf_Change, %s_Excursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "EN" : "Dis", ((msr) & 0x2) ? "EN" : "Dis");
7602 }
7603 if (get_msr(cpu, MSR_HWP_STATUS, &msr))
7604 return 0;
7605
7606 fprintf(outf, "cpu%d: MSR_HWP_STATUS: 0x%08llx "
7607 "(%sGuaranteed_Perf_Change, %sExcursion_Min)\n", cpu, msr, ((msr) & 0x1) ? "" : "No-", ((msr) & 0x4) ? "" : "No-");
7608
7609 return 0;
7610 }
7611
7612 /*
7613 * print_perf_limit()
7614 */
print_perf_limit(PER_THREAD_PARAMS)7615 int print_perf_limit(PER_THREAD_PARAMS)
7616 {
7617 unsigned long long msr;
7618 int cpu;
7619
7620 UNUSED(c);
7621 UNUSED(p);
7622
7623 if (no_msr)
7624 return 0;
7625
7626 cpu = t->cpu_id;
7627
7628 /* per-package */
7629 if (!is_cpu_first_thread_in_package(t, c, p))
7630 return 0;
7631
7632 if (cpu_migrate(cpu)) {
7633 fprintf(outf, "print_perf_limit: Could not migrate to CPU %d\n", cpu);
7634 return -1;
7635 }
7636
7637 if (platform->plr_msrs & PLR_CORE) {
7638 get_msr(cpu, MSR_CORE_PERF_LIMIT_REASONS, &msr);
7639 fprintf(outf, "cpu%d: MSR_CORE_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7640 fprintf(outf, " (Active: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)",
7641 (msr & 1 << 15) ? "bit15, " : "",
7642 (msr & 1 << 14) ? "bit14, " : "",
7643 (msr & 1 << 13) ? "Transitions, " : "",
7644 (msr & 1 << 12) ? "MultiCoreTurbo, " : "",
7645 (msr & 1 << 11) ? "PkgPwrL2, " : "",
7646 (msr & 1 << 10) ? "PkgPwrL1, " : "",
7647 (msr & 1 << 9) ? "CorePwr, " : "",
7648 (msr & 1 << 8) ? "Amps, " : "",
7649 (msr & 1 << 6) ? "VR-Therm, " : "",
7650 (msr & 1 << 5) ? "Auto-HWP, " : "",
7651 (msr & 1 << 4) ? "Graphics, " : "",
7652 (msr & 1 << 2) ? "bit2, " : "", (msr & 1 << 1) ? "ThermStatus, " : "", (msr & 1 << 0) ? "PROCHOT, " : "");
7653 fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s%s%s%s%s%s%s)\n",
7654 (msr & 1 << 31) ? "bit31, " : "",
7655 (msr & 1 << 30) ? "bit30, " : "",
7656 (msr & 1 << 29) ? "Transitions, " : "",
7657 (msr & 1 << 28) ? "MultiCoreTurbo, " : "",
7658 (msr & 1 << 27) ? "PkgPwrL2, " : "",
7659 (msr & 1 << 26) ? "PkgPwrL1, " : "",
7660 (msr & 1 << 25) ? "CorePwr, " : "",
7661 (msr & 1 << 24) ? "Amps, " : "",
7662 (msr & 1 << 22) ? "VR-Therm, " : "",
7663 (msr & 1 << 21) ? "Auto-HWP, " : "",
7664 (msr & 1 << 20) ? "Graphics, " : "",
7665 (msr & 1 << 18) ? "bit18, " : "", (msr & 1 << 17) ? "ThermStatus, " : "", (msr & 1 << 16) ? "PROCHOT, " : "");
7666
7667 }
7668 if (platform->plr_msrs & PLR_GFX) {
7669 get_msr(cpu, MSR_GFX_PERF_LIMIT_REASONS, &msr);
7670 fprintf(outf, "cpu%d: MSR_GFX_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7671 fprintf(outf, " (Active: %s%s%s%s%s%s%s%s)",
7672 (msr & 1 << 0) ? "PROCHOT, " : "",
7673 (msr & 1 << 1) ? "ThermStatus, " : "",
7674 (msr & 1 << 4) ? "Graphics, " : "",
7675 (msr & 1 << 6) ? "VR-Therm, " : "",
7676 (msr & 1 << 8) ? "Amps, " : "",
7677 (msr & 1 << 9) ? "GFXPwr, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
7678 fprintf(outf, " (Logged: %s%s%s%s%s%s%s%s)\n",
7679 (msr & 1 << 16) ? "PROCHOT, " : "",
7680 (msr & 1 << 17) ? "ThermStatus, " : "",
7681 (msr & 1 << 20) ? "Graphics, " : "",
7682 (msr & 1 << 22) ? "VR-Therm, " : "",
7683 (msr & 1 << 24) ? "Amps, " : "",
7684 (msr & 1 << 25) ? "GFXPwr, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
7685 }
7686 if (platform->plr_msrs & PLR_RING) {
7687 get_msr(cpu, MSR_RING_PERF_LIMIT_REASONS, &msr);
7688 fprintf(outf, "cpu%d: MSR_RING_PERF_LIMIT_REASONS, 0x%08llx", cpu, msr);
7689 fprintf(outf, " (Active: %s%s%s%s%s%s)",
7690 (msr & 1 << 0) ? "PROCHOT, " : "",
7691 (msr & 1 << 1) ? "ThermStatus, " : "",
7692 (msr & 1 << 6) ? "VR-Therm, " : "",
7693 (msr & 1 << 8) ? "Amps, " : "", (msr & 1 << 10) ? "PkgPwrL1, " : "", (msr & 1 << 11) ? "PkgPwrL2, " : "");
7694 fprintf(outf, " (Logged: %s%s%s%s%s%s)\n",
7695 (msr & 1 << 16) ? "PROCHOT, " : "",
7696 (msr & 1 << 17) ? "ThermStatus, " : "",
7697 (msr & 1 << 22) ? "VR-Therm, " : "",
7698 (msr & 1 << 24) ? "Amps, " : "", (msr & 1 << 26) ? "PkgPwrL1, " : "", (msr & 1 << 27) ? "PkgPwrL2, " : "");
7699 }
7700 return 0;
7701 }
7702
7703 #define RAPL_POWER_GRANULARITY 0x7FFF /* 15 bit power granularity */
7704 #define RAPL_TIME_GRANULARITY 0x3F /* 6 bit time granularity */
7705
get_quirk_tdp(void)7706 double get_quirk_tdp(void)
7707 {
7708 if (platform->rapl_quirk_tdp)
7709 return platform->rapl_quirk_tdp;
7710
7711 return 135.0;
7712 }
7713
get_tdp_intel(void)7714 double get_tdp_intel(void)
7715 {
7716 unsigned long long msr;
7717
7718 if (valid_rapl_msrs & RAPL_PKG_POWER_INFO)
7719 if (!get_msr(master_cpu, MSR_PKG_POWER_INFO, &msr))
7720 return ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units;
7721 return get_quirk_tdp();
7722 }
7723
get_tdp_amd(void)7724 double get_tdp_amd(void)
7725 {
7726 return get_quirk_tdp();
7727 }
7728
rapl_probe_intel(void)7729 void rapl_probe_intel(void)
7730 {
7731 unsigned long long msr;
7732 unsigned int time_unit;
7733 double tdp;
7734
7735 if (rapl_joules) {
7736 CLR_BIC(BIC_SysWatt, &bic_enabled);
7737 CLR_BIC(BIC_PkgWatt, &bic_enabled);
7738 CLR_BIC(BIC_CorWatt, &bic_enabled);
7739 CLR_BIC(BIC_RAMWatt, &bic_enabled);
7740 CLR_BIC(BIC_GFXWatt, &bic_enabled);
7741 } else {
7742 CLR_BIC(BIC_Sys_J, &bic_enabled);
7743 CLR_BIC(BIC_Pkg_J, &bic_enabled);
7744 CLR_BIC(BIC_Cor_J, &bic_enabled);
7745 CLR_BIC(BIC_RAM_J, &bic_enabled);
7746 CLR_BIC(BIC_GFX_J, &bic_enabled);
7747 }
7748
7749 if (!valid_rapl_msrs || no_msr)
7750 return;
7751
7752 if (!(valid_rapl_msrs & RAPL_PKG_PERF_STATUS))
7753 CLR_BIC(BIC_PKG__, &bic_enabled);
7754 if (!(valid_rapl_msrs & RAPL_DRAM_PERF_STATUS))
7755 CLR_BIC(BIC_RAM__, &bic_enabled);
7756
7757 /* units on package 0, verify later other packages match */
7758 if (get_msr(master_cpu, MSR_RAPL_POWER_UNIT, &msr))
7759 return;
7760
7761 rapl_power_units = 1.0 / (1 << (msr & 0xF));
7762 if (platform->has_rapl_divisor)
7763 rapl_energy_units = 1.0 * (1 << (msr >> 8 & 0x1F)) / 1000000;
7764 else
7765 rapl_energy_units = 1.0 / (1 << (msr >> 8 & 0x1F));
7766
7767 if (platform->has_fixed_rapl_unit)
7768 rapl_dram_energy_units = (15.3 / 1000000);
7769 else
7770 rapl_dram_energy_units = rapl_energy_units;
7771
7772 if (platform->has_fixed_rapl_psys_unit)
7773 rapl_psys_energy_units = 1.0;
7774 else
7775 rapl_psys_energy_units = rapl_energy_units;
7776
7777 time_unit = msr >> 16 & 0xF;
7778 if (time_unit == 0)
7779 time_unit = 0xA;
7780
7781 rapl_time_units = 1.0 / (1 << (time_unit));
7782
7783 tdp = get_tdp_intel();
7784
7785 rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7786 if (!quiet)
7787 fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7788 }
7789
rapl_probe_amd(void)7790 void rapl_probe_amd(void)
7791 {
7792 unsigned long long msr;
7793 double tdp;
7794
7795 if (rapl_joules) {
7796 CLR_BIC(BIC_SysWatt, &bic_enabled);
7797 CLR_BIC(BIC_CorWatt, &bic_enabled);
7798 } else {
7799 CLR_BIC(BIC_Pkg_J, &bic_enabled);
7800 CLR_BIC(BIC_Cor_J, &bic_enabled);
7801 }
7802
7803 if (!valid_rapl_msrs || no_msr)
7804 return;
7805
7806 if (get_msr(master_cpu, MSR_RAPL_PWR_UNIT, &msr))
7807 return;
7808
7809 rapl_time_units = ldexp(1.0, -(msr >> 16 & 0xf));
7810 rapl_energy_units = ldexp(1.0, -(msr >> 8 & 0x1f));
7811 rapl_power_units = ldexp(1.0, -(msr & 0xf));
7812
7813 tdp = get_tdp_amd();
7814
7815 rapl_joule_counter_range = 0xFFFFFFFF * rapl_energy_units / tdp;
7816 if (!quiet)
7817 fprintf(outf, "RAPL: %.0f sec. Joule Counter Range, at %.0f Watts\n", rapl_joule_counter_range, tdp);
7818 }
7819
print_power_limit_msr(int cpu,unsigned long long msr,char * label)7820 void print_power_limit_msr(int cpu, unsigned long long msr, char *label)
7821 {
7822 fprintf(outf, "cpu%d: %s: %sabled (%0.3f Watts, %f sec, clamp %sabled)\n",
7823 cpu, label,
7824 ((msr >> 15) & 1) ? "EN" : "DIS",
7825 ((msr >> 0) & 0x7FFF) * rapl_power_units,
7826 (1.0 + (((msr >> 22) & 0x3) / 4.0)) * (1 << ((msr >> 17) & 0x1F)) * rapl_time_units, (((msr >> 16) & 1) ? "EN" : "DIS"));
7827
7828 return;
7829 }
7830
fread_int(char * path,int * val)7831 static int fread_int(char *path, int *val)
7832 {
7833 FILE *filep;
7834 int ret;
7835
7836 filep = fopen(path, "r");
7837 if (!filep)
7838 return -1;
7839
7840 ret = fscanf(filep, "%d", val);
7841 fclose(filep);
7842 return ret;
7843 }
7844
fread_ull(char * path,unsigned long long * val)7845 static int fread_ull(char *path, unsigned long long *val)
7846 {
7847 FILE *filep;
7848 int ret;
7849
7850 filep = fopen(path, "r");
7851 if (!filep)
7852 return -1;
7853
7854 ret = fscanf(filep, "%llu", val);
7855 fclose(filep);
7856 return ret;
7857 }
7858
fread_str(char * path,char * buf,int size)7859 static int fread_str(char *path, char *buf, int size)
7860 {
7861 FILE *filep;
7862 int ret;
7863 char *cp;
7864
7865 filep = fopen(path, "r");
7866 if (!filep)
7867 return -1;
7868
7869 ret = fread(buf, 1, size, filep);
7870 fclose(filep);
7871
7872 /* replace '\n' with '\0' */
7873 cp = strchr(buf, '\n');
7874 if (cp != NULL)
7875 *cp = '\0';
7876
7877 return ret;
7878 }
7879
7880 #define PATH_RAPL_SYSFS "/sys/class/powercap"
7881
dump_one_domain(char * domain_path)7882 static int dump_one_domain(char *domain_path)
7883 {
7884 char path[PATH_MAX];
7885 char str[PATH_MAX];
7886 unsigned long long val;
7887 int constraint;
7888 int enable;
7889 int ret;
7890
7891 snprintf(path, PATH_MAX, "%s/name", domain_path);
7892 ret = fread_str(path, str, PATH_MAX);
7893 if (ret <= 0)
7894 return -1;
7895
7896 fprintf(outf, "%s: %s", domain_path + strlen(PATH_RAPL_SYSFS) + 1, str);
7897
7898 snprintf(path, PATH_MAX, "%s/enabled", domain_path);
7899 ret = fread_int(path, &enable);
7900 if (ret <= 0)
7901 return -1;
7902
7903 if (!enable) {
7904 fputs(" disabled\n", outf);
7905 return 0;
7906 }
7907
7908 for (constraint = 0;; constraint++) {
7909 snprintf(path, PATH_MAX, "%s/constraint_%d_time_window_us", domain_path, constraint);
7910 ret = fread_ull(path, &val);
7911 if (ret <= 0)
7912 break;
7913
7914 if (val > 1000000)
7915 fprintf(outf, " %0.1fs", (double)val / 1000000);
7916 else if (val > 1000)
7917 fprintf(outf, " %0.1fms", (double)val / 1000);
7918 else
7919 fprintf(outf, " %0.1fus", (double)val);
7920
7921 snprintf(path, PATH_MAX, "%s/constraint_%d_power_limit_uw", domain_path, constraint);
7922 ret = fread_ull(path, &val);
7923 if (ret > 0 && val)
7924 fprintf(outf, ":%lluW", val / 1000000);
7925
7926 snprintf(path, PATH_MAX, "%s/constraint_%d_max_power_uw", domain_path, constraint);
7927 ret = fread_ull(path, &val);
7928 if (ret > 0 && val)
7929 fprintf(outf, ",max:%lluW", val / 1000000);
7930 }
7931 fputc('\n', outf);
7932
7933 return 0;
7934 }
7935
print_rapl_sysfs(void)7936 static int print_rapl_sysfs(void)
7937 {
7938 DIR *dir, *cdir;
7939 struct dirent *entry, *centry;
7940 char path[PATH_MAX];
7941 char str[PATH_MAX];
7942
7943 if ((dir = opendir(PATH_RAPL_SYSFS)) == NULL) {
7944 warn("open %s failed", PATH_RAPL_SYSFS);
7945 return 1;
7946 }
7947
7948 while ((entry = readdir(dir)) != NULL) {
7949 if (strlen(entry->d_name) > 100)
7950 continue;
7951
7952 if (strncmp(entry->d_name, "intel-rapl", strlen("intel-rapl")))
7953 continue;
7954
7955 snprintf(path, PATH_MAX, "%s/%s/name", PATH_RAPL_SYSFS, entry->d_name);
7956
7957 /* Parse top level domains first, including package and psys */
7958 fread_str(path, str, PATH_MAX);
7959 if (strncmp(str, "package", strlen("package")) && strncmp(str, "psys", strlen("psys")))
7960 continue;
7961
7962 snprintf(path, PATH_MAX, "%s/%s", PATH_RAPL_SYSFS, entry->d_name);
7963 if ((cdir = opendir(path)) == NULL) {
7964 perror("opendir() error");
7965 return 1;
7966 }
7967
7968 dump_one_domain(path);
7969
7970 while ((centry = readdir(cdir)) != NULL) {
7971 if (strncmp(centry->d_name, "intel-rapl", strlen("intel-rapl")))
7972 continue;
7973 snprintf(path, PATH_MAX, "%s/%s/%s", PATH_RAPL_SYSFS, entry->d_name, centry->d_name);
7974 dump_one_domain(path);
7975 }
7976 closedir(cdir);
7977 }
7978
7979 closedir(dir);
7980 return 0;
7981 }
7982
print_rapl(PER_THREAD_PARAMS)7983 int print_rapl(PER_THREAD_PARAMS)
7984 {
7985 unsigned long long msr;
7986 const char *msr_name;
7987 int cpu;
7988
7989 UNUSED(c);
7990 UNUSED(p);
7991
7992 if (!valid_rapl_msrs)
7993 return 0;
7994
7995 /* RAPL counters are per package, so print only for 1st thread/package */
7996 if (!is_cpu_first_thread_in_package(t, c, p))
7997 return 0;
7998
7999 cpu = t->cpu_id;
8000 if (cpu_migrate(cpu)) {
8001 fprintf(outf, "print_rapl: Could not migrate to CPU %d\n", cpu);
8002 return -1;
8003 }
8004
8005 if (valid_rapl_msrs & RAPL_AMD_F17H) {
8006 msr_name = "MSR_RAPL_PWR_UNIT";
8007 if (get_msr(cpu, MSR_RAPL_PWR_UNIT, &msr))
8008 return -1;
8009 } else {
8010 msr_name = "MSR_RAPL_POWER_UNIT";
8011 if (get_msr(cpu, MSR_RAPL_POWER_UNIT, &msr))
8012 return -1;
8013 }
8014
8015 fprintf(outf, "cpu%d: %s: 0x%08llx (%f Watts, %f Joules, %f sec.)\n", cpu, msr_name, msr, rapl_power_units, rapl_energy_units, rapl_time_units);
8016
8017 if (valid_rapl_msrs & RAPL_PKG_POWER_INFO) {
8018
8019 if (get_msr(cpu, MSR_PKG_POWER_INFO, &msr))
8020 return -5;
8021
8022 fprintf(outf, "cpu%d: MSR_PKG_POWER_INFO: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
8023 cpu, msr,
8024 ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
8025 ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
8026 ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
8027
8028 }
8029 if (valid_rapl_msrs & RAPL_PKG) {
8030
8031 if (get_msr(cpu, MSR_PKG_POWER_LIMIT, &msr))
8032 return -9;
8033
8034 fprintf(outf, "cpu%d: MSR_PKG_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 63) & 1 ? "" : "UN");
8035
8036 print_power_limit_msr(cpu, msr, "PKG Limit #1");
8037 fprintf(outf, "cpu%d: PKG Limit #2: %sabled (%0.3f Watts, %f* sec, clamp %sabled)\n",
8038 cpu,
8039 ((msr >> 47) & 1) ? "EN" : "DIS",
8040 ((msr >> 32) & 0x7FFF) * rapl_power_units,
8041 (1.0 + (((msr >> 54) & 0x3) / 4.0)) * (1 << ((msr >> 49) & 0x1F)) * rapl_time_units, ((msr >> 48) & 1) ? "EN" : "DIS");
8042
8043 if (get_msr(cpu, MSR_VR_CURRENT_CONFIG, &msr))
8044 return -9;
8045
8046 fprintf(outf, "cpu%d: MSR_VR_CURRENT_CONFIG: 0x%08llx\n", cpu, msr);
8047 fprintf(outf, "cpu%d: PKG Limit #4: %f Watts (%slocked)\n", cpu, ((msr >> 0) & 0x1FFF) * rapl_power_units, (msr >> 31) & 1 ? "" : "UN");
8048 }
8049
8050 if (valid_rapl_msrs & RAPL_DRAM_POWER_INFO) {
8051 if (get_msr(cpu, MSR_DRAM_POWER_INFO, &msr))
8052 return -6;
8053
8054 fprintf(outf, "cpu%d: MSR_DRAM_POWER_INFO,: 0x%08llx (%.0f W TDP, RAPL %.0f - %.0f W, %f sec.)\n",
8055 cpu, msr,
8056 ((msr >> 0) & RAPL_POWER_GRANULARITY) * rapl_power_units,
8057 ((msr >> 16) & RAPL_POWER_GRANULARITY) * rapl_power_units,
8058 ((msr >> 32) & RAPL_POWER_GRANULARITY) * rapl_power_units, ((msr >> 48) & RAPL_TIME_GRANULARITY) * rapl_time_units);
8059 }
8060 if (valid_rapl_msrs & RAPL_DRAM) {
8061 if (get_msr(cpu, MSR_DRAM_POWER_LIMIT, &msr))
8062 return -9;
8063 fprintf(outf, "cpu%d: MSR_DRAM_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
8064
8065 print_power_limit_msr(cpu, msr, "DRAM Limit");
8066 }
8067 if (valid_rapl_msrs & RAPL_CORE_POLICY) {
8068 if (get_msr(cpu, MSR_PP0_POLICY, &msr))
8069 return -7;
8070
8071 fprintf(outf, "cpu%d: MSR_PP0_POLICY: %lld\n", cpu, msr & 0xF);
8072 }
8073 if (valid_rapl_msrs & RAPL_CORE_POWER_LIMIT) {
8074 if (get_msr(cpu, MSR_PP0_POWER_LIMIT, &msr))
8075 return -9;
8076 fprintf(outf, "cpu%d: MSR_PP0_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
8077 print_power_limit_msr(cpu, msr, "Cores Limit");
8078 }
8079 if (valid_rapl_msrs & RAPL_GFX) {
8080 if (get_msr(cpu, MSR_PP1_POLICY, &msr))
8081 return -8;
8082
8083 fprintf(outf, "cpu%d: MSR_PP1_POLICY: %lld\n", cpu, msr & 0xF);
8084
8085 if (get_msr(cpu, MSR_PP1_POWER_LIMIT, &msr))
8086 return -9;
8087 fprintf(outf, "cpu%d: MSR_PP1_POWER_LIMIT: 0x%08llx (%slocked)\n", cpu, msr, (msr >> 31) & 1 ? "" : "UN");
8088 print_power_limit_msr(cpu, msr, "GFX Limit");
8089 }
8090 return 0;
8091 }
8092
8093 /*
8094 * probe_rapl_msrs
8095 *
8096 * initialize global valid_rapl_msrs to platform->plat_rapl_msrs
8097 * only if PKG_ENERGY counter is enumerated and reads non-zero
8098 */
probe_rapl_msrs(void)8099 void probe_rapl_msrs(void)
8100 {
8101 int ret;
8102 off_t offset;
8103 unsigned long long msr_value;
8104
8105 if (no_msr)
8106 return;
8107
8108 if ((platform->plat_rapl_msrs & (RAPL_PKG | RAPL_AMD_F17H)) == 0)
8109 return;
8110
8111 offset = idx_to_offset(IDX_PKG_ENERGY);
8112 if (offset < 0)
8113 return;
8114
8115 ret = get_msr(master_cpu, offset, &msr_value);
8116 if (ret) {
8117 if (debug)
8118 fprintf(outf, "Can not read RAPL_PKG_ENERGY MSR(0x%llx)\n", (unsigned long long)offset);
8119 return;
8120 }
8121 if (msr_value == 0) {
8122 if (debug)
8123 fprintf(outf, "RAPL_PKG_ENERGY MSR(0x%llx) == ZERO: disabling all RAPL MSRs\n", (unsigned long long)offset);
8124 return;
8125 }
8126
8127 valid_rapl_msrs = platform->plat_rapl_msrs; /* success */
8128 }
8129
8130 /*
8131 * probe_rapl()
8132 *
8133 * sets rapl_power_units, rapl_energy_units, rapl_time_units
8134 */
probe_rapl(void)8135 void probe_rapl(void)
8136 {
8137 probe_rapl_msrs();
8138
8139 if (genuine_intel)
8140 rapl_probe_intel();
8141 if (authentic_amd || hygon_genuine)
8142 rapl_probe_amd();
8143
8144 if (quiet)
8145 return;
8146
8147 print_rapl_sysfs();
8148
8149 if (!valid_rapl_msrs || no_msr)
8150 return;
8151
8152 for_all_cpus(print_rapl, ODD_COUNTERS);
8153 }
8154
8155 /*
8156 * MSR_IA32_TEMPERATURE_TARGET indicates the temperature where
8157 * the Thermal Control Circuit (TCC) activates.
8158 * This is usually equal to tjMax.
8159 *
8160 * Older processors do not have this MSR, so there we guess,
8161 * but also allow cmdline over-ride with -T.
8162 *
8163 * Several MSR temperature values are in units of degrees-C
8164 * below this value, including the Digital Thermal Sensor (DTS),
8165 * Package Thermal Management Sensor (PTM), and thermal event thresholds.
8166 */
set_temperature_target(PER_THREAD_PARAMS)8167 int set_temperature_target(PER_THREAD_PARAMS)
8168 {
8169 unsigned long long msr;
8170 unsigned int tcc_default, tcc_offset;
8171 int cpu;
8172
8173 UNUSED(c);
8174 UNUSED(p);
8175
8176 /* tj_max is used only for dts or ptm */
8177 if (!(do_dts || do_ptm))
8178 return 0;
8179
8180 /* this is a per-package concept */
8181 if (!is_cpu_first_thread_in_package(t, c, p))
8182 return 0;
8183
8184 cpu = t->cpu_id;
8185 if (cpu_migrate(cpu)) {
8186 fprintf(outf, "Could not migrate to CPU %d\n", cpu);
8187 return -1;
8188 }
8189
8190 if (tj_max_override != 0) {
8191 tj_max = tj_max_override;
8192 fprintf(outf, "cpu%d: Using cmdline TCC Target (%d C)\n", cpu, tj_max);
8193 return 0;
8194 }
8195
8196 /* Temperature Target MSR is Nehalem and newer only */
8197 if (!platform->has_nhm_msrs || no_msr)
8198 goto guess;
8199
8200 if (get_msr(master_cpu, MSR_IA32_TEMPERATURE_TARGET, &msr))
8201 goto guess;
8202
8203 tcc_default = (msr >> 16) & 0xFF;
8204
8205 if (!quiet) {
8206 int bits = platform->tcc_offset_bits;
8207 unsigned long long enabled = 0;
8208
8209 if (bits && !get_msr(master_cpu, MSR_PLATFORM_INFO, &enabled))
8210 enabled = (enabled >> 30) & 1;
8211
8212 if (bits && enabled) {
8213 tcc_offset = (msr >> 24) & GENMASK(bits - 1, 0);
8214 fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C) (%d default - %d offset)\n",
8215 cpu, msr, tcc_default - tcc_offset, tcc_default, tcc_offset);
8216 } else {
8217 fprintf(outf, "cpu%d: MSR_IA32_TEMPERATURE_TARGET: 0x%08llx (%d C)\n", cpu, msr, tcc_default);
8218 }
8219 }
8220
8221 if (!tcc_default)
8222 goto guess;
8223
8224 tj_max = tcc_default;
8225
8226 return 0;
8227
8228 guess:
8229 tj_max = TJMAX_DEFAULT;
8230 fprintf(outf, "cpu%d: Guessing tjMax %d C, Please use -T to specify\n", cpu, tj_max);
8231
8232 return 0;
8233 }
8234
print_thermal(PER_THREAD_PARAMS)8235 int print_thermal(PER_THREAD_PARAMS)
8236 {
8237 unsigned long long msr;
8238 unsigned int dts, dts2;
8239 int cpu;
8240
8241 UNUSED(c);
8242 UNUSED(p);
8243
8244 if (no_msr)
8245 return 0;
8246
8247 if (!(do_dts || do_ptm))
8248 return 0;
8249
8250 cpu = t->cpu_id;
8251
8252 /* DTS is per-core, no need to print for each thread */
8253 if (!is_cpu_first_thread_in_core(t, c))
8254 return 0;
8255
8256 if (cpu_migrate(cpu)) {
8257 fprintf(outf, "print_thermal: Could not migrate to CPU %d\n", cpu);
8258 return -1;
8259 }
8260
8261 if (do_ptm && is_cpu_first_core_in_package(t, p)) {
8262 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_STATUS, &msr))
8263 return 0;
8264
8265 dts = (msr >> 16) & 0x7F;
8266 fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_STATUS: 0x%08llx (%d C)\n", cpu, msr, tj_max - dts);
8267
8268 if (get_msr(cpu, MSR_IA32_PACKAGE_THERM_INTERRUPT, &msr))
8269 return 0;
8270
8271 dts = (msr >> 16) & 0x7F;
8272 dts2 = (msr >> 8) & 0x7F;
8273 fprintf(outf, "cpu%d: MSR_IA32_PACKAGE_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
8274 }
8275
8276 if (do_dts && debug) {
8277 unsigned int resolution;
8278
8279 if (get_msr(cpu, MSR_IA32_THERM_STATUS, &msr))
8280 return 0;
8281
8282 dts = (msr >> 16) & 0x7F;
8283 resolution = (msr >> 27) & 0xF;
8284 fprintf(outf, "cpu%d: MSR_IA32_THERM_STATUS: 0x%08llx (%d C +/- %d)\n", cpu, msr, tj_max - dts, resolution);
8285
8286 if (get_msr(cpu, MSR_IA32_THERM_INTERRUPT, &msr))
8287 return 0;
8288
8289 dts = (msr >> 16) & 0x7F;
8290 dts2 = (msr >> 8) & 0x7F;
8291 fprintf(outf, "cpu%d: MSR_IA32_THERM_INTERRUPT: 0x%08llx (%d C, %d C)\n", cpu, msr, tj_max - dts, tj_max - dts2);
8292 }
8293
8294 return 0;
8295 }
8296
probe_thermal(void)8297 void probe_thermal(void)
8298 {
8299 if (!access("/sys/devices/system/cpu/cpu0/thermal_throttle/core_throttle_count", R_OK))
8300 BIC_PRESENT(BIC_CORE_THROT_CNT);
8301 else
8302 BIC_NOT_PRESENT(BIC_CORE_THROT_CNT);
8303
8304 for_all_cpus(set_temperature_target, ODD_COUNTERS);
8305
8306 if (quiet)
8307 return;
8308
8309 for_all_cpus(print_thermal, ODD_COUNTERS);
8310 }
8311
get_cpu_type(PER_THREAD_PARAMS)8312 int get_cpu_type(PER_THREAD_PARAMS)
8313 {
8314 unsigned int eax, ebx, ecx, edx;
8315
8316 UNUSED(c);
8317 UNUSED(p);
8318
8319 if (!genuine_intel)
8320 return 0;
8321
8322 if (cpu_migrate(t->cpu_id)) {
8323 fprintf(outf, "Could not migrate to CPU %d\n", t->cpu_id);
8324 return -1;
8325 }
8326
8327 if (max_level < 0x1a)
8328 return 0;
8329
8330 __cpuid(0x1a, eax, ebx, ecx, edx);
8331 eax = (eax >> 24) & 0xFF;
8332 if (eax == 0x20)
8333 t->is_atom = true;
8334 return 0;
8335 }
8336
decode_feature_control_msr(void)8337 void decode_feature_control_msr(void)
8338 {
8339 unsigned long long msr;
8340
8341 if (no_msr)
8342 return;
8343
8344 if (quiet)
8345 return;
8346
8347 if (!get_msr(master_cpu, MSR_IA32_FEAT_CTL, &msr))
8348 fprintf(outf, "cpu%d: MSR_IA32_FEATURE_CONTROL: 0x%08llx (%sLocked %s)\n",
8349 master_cpu, msr, msr & FEAT_CTL_LOCKED ? "" : "UN-", msr & (1 << 18) ? "SGX" : "");
8350 }
8351
decode_misc_enable_msr(void)8352 void decode_misc_enable_msr(void)
8353 {
8354 unsigned long long msr;
8355
8356 if (no_msr)
8357 return;
8358
8359 if (!genuine_intel)
8360 return;
8361
8362 if (!get_msr(master_cpu, MSR_IA32_MISC_ENABLE, &msr))
8363 fprintf(outf, "cpu%d: MSR_IA32_MISC_ENABLE: 0x%08llx (%sTCC %sEIST %sMWAIT %sPREFETCH %sTURBO)\n",
8364 master_cpu, msr,
8365 msr & MSR_IA32_MISC_ENABLE_TM1 ? "" : "No-",
8366 msr & MSR_IA32_MISC_ENABLE_ENHANCED_SPEEDSTEP ? "" : "No-",
8367 msr & MSR_IA32_MISC_ENABLE_MWAIT ? "" : "No-",
8368 msr & MSR_IA32_MISC_ENABLE_PREFETCH_DISABLE ? "No-" : "", msr & MSR_IA32_MISC_ENABLE_TURBO_DISABLE ? "No-" : "");
8369 }
8370
decode_misc_feature_control(void)8371 void decode_misc_feature_control(void)
8372 {
8373 unsigned long long msr;
8374
8375 if (no_msr)
8376 return;
8377
8378 if (!platform->has_msr_misc_feature_control)
8379 return;
8380
8381 if (!get_msr(master_cpu, MSR_MISC_FEATURE_CONTROL, &msr))
8382 fprintf(outf,
8383 "cpu%d: MSR_MISC_FEATURE_CONTROL: 0x%08llx (%sL2-Prefetch %sL2-Prefetch-pair %sL1-Prefetch %sL1-IP-Prefetch)\n",
8384 master_cpu, msr, msr & (0 << 0) ? "No-" : "", msr & (1 << 0) ? "No-" : "", msr & (2 << 0) ? "No-" : "", msr & (3 << 0) ? "No-" : "");
8385 }
8386
8387 /*
8388 * Decode MSR_MISC_PWR_MGMT
8389 *
8390 * Decode the bits according to the Nehalem documentation
8391 * bit[0] seems to continue to have same meaning going forward
8392 * bit[1] less so...
8393 */
decode_misc_pwr_mgmt_msr(void)8394 void decode_misc_pwr_mgmt_msr(void)
8395 {
8396 unsigned long long msr;
8397
8398 if (no_msr)
8399 return;
8400
8401 if (!platform->has_msr_misc_pwr_mgmt)
8402 return;
8403
8404 if (!get_msr(master_cpu, MSR_MISC_PWR_MGMT, &msr))
8405 fprintf(outf, "cpu%d: MSR_MISC_PWR_MGMT: 0x%08llx (%sable-EIST_Coordination %sable-EPB %sable-OOB)\n",
8406 master_cpu, msr, msr & (1 << 0) ? "DIS" : "EN", msr & (1 << 1) ? "EN" : "DIS", msr & (1 << 8) ? "EN" : "DIS");
8407 }
8408
8409 /*
8410 * Decode MSR_CC6_DEMOTION_POLICY_CONFIG, MSR_MC6_DEMOTION_POLICY_CONFIG
8411 *
8412 * This MSRs are present on Silvermont processors,
8413 * Intel Atom processor E3000 series (Baytrail), and friends.
8414 */
decode_c6_demotion_policy_msr(void)8415 void decode_c6_demotion_policy_msr(void)
8416 {
8417 unsigned long long msr;
8418
8419 if (no_msr)
8420 return;
8421
8422 if (!platform->has_msr_c6_demotion_policy_config)
8423 return;
8424
8425 if (!get_msr(master_cpu, MSR_CC6_DEMOTION_POLICY_CONFIG, &msr))
8426 fprintf(outf, "cpu%d: MSR_CC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-CC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
8427
8428 if (!get_msr(master_cpu, MSR_MC6_DEMOTION_POLICY_CONFIG, &msr))
8429 fprintf(outf, "cpu%d: MSR_MC6_DEMOTION_POLICY_CONFIG: 0x%08llx (%sable-MC6-Demotion)\n", master_cpu, msr, msr & (1 << 0) ? "EN" : "DIS");
8430 }
8431
print_dev_latency(void)8432 void print_dev_latency(void)
8433 {
8434 char *path = "/dev/cpu_dma_latency";
8435 int fd;
8436 int value;
8437 int retval;
8438
8439 fd = open(path, O_RDONLY);
8440 if (fd < 0) {
8441 if (debug)
8442 warnx("Read %s failed", path);
8443 return;
8444 }
8445
8446 retval = read(fd, (void *)&value, sizeof(int));
8447 if (retval != sizeof(int)) {
8448 warn("read failed %s", path);
8449 close(fd);
8450 return;
8451 }
8452 fprintf(outf, "/dev/cpu_dma_latency: %d usec (%s)\n", value, value == 2000000000 ? "default" : "constrained");
8453
8454 close(fd);
8455 }
8456
has_perf_instr_count_access(void)8457 static int has_perf_instr_count_access(void)
8458 {
8459 int fd;
8460
8461 if (no_perf)
8462 return 0;
8463
8464 fd = open_perf_counter(master_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_INSTRUCTIONS, -1, 0);
8465 if (fd != -1)
8466 close(fd);
8467
8468 if (fd == -1)
8469 warnx("Failed to access %s. Some of the counters may not be available\n"
8470 "\tRun as root to enable them or use %s to disable the access explicitly", "perf instructions retired counter",
8471 "'--hide IPC' or '--no-perf'");
8472
8473 return (fd != -1);
8474 }
8475
add_rapl_perf_counter(int cpu,struct rapl_counter_info_t * rci,const struct rapl_counter_arch_info * cai,double * scale_,enum rapl_unit * unit_)8476 int add_rapl_perf_counter(int cpu, struct rapl_counter_info_t *rci, const struct rapl_counter_arch_info *cai, double *scale_, enum rapl_unit *unit_)
8477 {
8478 int ret = -1;
8479
8480 if (no_perf)
8481 return -1;
8482
8483 if (!cai->perf_name)
8484 return -1;
8485
8486 const double scale = read_perf_scale(cai->perf_subsys, cai->perf_name);
8487
8488 if (scale == 0.0)
8489 goto end;
8490
8491 const enum rapl_unit unit = read_perf_rapl_unit(cai->perf_subsys, cai->perf_name);
8492
8493 if (unit == RAPL_UNIT_INVALID)
8494 goto end;
8495
8496 const unsigned int rapl_type = read_perf_type(cai->perf_subsys);
8497 const unsigned int rapl_energy_pkg_config = read_perf_config(cai->perf_subsys, cai->perf_name);
8498
8499 ret = open_perf_counter(cpu, rapl_type, rapl_energy_pkg_config, rci->fd_perf, PERF_FORMAT_GROUP);
8500 if (ret == -1)
8501 goto end;
8502
8503 /* If it's the first counter opened, make it a group descriptor */
8504 if (rci->fd_perf == -1)
8505 rci->fd_perf = ret;
8506
8507 *scale_ = scale;
8508 *unit_ = unit;
8509
8510 end:
8511 if (debug >= 2)
8512 fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
8513
8514 return ret;
8515 }
8516
8517 char cpuset_buf[1024];
initialize_cpu_set_from_sysfs(cpu_set_t * cpu_set,char * sysfs_path,char * sysfs_file)8518 int initialize_cpu_set_from_sysfs(cpu_set_t *cpu_set, char *sysfs_path, char *sysfs_file)
8519 {
8520 FILE *fp;
8521 char path[128];
8522
8523 if (snprintf(path, 128, "%s/%s", sysfs_path, sysfs_file) > 128)
8524 err(-1, "%s %s", sysfs_path, sysfs_file);
8525
8526 fp = fopen(path, "r");
8527 if (!fp) {
8528 warn("open %s", path);
8529 return -1;
8530 }
8531 if (fread(cpuset_buf, sizeof(char), 1024, fp) == 0) {
8532 warn("read %s", sysfs_path);
8533 goto err;
8534 }
8535 if (parse_cpu_str(cpuset_buf, cpu_set, cpu_possible_setsize)) {
8536 warnx("%s: cpu str malformat %s\n", sysfs_path, cpu_effective_str);
8537 goto err;
8538 }
8539 return 0;
8540
8541 err:
8542 fclose(fp);
8543 return -1;
8544 }
8545
print_cpu_set(char * s,cpu_set_t * set)8546 void print_cpu_set(char *s, cpu_set_t *set)
8547 {
8548 int i;
8549
8550 assert(MAX_BIC < CPU_SETSIZE);
8551
8552 printf("%s:", s);
8553
8554 for (i = 0; i <= topo.max_cpu_num; ++i)
8555 if (CPU_ISSET(i, set))
8556 printf(" %d", i);
8557 putchar('\n');
8558 }
8559
linux_perf_init_hybrid_cpus(void)8560 void linux_perf_init_hybrid_cpus(void)
8561 {
8562 char *perf_cpu_pcore_path = "/sys/devices/cpu_core";
8563 char *perf_cpu_ecore_path = "/sys/devices/cpu_atom";
8564 char *perf_cpu_lcore_path = "/sys/devices/cpu_lowpower";
8565 char path[128];
8566
8567 if (!access(perf_cpu_pcore_path, F_OK)) {
8568 perf_pcore_set = CPU_ALLOC((topo.max_cpu_num + 1));
8569 if (perf_pcore_set == NULL)
8570 err(3, "CPU_ALLOC");
8571 CPU_ZERO_S(cpu_possible_setsize, perf_pcore_set);
8572 initialize_cpu_set_from_sysfs(perf_pcore_set, perf_cpu_pcore_path, "cpus");
8573 if (debug)
8574 print_cpu_set("perf pcores", perf_pcore_set);
8575 sprintf(path, "%s/%s", perf_cpu_pcore_path, "type");
8576 perf_pmu_types.pcore = snapshot_sysfs_counter(path);
8577 }
8578
8579 if (!access(perf_cpu_ecore_path, F_OK)) {
8580 perf_ecore_set = CPU_ALLOC((topo.max_cpu_num + 1));
8581 if (perf_ecore_set == NULL)
8582 err(3, "CPU_ALLOC");
8583 CPU_ZERO_S(cpu_possible_setsize, perf_ecore_set);
8584 initialize_cpu_set_from_sysfs(perf_ecore_set, perf_cpu_ecore_path, "cpus");
8585 if (debug)
8586 print_cpu_set("perf ecores", perf_ecore_set);
8587 sprintf(path, "%s/%s", perf_cpu_ecore_path, "type");
8588 perf_pmu_types.ecore = snapshot_sysfs_counter(path);
8589 }
8590
8591 if (!access(perf_cpu_lcore_path, F_OK)) {
8592 perf_lcore_set = CPU_ALLOC((topo.max_cpu_num + 1));
8593 if (perf_lcore_set == NULL)
8594 err(3, "CPU_ALLOC");
8595 CPU_ZERO_S(cpu_possible_setsize, perf_lcore_set);
8596 initialize_cpu_set_from_sysfs(perf_lcore_set, perf_cpu_lcore_path, "cpus");
8597 if (debug)
8598 print_cpu_set("perf lcores", perf_lcore_set);
8599 sprintf(path, "%s/%s", perf_cpu_lcore_path, "type");
8600 perf_pmu_types.lcore = snapshot_sysfs_counter(path);
8601 }
8602 }
8603
8604 /*
8605 * Linux-perf related initialization
8606 */
linux_perf_init(void)8607 void linux_perf_init(void)
8608 {
8609 char path[128];
8610 char *perf_cpu_path = "/sys/devices/cpu";
8611
8612 if (access("/proc/sys/kernel/perf_event_paranoid", F_OK))
8613 return;
8614
8615 if (!access(perf_cpu_path, F_OK)) {
8616 sprintf(path, "%s/%s", perf_cpu_path, "type");
8617 perf_pmu_types.uniform = snapshot_sysfs_counter(path);
8618 } else {
8619 linux_perf_init_hybrid_cpus();
8620 }
8621
8622 if (BIC_IS_ENABLED(BIC_IPC) && cpuid_has_aperf_mperf) {
8623 fd_instr_count_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8624 if (fd_instr_count_percpu == NULL)
8625 err(-1, "calloc fd_instr_count_percpu");
8626 }
8627 if (BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT)) {
8628 fd_llc_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8629 if (fd_llc_percpu == NULL)
8630 err(-1, "calloc fd_llc_percpu");
8631 }
8632 if (BIC_IS_ENABLED(BIC_L2_MRPS) || BIC_IS_ENABLED(BIC_L2_HIT)) {
8633 fd_l2_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
8634 if (fd_l2_percpu == NULL)
8635 err(-1, "calloc fd_l2_percpu");
8636 }
8637 }
8638
rapl_perf_init(void)8639 void rapl_perf_init(void)
8640 {
8641 const unsigned int num_domains = get_rapl_num_domains();
8642 bool *domain_visited = calloc(num_domains, sizeof(bool));
8643
8644 rapl_counter_info_perdomain = calloc(num_domains, sizeof(*rapl_counter_info_perdomain));
8645 if (rapl_counter_info_perdomain == NULL)
8646 err(-1, "calloc rapl_counter_info_percpu");
8647 rapl_counter_info_perdomain_size = num_domains;
8648
8649 /*
8650 * Initialize rapl_counter_info_percpu
8651 */
8652 for (unsigned int domain_id = 0; domain_id < num_domains; ++domain_id) {
8653 struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[domain_id];
8654
8655 rci->fd_perf = -1;
8656 for (size_t i = 0; i < NUM_RAPL_COUNTERS; ++i) {
8657 rci->data[i] = 0;
8658 rci->source[i] = COUNTER_SOURCE_NONE;
8659 }
8660 }
8661
8662 /*
8663 * Open/probe the counters
8664 * If can't get it via perf, fallback to MSR
8665 */
8666 for (size_t i = 0; i < ARRAY_SIZE(rapl_counter_arch_infos); ++i) {
8667
8668 const struct rapl_counter_arch_info *const cai = &rapl_counter_arch_infos[i];
8669 bool has_counter = 0;
8670 double scale;
8671 enum rapl_unit unit;
8672 unsigned int next_domain;
8673
8674 if (!BIC_IS_ENABLED(cai->bic_number))
8675 continue;
8676
8677 memset(domain_visited, 0, num_domains * sizeof(*domain_visited));
8678
8679 for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
8680
8681 if (cpu_is_not_allowed(cpu))
8682 continue;
8683
8684 /* Skip already seen and handled RAPL domains */
8685 next_domain = get_rapl_domain_id(cpu);
8686
8687 assert(next_domain < num_domains);
8688
8689 if (domain_visited[next_domain])
8690 continue;
8691
8692 domain_visited[next_domain] = 1;
8693
8694 if ((cai->flags & RAPL_COUNTER_FLAG_PLATFORM_COUNTER) && (cpu != master_cpu))
8695 continue;
8696
8697 struct rapl_counter_info_t *rci = &rapl_counter_info_perdomain[next_domain];
8698
8699 /*
8700 * rapl_counter_arch_infos[] can have multiple entries describing the same
8701 * counter, due to the difference from different platforms/Vendors.
8702 * E.g. rapl_counter_arch_infos[0] and rapl_counter_arch_infos[1] share the
8703 * same perf_subsys and perf_name, but with different MSR address.
8704 * rapl_counter_arch_infos[0] is for Intel and rapl_counter_arch_infos[1]
8705 * is for AMD.
8706 * In this case, it is possible that multiple rapl_counter_arch_infos[]
8707 * entries are probed just because their perf/msr is duplicate and valid.
8708 *
8709 * Thus need a check to avoid re-probe the same counters.
8710 */
8711 if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE)
8712 break;
8713
8714 /* Use perf API for this counter */
8715 if (add_rapl_perf_counter(cpu, rci, cai, &scale, &unit) != -1) {
8716 rci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8717 rci->scale[cai->rci_index] = scale * cai->compat_scale;
8718 rci->unit[cai->rci_index] = unit;
8719 rci->flags[cai->rci_index] = cai->flags;
8720
8721 /* Use MSR for this counter */
8722 } else if (add_rapl_msr_counter(cpu, cai) >= 0) {
8723 rci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8724 rci->msr[cai->rci_index] = cai->msr;
8725 rci->msr_mask[cai->rci_index] = cai->msr_mask;
8726 rci->msr_shift[cai->rci_index] = cai->msr_shift;
8727 rci->unit[cai->rci_index] = RAPL_UNIT_JOULES;
8728 rci->scale[cai->rci_index] = *cai->platform_rapl_msr_scale * cai->compat_scale;
8729 rci->flags[cai->rci_index] = cai->flags;
8730 }
8731
8732 if (rci->source[cai->rci_index] != COUNTER_SOURCE_NONE)
8733 has_counter = 1;
8734 }
8735
8736 /* If any CPU has access to the counter, make it present */
8737 if (has_counter)
8738 BIC_PRESENT(cai->bic_number);
8739 }
8740
8741 free(domain_visited);
8742 }
8743
8744 /* Assumes msr_counter_info is populated */
has_amperf_access(void)8745 static int has_amperf_access(void)
8746 {
8747 return cpuid_has_aperf_mperf && msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].present && msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].present;
8748 }
8749
get_cstate_perf_group_fd(struct cstate_counter_info_t * cci,const char * group_name)8750 int *get_cstate_perf_group_fd(struct cstate_counter_info_t *cci, const char *group_name)
8751 {
8752 if (strcmp(group_name, "cstate_core") == 0)
8753 return &cci->fd_perf_core;
8754
8755 if (strcmp(group_name, "cstate_pkg") == 0)
8756 return &cci->fd_perf_pkg;
8757
8758 return NULL;
8759 }
8760
add_cstate_perf_counter(int cpu,struct cstate_counter_info_t * cci,const struct cstate_counter_arch_info * cai)8761 int add_cstate_perf_counter(int cpu, struct cstate_counter_info_t *cci, const struct cstate_counter_arch_info *cai)
8762 {
8763 int ret = -1;
8764
8765 if (no_perf)
8766 return -1;
8767
8768 if (!cai->perf_name)
8769 return -1;
8770
8771 int *pfd_group = get_cstate_perf_group_fd(cci, cai->perf_subsys);
8772
8773 if (pfd_group == NULL)
8774 goto end;
8775
8776 const unsigned int type = read_perf_type(cai->perf_subsys);
8777 const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name);
8778
8779 ret = open_perf_counter(cpu, type, config, *pfd_group, PERF_FORMAT_GROUP);
8780
8781 if (ret == -1)
8782 goto end;
8783
8784 /* If it's the first counter opened, make it a group descriptor */
8785 if (*pfd_group == -1)
8786 *pfd_group = ret;
8787
8788 end:
8789 if (debug >= 2)
8790 fprintf(stderr, "%s: %d (cpu: %d)\n", __func__, ret, cpu);
8791
8792 return ret;
8793 }
8794
add_msr_perf_counter(int cpu,struct msr_counter_info_t * cci,const struct msr_counter_arch_info * cai)8795 int add_msr_perf_counter(int cpu, struct msr_counter_info_t *cci, const struct msr_counter_arch_info *cai)
8796 {
8797 int ret = -1;
8798
8799 if (no_perf)
8800 return -1;
8801
8802 if (!cai->perf_name)
8803 return -1;
8804
8805 const unsigned int type = read_perf_type(cai->perf_subsys);
8806 const unsigned int config = read_perf_config(cai->perf_subsys, cai->perf_name);
8807
8808 ret = open_perf_counter(cpu, type, config, cci->fd_perf, PERF_FORMAT_GROUP);
8809
8810 if (ret == -1)
8811 goto end;
8812
8813 /* If it's the first counter opened, make it a group descriptor */
8814 if (cci->fd_perf == -1)
8815 cci->fd_perf = ret;
8816
8817 end:
8818 if (debug)
8819 fprintf(stderr, "%s: %s/%s: %d (cpu: %d)\n", __func__, cai->perf_subsys, cai->perf_name, ret, cpu);
8820
8821 return ret;
8822 }
8823
msr_perf_init_(void)8824 void msr_perf_init_(void)
8825 {
8826 const int mci_num = topo.max_cpu_num + 1;
8827
8828 msr_counter_info = calloc(mci_num, sizeof(*msr_counter_info));
8829 if (!msr_counter_info)
8830 err(1, "calloc msr_counter_info");
8831 msr_counter_info_size = mci_num;
8832
8833 for (int cpu = 0; cpu < mci_num; ++cpu)
8834 msr_counter_info[cpu].fd_perf = -1;
8835
8836 for (int cidx = 0; cidx < NUM_MSR_COUNTERS; ++cidx) {
8837
8838 struct msr_counter_arch_info *cai = &msr_counter_arch_infos[cidx];
8839
8840 cai->present = false;
8841
8842 for (int cpu = 0; cpu < mci_num; ++cpu) {
8843
8844 struct msr_counter_info_t *const cci = &msr_counter_info[cpu];
8845
8846 if (cpu_is_not_allowed(cpu))
8847 continue;
8848
8849 if (cai->needed) {
8850 /* Use perf API for this counter */
8851 if (add_msr_perf_counter(cpu, cci, cai) != -1) {
8852 cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8853 cai->present = true;
8854
8855 /* User MSR for this counter */
8856 } else if (add_msr_counter(cpu, cai->msr) >= 0) {
8857 cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8858 cci->msr[cai->rci_index] = cai->msr;
8859 cci->msr_mask[cai->rci_index] = cai->msr_mask;
8860 cai->present = true;
8861 }
8862 }
8863 }
8864 }
8865 }
8866
8867 /* Initialize data for reading perf counters from the MSR group. */
msr_perf_init(void)8868 void msr_perf_init(void)
8869 {
8870 bool need_amperf = false, need_smi = false;
8871 const bool need_soft_c1 = (!platform->has_msr_core_c1_res) && (platform->supported_cstates & CC1);
8872
8873 need_amperf = BIC_IS_ENABLED(BIC_Avg_MHz) || BIC_IS_ENABLED(BIC_Busy) || BIC_IS_ENABLED(BIC_Bzy_MHz)
8874 || BIC_IS_ENABLED(BIC_IPC) || need_soft_c1;
8875
8876 if (BIC_IS_ENABLED(BIC_SMI))
8877 need_smi = true;
8878
8879 /* Enable needed counters */
8880 msr_counter_arch_infos[MSR_ARCH_INFO_APERF_INDEX].needed = need_amperf;
8881 msr_counter_arch_infos[MSR_ARCH_INFO_MPERF_INDEX].needed = need_amperf;
8882 msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].needed = need_smi;
8883
8884 msr_perf_init_();
8885
8886 const bool has_amperf = has_amperf_access();
8887 const bool has_smi = msr_counter_arch_infos[MSR_ARCH_INFO_SMI_INDEX].present;
8888
8889 has_aperf_access = has_amperf;
8890
8891 if (has_amperf) {
8892 BIC_PRESENT(BIC_Avg_MHz);
8893 BIC_PRESENT(BIC_Busy);
8894 BIC_PRESENT(BIC_Bzy_MHz);
8895 BIC_PRESENT(BIC_SMI);
8896 }
8897
8898 if (has_smi)
8899 BIC_PRESENT(BIC_SMI);
8900 }
8901
cstate_perf_init_(bool soft_c1)8902 void cstate_perf_init_(bool soft_c1)
8903 {
8904 bool has_counter;
8905 bool *cores_visited = NULL, *pkg_visited = NULL;
8906 const int cores_visited_elems = topo.max_core_id + 1;
8907 const int pkg_visited_elems = topo.max_package_id + 1;
8908 const int cci_num = topo.max_cpu_num + 1;
8909
8910 ccstate_counter_info = calloc(cci_num, sizeof(*ccstate_counter_info));
8911 if (!ccstate_counter_info)
8912 err(1, "calloc ccstate_counter_arch_info");
8913 ccstate_counter_info_size = cci_num;
8914
8915 cores_visited = calloc(cores_visited_elems, sizeof(*cores_visited));
8916 if (!cores_visited)
8917 err(1, "calloc cores_visited");
8918
8919 pkg_visited = calloc(pkg_visited_elems, sizeof(*pkg_visited));
8920 if (!pkg_visited)
8921 err(1, "calloc pkg_visited");
8922
8923 /* Initialize cstate_counter_info_percpu */
8924 for (int cpu = 0; cpu < cci_num; ++cpu) {
8925 ccstate_counter_info[cpu].fd_perf_core = -1;
8926 ccstate_counter_info[cpu].fd_perf_pkg = -1;
8927 }
8928
8929 for (int cidx = 0; cidx < NUM_CSTATE_COUNTERS; ++cidx) {
8930 has_counter = false;
8931 memset(cores_visited, 0, cores_visited_elems * sizeof(*cores_visited));
8932 memset(pkg_visited, 0, pkg_visited_elems * sizeof(*pkg_visited));
8933
8934 const struct cstate_counter_arch_info *cai = &ccstate_counter_arch_infos[cidx];
8935
8936 for (int cpu = 0; cpu < cci_num; ++cpu) {
8937
8938 struct cstate_counter_info_t *const cci = &ccstate_counter_info[cpu];
8939
8940 if (cpu_is_not_allowed(cpu))
8941 continue;
8942
8943 const int core_id = cpus[cpu].core_id;
8944 const int pkg_id = cpus[cpu].package_id;
8945
8946 assert(core_id < cores_visited_elems);
8947 assert(pkg_id < pkg_visited_elems);
8948
8949 const bool per_thread = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_THREAD;
8950 const bool per_core = cai->flags & CSTATE_COUNTER_FLAG_COLLECT_PER_CORE;
8951
8952 if (!per_thread && cores_visited[core_id])
8953 continue;
8954
8955 if (!per_core && pkg_visited[pkg_id])
8956 continue;
8957
8958 const bool counter_needed = BIC_IS_ENABLED(cai->bic_number) || (soft_c1 && (cai->flags & CSTATE_COUNTER_FLAG_SOFT_C1_DEPENDENCY));
8959 const bool counter_supported = (platform->supported_cstates & cai->feature_mask);
8960
8961 if (counter_needed && counter_supported) {
8962 /* Use perf API for this counter */
8963 if (add_cstate_perf_counter(cpu, cci, cai) != -1) {
8964
8965 cci->source[cai->rci_index] = COUNTER_SOURCE_PERF;
8966
8967 /* User MSR for this counter */
8968 } else if (pkg_cstate_limit >= cai->pkg_cstate_limit && add_msr_counter(cpu, cai->msr) >= 0) {
8969 cci->source[cai->rci_index] = COUNTER_SOURCE_MSR;
8970 cci->msr[cai->rci_index] = cai->msr;
8971 }
8972 }
8973
8974 if (cci->source[cai->rci_index] != COUNTER_SOURCE_NONE) {
8975 has_counter = true;
8976 cores_visited[core_id] = true;
8977 pkg_visited[pkg_id] = true;
8978 }
8979 }
8980
8981 /* If any CPU has access to the counter, make it present */
8982 if (has_counter)
8983 BIC_PRESENT(cai->bic_number);
8984 }
8985
8986 free(cores_visited);
8987 free(pkg_visited);
8988 }
8989
cstate_perf_init(void)8990 void cstate_perf_init(void)
8991 {
8992 /*
8993 * If we don't have a C1 residency MSR, we calculate it "in software",
8994 * but we need APERF, MPERF too.
8995 */
8996 const bool soft_c1 = !platform->has_msr_core_c1_res && has_amperf_access()
8997 && platform->supported_cstates & CC1;
8998
8999 if (soft_c1)
9000 BIC_PRESENT(BIC_CPU_c1);
9001
9002 cstate_perf_init_(soft_c1);
9003 }
9004
probe_cstates(void)9005 void probe_cstates(void)
9006 {
9007 probe_cst_limit();
9008
9009 if (platform->has_msr_module_c6_res_ms)
9010 BIC_PRESENT(BIC_Mod_c6);
9011
9012 if (platform->has_ext_cst_msrs && !no_msr) {
9013 BIC_PRESENT(BIC_Totl_c0);
9014 BIC_PRESENT(BIC_Any_c0);
9015 BIC_PRESENT(BIC_GFX_c0);
9016 BIC_PRESENT(BIC_CPUGFX);
9017 }
9018
9019 if (quiet)
9020 return;
9021
9022 dump_power_ctl();
9023 dump_cst_cfg();
9024 decode_c6_demotion_policy_msr();
9025 print_dev_latency();
9026 dump_sysfs_cstate_config();
9027 print_irtl();
9028 }
9029
probe_lpi(void)9030 void probe_lpi(void)
9031 {
9032 if (!access("/sys/devices/system/cpu/cpuidle/low_power_idle_cpu_residency_us", R_OK))
9033 BIC_PRESENT(BIC_CPU_LPI);
9034 else
9035 BIC_NOT_PRESENT(BIC_CPU_LPI);
9036
9037 if (!access(sys_lpi_file_sysfs, R_OK)) {
9038 sys_lpi_file = sys_lpi_file_sysfs;
9039 BIC_PRESENT(BIC_SYS_LPI);
9040 } else if (!access(sys_lpi_file_debugfs, R_OK)) {
9041 sys_lpi_file = sys_lpi_file_debugfs;
9042 BIC_PRESENT(BIC_SYS_LPI);
9043 } else {
9044 sys_lpi_file_sysfs = NULL;
9045 BIC_NOT_PRESENT(BIC_SYS_LPI);
9046 }
9047
9048 }
9049
probe_pstates(void)9050 void probe_pstates(void)
9051 {
9052 probe_bclk();
9053
9054 if (quiet)
9055 return;
9056
9057 dump_platform_info();
9058 dump_turbo_ratio_info();
9059 dump_sysfs_pstate_config();
9060 decode_misc_pwr_mgmt_msr();
9061
9062 for_all_cpus(print_hwp, ODD_COUNTERS);
9063 for_all_cpus(print_epb, ODD_COUNTERS);
9064 for_all_cpus(print_perf_limit, ODD_COUNTERS);
9065 }
9066
dump_word_chars(unsigned int word)9067 void dump_word_chars(unsigned int word)
9068 {
9069 int i;
9070
9071 for (i = 0; i < 4; ++i)
9072 fprintf(outf, "%c", (word >> (i * 8)) & 0xFF);
9073 }
9074
dump_cpuid_hypervisor(void)9075 void dump_cpuid_hypervisor(void)
9076 {
9077 unsigned int ebx = 0;
9078 unsigned int ecx = 0;
9079 unsigned int edx = 0;
9080
9081 __cpuid(0x40000000, max_extended_level, ebx, ecx, edx);
9082
9083 fprintf(outf, "Hypervisor: ");
9084 dump_word_chars(ebx);
9085 dump_word_chars(ecx);
9086 dump_word_chars(edx);
9087 fprintf(outf, "\n");
9088 }
9089
process_cpuid()9090 void process_cpuid()
9091 {
9092 unsigned int eax, ebx, ecx, edx;
9093 unsigned int fms, family, model, stepping, ecx_flags, edx_flags;
9094 unsigned long long ucode_patch = 0;
9095 bool ucode_patch_valid = false;
9096
9097 eax = ebx = ecx = edx = 0;
9098
9099 __cpuid(0, max_level, ebx, ecx, edx);
9100
9101 if (ebx == 0x756e6547 && ecx == 0x6c65746e && edx == 0x49656e69)
9102 genuine_intel = 1;
9103 else if (ebx == 0x68747541 && ecx == 0x444d4163 && edx == 0x69746e65)
9104 authentic_amd = 1;
9105 else if (ebx == 0x6f677948 && ecx == 0x656e6975 && edx == 0x6e65476e)
9106 hygon_genuine = 1;
9107
9108 if (!quiet)
9109 fprintf(outf, "CPUID(0): %.4s%.4s%.4s 0x%x CPUID levels\n", (char *)&ebx, (char *)&edx, (char *)&ecx, max_level);
9110
9111 __cpuid(1, fms, ebx, ecx, edx);
9112 family = (fms >> 8) & 0xf;
9113 model = (fms >> 4) & 0xf;
9114 stepping = fms & 0xf;
9115 if (family == 0xf)
9116 family += (fms >> 20) & 0xff;
9117 if (family >= 6)
9118 model += ((fms >> 16) & 0xf) << 4;
9119 ecx_flags = ecx;
9120 edx_flags = edx;
9121 cpuid_has_hv = ecx_flags & (1 << 31);
9122
9123 if (!no_msr) {
9124 if (get_msr(sched_getcpu(), MSR_IA32_UCODE_REV, &ucode_patch)) {
9125 warnx("get_msr(UCODE)");
9126 } else {
9127 ucode_patch_valid = true;
9128 if (!authentic_amd && !hygon_genuine)
9129 ucode_patch >>= 32;
9130 }
9131 }
9132
9133 /*
9134 * check max extended function levels of CPUID.
9135 * This is needed to check for invariant TSC.
9136 * This check is valid for both Intel and AMD.
9137 */
9138 ebx = ecx = edx = 0;
9139 __cpuid(0x80000000, max_extended_level, ebx, ecx, edx);
9140
9141 if (!quiet) {
9142 fprintf(outf, "CPUID(1): family:model:stepping 0x%x:%x:%x (%d:%d:%d)", family, model, stepping, family, model, stepping);
9143 if (ucode_patch_valid)
9144 fprintf(outf, " microcode 0x%x", (unsigned int)ucode_patch);
9145 fputc('\n', outf);
9146
9147 fprintf(outf, "CPUID(0x80000000): max_extended_levels: 0x%x\n", max_extended_level);
9148 fprintf(outf, "CPUID(1): %sSSE3 %sMONITOR %sSMX %sEIST %sTM2 %sHV %sTSC %sMSR %sACPI-TM %sHT %sTM\n",
9149 ecx_flags & (1 << 0) ? "" : "No-",
9150 ecx_flags & (1 << 3) ? "" : "No-",
9151 ecx_flags & (1 << 6) ? "" : "No-",
9152 ecx_flags & (1 << 7) ? "" : "No-",
9153 ecx_flags & (1 << 8) ? "" : "No-",
9154 cpuid_has_hv ? "" : "No-",
9155 edx_flags & (1 << 4) ? "" : "No-",
9156 edx_flags & (1 << 5) ? "" : "No-",
9157 edx_flags & (1 << 22) ? "" : "No-", edx_flags & (1 << 28) ? "" : "No-", edx_flags & (1 << 29) ? "" : "No-");
9158 }
9159 if (!quiet && cpuid_has_hv)
9160 dump_cpuid_hypervisor();
9161
9162 probe_platform_features(family, model);
9163 init_perf_model_support(family, model);
9164
9165 if (!(edx_flags & (1 << 5)))
9166 errx(1, "CPUID: no MSR");
9167
9168 if (max_extended_level >= 0x80000007) {
9169
9170 /*
9171 * Non-Stop TSC is advertised by CPUID.EAX=0x80000007: EDX.bit8
9172 * this check is valid for both Intel and AMD
9173 */
9174 __cpuid(0x80000007, eax, ebx, ecx, edx);
9175 has_invariant_tsc = edx & (1 << 8);
9176 }
9177
9178 /*
9179 * APERF/MPERF is advertised by CPUID.EAX=0x6: ECX.bit0
9180 * this check is valid for both Intel and AMD
9181 */
9182
9183 __cpuid(0x6, eax, ebx, ecx, edx);
9184 cpuid_has_aperf_mperf = ecx & (1 << 0);
9185 do_dts = eax & (1 << 0);
9186 if (do_dts)
9187 BIC_PRESENT(BIC_CoreTmp);
9188 has_turbo = eax & (1 << 1);
9189 do_ptm = eax & (1 << 6);
9190 if (do_ptm)
9191 BIC_PRESENT(BIC_PkgTmp);
9192 has_hwp = eax & (1 << 7);
9193 has_hwp_notify = eax & (1 << 8);
9194 has_hwp_activity_window = eax & (1 << 9);
9195 has_hwp_epp = eax & (1 << 10);
9196 has_hwp_pkg = eax & (1 << 11);
9197 has_epb = ecx & (1 << 3);
9198
9199 if (!quiet)
9200 fprintf(outf, "CPUID(6): %sAPERF, %sTURBO, %sDTS, %sPTM, %sHWP, "
9201 "%sHWPnotify, %sHWPwindow, %sHWPepp, %sHWPpkg, %sEPB\n",
9202 cpuid_has_aperf_mperf ? "" : "No-",
9203 has_turbo ? "" : "No-",
9204 do_dts ? "" : "No-",
9205 do_ptm ? "" : "No-",
9206 has_hwp ? "" : "No-",
9207 has_hwp_notify ? "" : "No-",
9208 has_hwp_activity_window ? "" : "No-", has_hwp_epp ? "" : "No-", has_hwp_pkg ? "" : "No-", has_epb ? "" : "No-");
9209
9210 if (!quiet)
9211 decode_misc_enable_msr();
9212
9213 if (max_level >= 0x7) {
9214 int has_sgx;
9215
9216 ecx = 0;
9217
9218 __cpuid_count(0x7, 0, eax, ebx, ecx, edx);
9219
9220 has_sgx = ebx & (1 << 2);
9221
9222 is_hybrid = !!(edx & (1 << 15));
9223
9224 if (!quiet)
9225 fprintf(outf, "CPUID(7): %sSGX %sHybrid\n", has_sgx ? "" : "No-", is_hybrid ? "" : "No-");
9226
9227 if (has_sgx)
9228 decode_feature_control_msr();
9229 }
9230
9231 if (max_level >= 0x15) {
9232 unsigned int eax_crystal;
9233 unsigned int ebx_tsc;
9234
9235 /*
9236 * CPUID 15H TSC/Crystal ratio, possibly Crystal Hz
9237 */
9238 eax_crystal = ebx_tsc = crystal_hz = edx = 0;
9239 __cpuid(0x15, eax_crystal, ebx_tsc, crystal_hz, edx);
9240
9241 if (ebx_tsc != 0) {
9242 if (!quiet && (ebx != 0))
9243 fprintf(outf, "CPUID(0x15): eax_crystal: %d ebx_tsc: %d ecx_crystal_hz: %d\n", eax_crystal, ebx_tsc, crystal_hz);
9244
9245 if (crystal_hz == 0)
9246 crystal_hz = platform->crystal_freq;
9247
9248 if (crystal_hz) {
9249 tsc_hz = (unsigned long long)crystal_hz *ebx_tsc / eax_crystal;
9250 if (!quiet)
9251 fprintf(outf, "TSC: %lld MHz (%d Hz * %d / %d / 1000000)\n", tsc_hz / 1000000, crystal_hz, ebx_tsc, eax_crystal);
9252 }
9253 }
9254 }
9255 if (max_level >= 0x16) {
9256 unsigned int base_mhz, max_mhz, bus_mhz, edx;
9257
9258 /*
9259 * CPUID 16H Base MHz, Max MHz, Bus MHz
9260 */
9261 base_mhz = max_mhz = bus_mhz = edx = 0;
9262
9263 __cpuid(0x16, base_mhz, max_mhz, bus_mhz, edx);
9264
9265 bclk = bus_mhz;
9266
9267 base_hz = base_mhz * 1000000;
9268 has_base_hz = 1;
9269
9270 if (platform->enable_tsc_tweak)
9271 tsc_tweak = base_hz / tsc_hz;
9272
9273 if (!quiet)
9274 fprintf(outf, "CPUID(0x16): base_mhz: %d max_mhz: %d bus_mhz: %d\n", base_mhz, max_mhz, bus_mhz);
9275 }
9276
9277 if (cpuid_has_aperf_mperf)
9278 aperf_mperf_multiplier = platform->need_perf_multiplier ? 1024 : 1;
9279
9280 BIC_PRESENT(BIC_IRQ);
9281 BIC_PRESENT(BIC_NMI);
9282 BIC_PRESENT(BIC_TSC_MHz);
9283 }
9284
counter_info_init(void)9285 static void counter_info_init(void)
9286 {
9287 for (int i = 0; i < NUM_CSTATE_COUNTERS; ++i) {
9288 struct cstate_counter_arch_info *const cai = &ccstate_counter_arch_infos[i];
9289
9290 if (platform->has_msr_knl_core_c6_residency && cai->msr == MSR_CORE_C6_RESIDENCY)
9291 cai->msr = MSR_KNL_CORE_C6_RESIDENCY;
9292
9293 if (!platform->has_msr_core_c1_res && cai->msr == MSR_CORE_C1_RES)
9294 cai->msr = 0;
9295
9296 if (platform->has_msr_atom_pkg_c6_residency && cai->msr == MSR_PKG_C6_RESIDENCY)
9297 cai->msr = MSR_ATOM_PKG_C6_RESIDENCY;
9298 }
9299
9300 for (int i = 0; i < NUM_MSR_COUNTERS; ++i) {
9301 msr_counter_arch_infos[i].present = false;
9302 msr_counter_arch_infos[i].needed = false;
9303 }
9304 }
9305
probe_pm_features(void)9306 void probe_pm_features(void)
9307 {
9308 probe_pstates();
9309
9310 probe_cstates();
9311
9312 probe_lpi();
9313
9314 probe_intel_uncore_frequency();
9315
9316 probe_graphics();
9317
9318 probe_rapl();
9319
9320 probe_thermal();
9321
9322 if (platform->has_nhm_msrs && !no_msr)
9323 BIC_PRESENT(BIC_SMI);
9324
9325 if (!quiet)
9326 decode_misc_feature_control();
9327 }
9328
9329 /*
9330 * has_perf_llc_access()
9331 *
9332 * return 1 on success, else 0
9333 */
has_perf_llc_access(void)9334 int has_perf_llc_access(void)
9335 {
9336 int fd;
9337
9338 if (no_perf)
9339 return 0;
9340
9341 fd = open_perf_counter(master_cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP);
9342 if (fd != -1)
9343 close(fd);
9344
9345 if (fd == -1)
9346 warnx("Failed to access %s. Some of the counters may not be available\n"
9347 "\tRun as root to enable them or use %s to disable the access explicitly", "perf LLC counters", "'--hide LLC' or '--no-perf'");
9348
9349 return (fd != -1);
9350 }
9351
perf_llc_init(void)9352 void perf_llc_init(void)
9353 {
9354 int cpu;
9355 int retval;
9356
9357 if (no_perf)
9358 return;
9359 if (!(BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT)))
9360 return;
9361
9362 assert(fd_llc_percpu != 0);
9363
9364 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
9365
9366 if (cpu_is_not_allowed(cpu))
9367 continue;
9368
9369 fd_llc_percpu[cpu] = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_REFERENCES, -1, PERF_FORMAT_GROUP);
9370 if (fd_llc_percpu[cpu] == -1) {
9371 warnx("%s: perf REFS: failed to open counter on cpu%d", __func__, cpu);
9372 free_fd_llc_percpu();
9373 return;
9374 }
9375 retval = open_perf_counter(cpu, PERF_TYPE_HARDWARE, PERF_COUNT_HW_CACHE_MISSES, fd_llc_percpu[cpu], PERF_FORMAT_GROUP);
9376 if (retval == -1) {
9377 warnx("%s: perf MISS: failed to open counter on cpu%d", __func__, cpu);
9378 free_fd_llc_percpu();
9379 return;
9380 }
9381 }
9382 BIC_PRESENT(BIC_LLC_MRPS);
9383 BIC_PRESENT(BIC_LLC_HIT);
9384 }
9385
perf_l2_init(void)9386 void perf_l2_init(void)
9387 {
9388 int cpu;
9389 int retval;
9390
9391 if (no_perf)
9392 return;
9393 if (!(BIC_IS_ENABLED(BIC_L2_MRPS) || BIC_IS_ENABLED(BIC_L2_HIT)))
9394 return;
9395 if (perf_model_support == NULL)
9396 return;
9397
9398 assert(fd_l2_percpu != 0);
9399
9400 for (cpu = 0; cpu <= topo.max_cpu_num; ++cpu) {
9401
9402 if (cpu_is_not_allowed(cpu))
9403 continue;
9404
9405 if (!is_hybrid) {
9406 fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.uniform, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP);
9407 if (fd_l2_percpu[cpu] == -1) {
9408 warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.uniform, perf_model_support->first.refs);
9409 free_fd_l2_percpu();
9410 return;
9411 }
9412 retval = open_perf_counter(cpu, perf_pmu_types.uniform, perf_model_support->first.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP);
9413 if (retval == -1) {
9414 warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.uniform, perf_model_support->first.hits);
9415 free_fd_l2_percpu();
9416 return;
9417 }
9418 continue;
9419 }
9420 if (perf_pcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_pcore_set)) {
9421 fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.refs, -1, PERF_FORMAT_GROUP);
9422 if (fd_l2_percpu[cpu] == -1) {
9423 warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.refs);
9424 free_fd_l2_percpu();
9425 return;
9426 }
9427 retval = open_perf_counter(cpu, perf_pmu_types.pcore, perf_model_support->first.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP);
9428 if (retval == -1) {
9429 warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.pcore, perf_model_support->first.hits);
9430 free_fd_l2_percpu();
9431 return;
9432 }
9433 } else if (perf_ecore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_ecore_set)) {
9434 fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.refs, -1, PERF_FORMAT_GROUP);
9435 if (fd_l2_percpu[cpu] == -1) {
9436 warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.ecore, perf_model_support->second.refs);
9437 free_fd_l2_percpu();
9438 return;
9439 }
9440 retval = open_perf_counter(cpu, perf_pmu_types.ecore, perf_model_support->second.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP);
9441 if (retval == -1) {
9442 warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.ecore, perf_model_support->second.hits);
9443 free_fd_l2_percpu();
9444 return;
9445 }
9446 } else if (perf_lcore_set && CPU_ISSET_S(cpu, cpu_possible_setsize, perf_lcore_set)) {
9447 fd_l2_percpu[cpu] = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.refs, -1, PERF_FORMAT_GROUP);
9448 if (fd_l2_percpu[cpu] == -1) {
9449 warnx("%s(cpu%d, 0x%x, 0x%llx) REFS", __func__, cpu, perf_pmu_types.lcore, perf_model_support->third.refs);
9450 free_fd_l2_percpu();
9451 return;
9452 }
9453 retval = open_perf_counter(cpu, perf_pmu_types.lcore, perf_model_support->third.hits, fd_l2_percpu[cpu], PERF_FORMAT_GROUP);
9454 if (retval == -1) {
9455 warnx("%s(cpu%d, 0x%x, 0x%llx) HITS", __func__, cpu, perf_pmu_types.lcore, perf_model_support->third.hits);
9456 free_fd_l2_percpu();
9457 return;
9458 }
9459 } else
9460 err(-1, "%s: cpu%d: type %d", __func__, cpu, cpus[cpu].type);
9461 }
9462 BIC_PRESENT(BIC_L2_MRPS);
9463 BIC_PRESENT(BIC_L2_HIT);
9464 }
9465
9466 /*
9467 * in /dev/cpu/ return success for names that are numbers
9468 * ie. filter out ".", "..", "microcode".
9469 */
dir_filter(const struct dirent * dirp)9470 int dir_filter(const struct dirent *dirp)
9471 {
9472 if (isdigit(dirp->d_name[0]))
9473 return 1;
9474 else
9475 return 0;
9476 }
9477
topology_probe(bool startup)9478 void topology_probe(bool startup)
9479 {
9480 int i;
9481 int max_core_id = 0;
9482 int max_package_id = 0;
9483 int max_siblings = 0;
9484
9485 /* Initialize num_cpus, max_cpu_num */
9486 set_max_cpu_num();
9487 topo.num_cpus = 0;
9488 for_all_proc_cpus(count_cpus);
9489 if (!summary_only)
9490 BIC_PRESENT(BIC_CPU);
9491
9492 if (debug > 1)
9493 fprintf(outf, "num_cpus %d max_cpu_num %d\n", topo.num_cpus, topo.max_cpu_num);
9494
9495 cpus = calloc(1, (topo.max_cpu_num + 1) * sizeof(struct cpu_topology));
9496 if (cpus == NULL)
9497 err(1, "calloc cpus");
9498
9499 /*
9500 * Allocate and initialize cpu_present_set
9501 */
9502 cpu_present_set = CPU_ALLOC((topo.max_cpu_num + 1));
9503 if (cpu_present_set == NULL)
9504 err(3, "CPU_ALLOC");
9505 cpu_present_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9506 CPU_ZERO_S(cpu_present_setsize, cpu_present_set);
9507 for_all_proc_cpus(mark_cpu_present);
9508
9509 /*
9510 * Allocate and initialize cpu_possible_set
9511 */
9512 cpu_possible_set = CPU_ALLOC((topo.max_cpu_num + 1));
9513 if (cpu_possible_set == NULL)
9514 err(3, "CPU_ALLOC");
9515 cpu_possible_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9516 CPU_ZERO_S(cpu_possible_setsize, cpu_possible_set);
9517 initialize_cpu_set_from_sysfs(cpu_possible_set, "/sys/devices/system/cpu", "possible");
9518
9519 /*
9520 * Allocate and initialize cpu_effective_set
9521 */
9522 cpu_effective_set = CPU_ALLOC((topo.max_cpu_num + 1));
9523 if (cpu_effective_set == NULL)
9524 err(3, "CPU_ALLOC");
9525 cpu_effective_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9526 CPU_ZERO_S(cpu_effective_setsize, cpu_effective_set);
9527 update_effective_set(startup);
9528
9529 /*
9530 * Allocate and initialize cpu_allowed_set
9531 */
9532 cpu_allowed_set = CPU_ALLOC((topo.max_cpu_num + 1));
9533 if (cpu_allowed_set == NULL)
9534 err(3, "CPU_ALLOC");
9535 cpu_allowed_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9536 CPU_ZERO_S(cpu_allowed_setsize, cpu_allowed_set);
9537
9538 /*
9539 * Validate and update cpu_allowed_set.
9540 *
9541 * Make sure all cpus in cpu_subset are also in cpu_present_set during startup.
9542 * Give a warning when cpus in cpu_subset become unavailable at runtime.
9543 * Give a warning when cpus are not effective because of cgroup setting.
9544 *
9545 * cpu_allowed_set is the intersection of cpu_present_set/cpu_effective_set/cpu_subset.
9546 */
9547 for (i = 0; i < CPU_SUBSET_MAXCPUS; ++i) {
9548 if (cpu_subset && !CPU_ISSET_S(i, cpu_subset_size, cpu_subset))
9549 continue;
9550
9551 if (!CPU_ISSET_S(i, cpu_present_setsize, cpu_present_set)) {
9552 if (cpu_subset) {
9553 /* cpus in cpu_subset must be in cpu_present_set during startup */
9554 if (startup)
9555 err(1, "cpu%d not present", i);
9556 else
9557 fprintf(stderr, "cpu%d not present\n", i);
9558 }
9559 continue;
9560 }
9561
9562 if (CPU_COUNT_S(cpu_effective_setsize, cpu_effective_set)) {
9563 if (!CPU_ISSET_S(i, cpu_effective_setsize, cpu_effective_set)) {
9564 fprintf(stderr, "cpu%d not effective\n", i);
9565 continue;
9566 }
9567 }
9568
9569 CPU_SET_S(i, cpu_allowed_setsize, cpu_allowed_set);
9570 }
9571
9572 if (!CPU_COUNT_S(cpu_allowed_setsize, cpu_allowed_set))
9573 err(-ENODEV, "No valid cpus found");
9574 sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set);
9575
9576 /*
9577 * Allocate and initialize cpu_affinity_set
9578 */
9579 cpu_affinity_set = CPU_ALLOC((topo.max_cpu_num + 1));
9580 if (cpu_affinity_set == NULL)
9581 err(3, "CPU_ALLOC");
9582 cpu_affinity_setsize = CPU_ALLOC_SIZE((topo.max_cpu_num + 1));
9583 CPU_ZERO_S(cpu_affinity_setsize, cpu_affinity_set);
9584
9585 for_all_proc_cpus(clear_ht_id);
9586
9587 for_all_proc_cpus(set_cpu_hybrid_type);
9588
9589 /*
9590 * For online cpus
9591 * find max_core_id, max_package_id, num_cores (per system)
9592 */
9593 for (i = 0; i <= topo.max_cpu_num; ++i) {
9594 int siblings;
9595
9596 if (cpu_is_not_present(i)) {
9597 if (debug > 1)
9598 fprintf(outf, "cpu%d NOT PRESENT\n", i);
9599 continue;
9600 }
9601
9602 cpus[i].cpu_id = i;
9603
9604 /* get package information */
9605 cpus[i].package_id = get_package_id(i);
9606 if (cpus[i].package_id > max_package_id)
9607 max_package_id = cpus[i].package_id;
9608
9609 /* get die information */
9610 cpus[i].die_id = get_die_id(i);
9611 if (cpus[i].die_id > topo.max_die_id)
9612 topo.max_die_id = cpus[i].die_id;
9613
9614 /* get l3 information */
9615 cpus[i].l3_id = get_l3_id(i);
9616 if (cpus[i].l3_id > topo.max_l3_id)
9617 topo.max_l3_id = cpus[i].l3_id;
9618
9619 /* get numa node information */
9620 cpus[i].physical_node_id = get_physical_node_id(&cpus[i]);
9621 if (cpus[i].physical_node_id > topo.max_node_num)
9622 topo.max_node_num = cpus[i].physical_node_id;
9623
9624 /* get core information */
9625 cpus[i].core_id = get_core_id(i);
9626 if (cpus[i].core_id > max_core_id)
9627 max_core_id = cpus[i].core_id;
9628
9629 /* get thread information */
9630 siblings = set_thread_siblings(&cpus[i]);
9631 if (siblings > max_siblings)
9632 max_siblings = siblings;
9633 if (cpus[i].ht_id == 0)
9634 topo.num_cores++;
9635 }
9636 topo.max_core_id = max_core_id; /* within a package */
9637 topo.max_package_id = max_package_id;
9638
9639 topo.cores_per_pkg = max_core_id + 1;
9640 if (debug > 1)
9641 fprintf(outf, "max_core_id %d, sizing for %d cores per package\n", max_core_id, topo.cores_per_pkg);
9642 if (!summary_only)
9643 BIC_PRESENT(BIC_Core);
9644
9645 topo.num_die = topo.max_die_id + 1;
9646 if (debug > 1)
9647 fprintf(outf, "max_die_id %d, sizing for %d die\n", topo.max_die_id, topo.num_die);
9648 if (!summary_only && topo.num_die > 1)
9649 BIC_PRESENT(BIC_Die);
9650
9651 if (!summary_only && topo.max_l3_id > 0)
9652 BIC_PRESENT(BIC_L3);
9653
9654 topo.num_packages = max_package_id + 1;
9655 if (debug > 1)
9656 fprintf(outf, "max_package_id %d, sizing for %d packages\n", max_package_id, topo.num_packages);
9657 if (!summary_only && topo.num_packages > 1)
9658 BIC_PRESENT(BIC_Package);
9659
9660 set_node_data();
9661 if (debug > 1)
9662 fprintf(outf, "nodes_per_pkg %d\n", topo.nodes_per_pkg);
9663 if (!summary_only && topo.nodes_per_pkg > 1)
9664 BIC_PRESENT(BIC_Node);
9665
9666 topo.threads_per_core = max_siblings;
9667 if (debug > 1)
9668 fprintf(outf, "max_siblings %d\n", max_siblings);
9669
9670 if (debug < 1)
9671 return;
9672
9673 for (i = 0; i <= topo.max_cpu_num; ++i) {
9674 if (cpu_is_not_present(i))
9675 continue;
9676 fprintf(outf,
9677 "cpu %d pkg %d die %d l3 %d node %d lnode %d core %d thread %d\n",
9678 i, cpus[i].package_id, cpus[i].die_id, cpus[i].l3_id,
9679 cpus[i].physical_node_id, cpus[i].logical_node_id, cpus[i].core_id, cpus[i].ht_id);
9680 }
9681
9682 }
9683
allocate_counters_1(struct counters * counters)9684 void allocate_counters_1(struct counters *counters)
9685 {
9686 counters->threads = calloc(1, sizeof(struct thread_data));
9687 if (counters->threads == NULL)
9688 goto error;
9689
9690 counters->cores = calloc(1, sizeof(struct core_data));
9691 if (counters->cores == NULL)
9692 goto error;
9693
9694 counters->packages = calloc(1, sizeof(struct pkg_data));
9695 if (counters->packages == NULL)
9696 goto error;
9697
9698 return;
9699 error:
9700 err(1, "calloc counters_1");
9701 }
9702
allocate_counters(struct counters * counters)9703 void allocate_counters(struct counters *counters)
9704 {
9705 int i;
9706 int num_cores = topo.cores_per_pkg * topo.num_packages;
9707
9708 counters->threads = calloc(topo.max_cpu_num + 1, sizeof(struct thread_data));
9709 if (counters->threads == NULL)
9710 goto error;
9711
9712 for (i = 0; i < topo.max_cpu_num + 1; i++)
9713 (counters->threads)[i].cpu_id = -1;
9714
9715 counters->cores = calloc(num_cores, sizeof(struct core_data));
9716 if (counters->cores == NULL)
9717 goto error;
9718
9719 for (i = 0; i < num_cores; i++)
9720 (counters->cores)[i].first_cpu = -1;
9721
9722 counters->packages = calloc(topo.num_packages, sizeof(struct pkg_data));
9723 if (counters->packages == NULL)
9724 goto error;
9725
9726 for (i = 0; i < topo.num_packages; i++)
9727 (counters->packages)[i].first_cpu = -1;
9728
9729 return;
9730 error:
9731 err(1, "calloc counters");
9732 }
9733
9734 /*
9735 * init_counter()
9736 *
9737 * set t->cpu_id, FIRST_THREAD_IN_CORE and FIRST_CORE_IN_PACKAGE
9738 */
init_counter(struct thread_data * thread_base,struct core_data * core_base,struct pkg_data * pkg_base,int cpu_id)9739 void init_counter(struct thread_data *thread_base, struct core_data *core_base, struct pkg_data *pkg_base, int cpu_id)
9740 {
9741 int pkg_id = cpus[cpu_id].package_id;
9742 int node_id = cpus[cpu_id].logical_node_id;
9743 int core_id = cpus[cpu_id].core_id;
9744 struct thread_data *t;
9745 struct core_data *c;
9746
9747 /* Workaround for systems where physical_node_id==-1
9748 * and logical_node_id==(-1 - topo.num_cpus)
9749 */
9750 if (node_id < 0)
9751 node_id = 0;
9752
9753 t = &thread_base[cpu_id];
9754 c = &core_base[GLOBAL_CORE_ID(core_id, pkg_id)];
9755
9756 t->cpu_id = cpu_id;
9757 if (!cpu_is_not_allowed(cpu_id)) {
9758
9759 if (c->first_cpu < 0)
9760 c->first_cpu = t->cpu_id;
9761 if (pkg_base[pkg_id].first_cpu < 0)
9762 pkg_base[pkg_id].first_cpu = t->cpu_id;
9763 }
9764 }
9765
initialize_counters(int cpu_id)9766 int initialize_counters(int cpu_id)
9767 {
9768 init_counter(EVEN_COUNTERS, cpu_id);
9769 init_counter(ODD_COUNTERS, cpu_id);
9770 return 0;
9771 }
9772
allocate_output_buffer()9773 void allocate_output_buffer()
9774 {
9775 output_buffer = calloc(1, (1 + topo.num_cpus) * 2048);
9776 outp = output_buffer;
9777 if (outp == NULL)
9778 err(-1, "calloc output buffer");
9779 }
9780
allocate_fd_percpu(void)9781 void allocate_fd_percpu(void)
9782 {
9783 fd_percpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9784 if (fd_percpu == NULL)
9785 err(-1, "calloc fd_percpu");
9786 }
9787
allocate_irq_buffers(void)9788 void allocate_irq_buffers(void)
9789 {
9790 irq_column_2_cpu = calloc(topo.num_cpus, sizeof(int));
9791 if (irq_column_2_cpu == NULL)
9792 err(-1, "calloc %d", topo.num_cpus);
9793
9794 irqs_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9795 if (irqs_per_cpu == NULL)
9796 err(-1, "calloc %d IRQ", topo.max_cpu_num + 1);
9797
9798 nmi_per_cpu = calloc(topo.max_cpu_num + 1, sizeof(int));
9799 if (nmi_per_cpu == NULL)
9800 err(-1, "calloc %d NMI", topo.max_cpu_num + 1);
9801 }
9802
update_topo(PER_THREAD_PARAMS)9803 int update_topo(PER_THREAD_PARAMS)
9804 {
9805 topo.allowed_cpus++;
9806 if ((int)t->cpu_id == c->first_cpu)
9807 topo.allowed_cores++;
9808 if ((int)t->cpu_id == p->first_cpu)
9809 topo.allowed_packages++;
9810
9811 return 0;
9812 }
9813
topology_update(void)9814 void topology_update(void)
9815 {
9816 topo.allowed_cpus = 0;
9817 topo.allowed_cores = 0;
9818 topo.allowed_packages = 0;
9819 for_all_cpus(update_topo, ODD_COUNTERS);
9820 }
9821
setup_all_buffers(bool startup)9822 void setup_all_buffers(bool startup)
9823 {
9824 topology_probe(startup);
9825 allocate_irq_buffers();
9826 allocate_fd_percpu();
9827 allocate_counters_1(&average);
9828 allocate_counters(&even);
9829 allocate_counters(&odd);
9830 allocate_output_buffer();
9831 for_all_proc_cpus(initialize_counters);
9832 topology_update();
9833 }
9834
set_master_cpu(void)9835 void set_master_cpu(void)
9836 {
9837 int i;
9838
9839 for (i = 0; i < topo.max_cpu_num + 1; ++i) {
9840 if (cpu_is_not_allowed(i))
9841 continue;
9842 master_cpu = i;
9843 if (debug > 1)
9844 fprintf(outf, "master_cpu = %d\n", master_cpu);
9845 return;
9846 }
9847 err(-ENODEV, "No valid cpus found");
9848 }
9849
has_added_counters(void)9850 bool has_added_counters(void)
9851 {
9852 /*
9853 * It only makes sense to call this after the command line is parsed,
9854 * otherwise sys structure is not populated.
9855 */
9856
9857 return sys.added_core_counters | sys.added_thread_counters | sys.added_package_counters;
9858 }
9859
check_msr_access(void)9860 void check_msr_access(void)
9861 {
9862 check_msr_driver();
9863 check_msr_permission();
9864
9865 if (no_msr)
9866 bic_disable_msr_access();
9867 }
9868
check_perf_access(void)9869 void check_perf_access(void)
9870 {
9871 if (BIC_IS_ENABLED(BIC_IPC))
9872 if (!has_perf_instr_count_access())
9873 no_perf = 1;
9874
9875 if (BIC_IS_ENABLED(BIC_LLC_MRPS) || BIC_IS_ENABLED(BIC_LLC_HIT))
9876 if (!has_perf_llc_access())
9877 no_perf = 1;
9878
9879 if (no_perf)
9880 bic_disable_perf_access();
9881 }
9882
perf_has_hybrid_devices(void)9883 bool perf_has_hybrid_devices(void)
9884 {
9885 /*
9886 * 0: unknown
9887 * 1: has separate perf device for p and e core
9888 * -1: doesn't have separate perf device for p and e core
9889 */
9890 static int cached;
9891
9892 if (cached > 0)
9893 return true;
9894
9895 if (cached < 0)
9896 return false;
9897
9898 if (access("/sys/bus/event_source/devices/cpu_core", F_OK)) {
9899 cached = -1;
9900 return false;
9901 }
9902
9903 if (access("/sys/bus/event_source/devices/cpu_atom", F_OK)) {
9904 cached = -1;
9905 return false;
9906 }
9907
9908 cached = 1;
9909 return true;
9910 }
9911
added_perf_counters_init_(struct perf_counter_info * pinfo)9912 int added_perf_counters_init_(struct perf_counter_info *pinfo)
9913 {
9914 size_t num_domains = 0;
9915 unsigned int next_domain;
9916 bool *domain_visited;
9917 unsigned int perf_type, perf_config;
9918 double perf_scale;
9919 int fd_perf;
9920
9921 if (!pinfo)
9922 return 0;
9923
9924 const size_t max_num_domains = MAX(topo.max_cpu_num + 1, MAX(topo.max_core_id + 1, topo.max_package_id + 1));
9925
9926 domain_visited = calloc(max_num_domains, sizeof(*domain_visited));
9927
9928 while (pinfo) {
9929 switch (pinfo->scope) {
9930 case SCOPE_CPU:
9931 num_domains = topo.max_cpu_num + 1;
9932 break;
9933
9934 case SCOPE_CORE:
9935 num_domains = topo.max_core_id + 1;
9936 break;
9937
9938 case SCOPE_PACKAGE:
9939 num_domains = topo.max_package_id + 1;
9940 break;
9941 }
9942
9943 /* Allocate buffer for file descriptor for each domain. */
9944 pinfo->fd_perf_per_domain = calloc(num_domains, sizeof(*pinfo->fd_perf_per_domain));
9945 if (!pinfo->fd_perf_per_domain)
9946 errx(1, "%s: alloc %s", __func__, "fd_perf_per_domain");
9947
9948 for (size_t i = 0; i < num_domains; ++i)
9949 pinfo->fd_perf_per_domain[i] = -1;
9950
9951 pinfo->num_domains = num_domains;
9952 pinfo->scale = 1.0;
9953
9954 memset(domain_visited, 0, max_num_domains * sizeof(*domain_visited));
9955
9956 for (int cpu = 0; cpu < topo.max_cpu_num + 1; ++cpu) {
9957
9958 next_domain = cpu_to_domain(pinfo, cpu);
9959
9960 assert(next_domain < num_domains);
9961
9962 if (cpu_is_not_allowed(cpu))
9963 continue;
9964
9965 if (domain_visited[next_domain])
9966 continue;
9967
9968 /*
9969 * Intel hybrid platforms expose different perf devices for P and E cores.
9970 * Instead of one, "/sys/bus/event_source/devices/cpu" device, there are
9971 * "/sys/bus/event_source/devices/{cpu_core,cpu_atom}".
9972 *
9973 * This makes it more complicated to the user, because most of the counters
9974 * are available on both and have to be handled manually, otherwise.
9975 *
9976 * Code below, allow user to use the old "cpu" name, which is translated accordingly.
9977 */
9978 const char *perf_device = pinfo->device;
9979
9980 if (strcmp(perf_device, "cpu") == 0 && perf_has_hybrid_devices()) {
9981 switch (cpus[cpu].type) {
9982 case INTEL_PCORE_TYPE:
9983 perf_device = "cpu_core";
9984 break;
9985
9986 case INTEL_ECORE_TYPE:
9987 perf_device = "cpu_atom";
9988 break;
9989
9990 default: /* Don't change, we will probably fail and report a problem soon. */
9991 break;
9992 }
9993 }
9994
9995 perf_type = read_perf_type(perf_device);
9996 if (perf_type == (unsigned int)-1) {
9997 warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "type");
9998 continue;
9999 }
10000
10001 perf_config = read_perf_config(perf_device, pinfo->event);
10002 if (perf_config == (unsigned int)-1) {
10003 warnx("%s: perf/%s/%s: failed to read %s", __func__, perf_device, pinfo->event, "config");
10004 continue;
10005 }
10006
10007 /* Scale is not required, some counters just don't have it. */
10008 perf_scale = read_perf_scale(perf_device, pinfo->event);
10009 if (perf_scale == 0.0)
10010 perf_scale = 1.0;
10011
10012 fd_perf = open_perf_counter(cpu, perf_type, perf_config, -1, 0);
10013 if (fd_perf == -1) {
10014 warnx("%s: perf/%s/%s: failed to open counter on cpu%d", __func__, perf_device, pinfo->event, cpu);
10015 continue;
10016 }
10017
10018 domain_visited[next_domain] = 1;
10019 pinfo->fd_perf_per_domain[next_domain] = fd_perf;
10020 pinfo->scale = perf_scale;
10021
10022 if (debug)
10023 fprintf(stderr, "Add perf/%s/%s cpu%d: %d\n", perf_device, pinfo->event, cpu, pinfo->fd_perf_per_domain[next_domain]);
10024 }
10025
10026 pinfo = pinfo->next;
10027 }
10028
10029 free(domain_visited);
10030
10031 return 0;
10032 }
10033
added_perf_counters_init(void)10034 void added_perf_counters_init(void)
10035 {
10036 if (added_perf_counters_init_(sys.perf_tp))
10037 errx(1, "%s: %s", __func__, "thread");
10038
10039 if (added_perf_counters_init_(sys.perf_cp))
10040 errx(1, "%s: %s", __func__, "core");
10041
10042 if (added_perf_counters_init_(sys.perf_pp))
10043 errx(1, "%s: %s", __func__, "package");
10044 }
10045
parse_telem_info_file(int fd_dir,const char * info_filename,const char * format,unsigned long * output)10046 int parse_telem_info_file(int fd_dir, const char *info_filename, const char *format, unsigned long *output)
10047 {
10048 int fd_telem_info;
10049 FILE *file_telem_info;
10050 unsigned long value;
10051
10052 fd_telem_info = openat(fd_dir, info_filename, O_RDONLY);
10053 if (fd_telem_info == -1)
10054 return -1;
10055
10056 file_telem_info = fdopen(fd_telem_info, "r");
10057 if (file_telem_info == NULL) {
10058 close(fd_telem_info);
10059 return -1;
10060 }
10061
10062 if (fscanf(file_telem_info, format, &value) != 1) {
10063 fclose(file_telem_info);
10064 return -1;
10065 }
10066
10067 fclose(file_telem_info);
10068
10069 *output = value;
10070
10071 return 0;
10072 }
10073
pmt_mmio_open(unsigned int target_guid)10074 struct pmt_mmio *pmt_mmio_open(unsigned int target_guid)
10075 {
10076 struct pmt_diriter_t pmt_iter;
10077 const struct dirent *entry;
10078 struct stat st;
10079 int fd_telem_dir, fd_pmt;
10080 unsigned long guid, size, offset;
10081 size_t mmap_size;
10082 void *mmio;
10083 struct pmt_mmio *head = NULL, *last = NULL;
10084 struct pmt_mmio *new_pmt = NULL;
10085
10086 if (stat(SYSFS_TELEM_PATH, &st) == -1)
10087 return NULL;
10088
10089 pmt_diriter_init(&pmt_iter);
10090 entry = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH);
10091 if (!entry) {
10092 pmt_diriter_remove(&pmt_iter);
10093 return NULL;
10094 }
10095
10096 for (; entry != NULL; entry = pmt_diriter_next(&pmt_iter)) {
10097 if (fstatat(dirfd(pmt_iter.dir), entry->d_name, &st, 0) == -1)
10098 break;
10099
10100 if (!S_ISDIR(st.st_mode))
10101 continue;
10102
10103 fd_telem_dir = openat(dirfd(pmt_iter.dir), entry->d_name, O_RDONLY);
10104 if (fd_telem_dir == -1)
10105 break;
10106
10107 if (parse_telem_info_file(fd_telem_dir, "guid", "%lx", &guid)) {
10108 close(fd_telem_dir);
10109 break;
10110 }
10111
10112 if (parse_telem_info_file(fd_telem_dir, "size", "%lu", &size)) {
10113 close(fd_telem_dir);
10114 break;
10115 }
10116
10117 if (guid != target_guid) {
10118 close(fd_telem_dir);
10119 continue;
10120 }
10121
10122 if (parse_telem_info_file(fd_telem_dir, "offset", "%lu", &offset)) {
10123 close(fd_telem_dir);
10124 break;
10125 }
10126
10127 assert(offset == 0);
10128
10129 fd_pmt = openat(fd_telem_dir, "telem", O_RDONLY);
10130 if (fd_pmt == -1)
10131 goto loop_cleanup_and_break;
10132
10133 mmap_size = ROUND_UP_TO_PAGE_SIZE(size);
10134 mmio = mmap(0, mmap_size, PROT_READ, MAP_SHARED, fd_pmt, 0);
10135 if (mmio != MAP_FAILED) {
10136 if (debug)
10137 fprintf(stderr, "%s: 0x%lx mmaped at: %p\n", __func__, guid, mmio);
10138
10139 new_pmt = calloc(1, sizeof(*new_pmt));
10140
10141 if (!new_pmt) {
10142 fprintf(stderr, "%s: Failed to allocate pmt_mmio\n", __func__);
10143 exit(1);
10144 }
10145
10146 /*
10147 * Create linked list of mmaped regions,
10148 * but preserve the ordering from sysfs.
10149 * Ordering is important for the user to
10150 * use the seq=%u parameter when adding a counter.
10151 */
10152 new_pmt->guid = guid;
10153 new_pmt->mmio_base = mmio;
10154 new_pmt->pmt_offset = offset;
10155 new_pmt->size = size;
10156 new_pmt->next = pmt_mmios;
10157
10158 if (last)
10159 last->next = new_pmt;
10160 else
10161 head = new_pmt;
10162
10163 last = new_pmt;
10164 }
10165
10166 loop_cleanup_and_break:
10167 close(fd_pmt);
10168 close(fd_telem_dir);
10169 }
10170
10171 pmt_diriter_remove(&pmt_iter);
10172
10173 /*
10174 * If we found something, stick just
10175 * created linked list to the front.
10176 */
10177 if (head)
10178 pmt_mmios = head;
10179
10180 return head;
10181 }
10182
pmt_mmio_find(unsigned int guid)10183 struct pmt_mmio *pmt_mmio_find(unsigned int guid)
10184 {
10185 struct pmt_mmio *pmmio = pmt_mmios;
10186
10187 while (pmmio) {
10188 if (pmmio->guid == guid)
10189 return pmmio;
10190
10191 pmmio = pmmio->next;
10192 }
10193
10194 return NULL;
10195 }
10196
pmt_get_counter_pointer(struct pmt_mmio * pmmio,unsigned long counter_offset)10197 void *pmt_get_counter_pointer(struct pmt_mmio *pmmio, unsigned long counter_offset)
10198 {
10199 char *ret;
10200
10201 /* Get base of mmaped PMT file. */
10202 ret = (char *)pmmio->mmio_base;
10203
10204 /*
10205 * Apply PMT MMIO offset to obtain beginning of the mmaped telemetry data.
10206 * It's not guaranteed that the mmaped memory begins with the telemetry data
10207 * - we might have to apply the offset first.
10208 */
10209 ret += pmmio->pmt_offset;
10210
10211 /* Apply the counter offset to get the address to the mmaped counter. */
10212 ret += counter_offset;
10213
10214 return ret;
10215 }
10216
pmt_add_guid(unsigned int guid,unsigned int seq)10217 struct pmt_mmio *pmt_add_guid(unsigned int guid, unsigned int seq)
10218 {
10219 struct pmt_mmio *ret;
10220
10221 ret = pmt_mmio_find(guid);
10222 if (!ret)
10223 ret = pmt_mmio_open(guid);
10224
10225 while (ret && seq) {
10226 ret = ret->next;
10227 --seq;
10228 }
10229
10230 return ret;
10231 }
10232
10233 enum pmt_open_mode {
10234 PMT_OPEN_TRY, /* Open failure is not an error. */
10235 PMT_OPEN_REQUIRED, /* Open failure is a fatal error. */
10236 };
10237
pmt_find_counter(struct pmt_counter * pcounter,const char * name)10238 struct pmt_counter *pmt_find_counter(struct pmt_counter *pcounter, const char *name)
10239 {
10240 while (pcounter) {
10241 if (strcmp(pcounter->name, name) == 0)
10242 break;
10243
10244 pcounter = pcounter->next;
10245 }
10246
10247 return pcounter;
10248 }
10249
pmt_get_scope_root(enum counter_scope scope)10250 struct pmt_counter **pmt_get_scope_root(enum counter_scope scope)
10251 {
10252 switch (scope) {
10253 case SCOPE_CPU:
10254 return &sys.pmt_tp;
10255 case SCOPE_CORE:
10256 return &sys.pmt_cp;
10257 case SCOPE_PACKAGE:
10258 return &sys.pmt_pp;
10259 }
10260
10261 __builtin_unreachable();
10262 }
10263
pmt_counter_add_domain(struct pmt_counter * pcounter,unsigned long * pmmio,unsigned int domain_id)10264 void pmt_counter_add_domain(struct pmt_counter *pcounter, unsigned long *pmmio, unsigned int domain_id)
10265 {
10266 /* Make sure the new domain fits. */
10267 if (domain_id >= pcounter->num_domains)
10268 pmt_counter_resize(pcounter, domain_id + 1);
10269
10270 assert(pcounter->domains);
10271 assert(domain_id < pcounter->num_domains);
10272
10273 pcounter->domains[domain_id].pcounter = pmmio;
10274 }
10275
pmt_add_counter(unsigned int guid,unsigned int seq,const char * name,enum pmt_datatype type,unsigned int lsb,unsigned int msb,unsigned int offset,enum counter_scope scope,enum counter_format format,unsigned int domain_id,enum pmt_open_mode mode)10276 int pmt_add_counter(unsigned int guid, unsigned int seq, const char *name, enum pmt_datatype type,
10277 unsigned int lsb, unsigned int msb, unsigned int offset, enum counter_scope scope,
10278 enum counter_format format, unsigned int domain_id, enum pmt_open_mode mode)
10279 {
10280 struct pmt_mmio *mmio;
10281 struct pmt_counter *pcounter;
10282 struct pmt_counter **const pmt_root = pmt_get_scope_root(scope);
10283 bool new_counter = false;
10284 int conflict = 0;
10285
10286 if (lsb > msb) {
10287 fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "lsb <= msb", name);
10288 exit(1);
10289 }
10290
10291 if (msb >= 64) {
10292 fprintf(stderr, "%s: %s: `%s` must be satisfied\n", __func__, "msb < 64", name);
10293 exit(1);
10294 }
10295
10296 mmio = pmt_add_guid(guid, seq);
10297 if (!mmio) {
10298 if (mode != PMT_OPEN_TRY) {
10299 fprintf(stderr, "%s: failed to map PMT MMIO for guid %x, seq %u\n", __func__, guid, seq);
10300 exit(1);
10301 }
10302
10303 return 1;
10304 }
10305
10306 if (offset >= mmio->size) {
10307 if (mode != PMT_OPEN_TRY) {
10308 fprintf(stderr, "%s: offset %u outside of PMT MMIO size %u\n", __func__, offset, mmio->size);
10309 exit(1);
10310 }
10311
10312 return 1;
10313 }
10314
10315 pcounter = pmt_find_counter(*pmt_root, name);
10316 if (!pcounter) {
10317 pcounter = calloc(1, sizeof(*pcounter));
10318 new_counter = true;
10319 }
10320
10321 if (new_counter) {
10322 strncpy(pcounter->name, name, ARRAY_SIZE(pcounter->name) - 1);
10323 pcounter->type = type;
10324 pcounter->scope = scope;
10325 pcounter->lsb = lsb;
10326 pcounter->msb = msb;
10327 pcounter->format = format;
10328 } else {
10329 conflict += pcounter->type != type;
10330 conflict += pcounter->scope != scope;
10331 conflict += pcounter->lsb != lsb;
10332 conflict += pcounter->msb != msb;
10333 conflict += pcounter->format != format;
10334 }
10335
10336 if (conflict) {
10337 fprintf(stderr, "%s: conflicting parameters for the PMT counter with the same name %s\n", __func__, name);
10338 exit(1);
10339 }
10340
10341 pmt_counter_add_domain(pcounter, pmt_get_counter_pointer(mmio, offset), domain_id);
10342
10343 if (new_counter) {
10344 pcounter->next = *pmt_root;
10345 *pmt_root = pcounter;
10346 }
10347
10348 return 0;
10349 }
10350
pmt_init(void)10351 void pmt_init(void)
10352 {
10353 int cpu_num;
10354 unsigned long seq, offset, mod_num;
10355
10356 if (BIC_IS_ENABLED(BIC_Diec6)) {
10357 pmt_add_counter(PMT_MTL_DC6_GUID, PMT_MTL_DC6_SEQ, "Die%c6", PMT_TYPE_XTAL_TIME,
10358 PMT_COUNTER_MTL_DC6_LSB, PMT_COUNTER_MTL_DC6_MSB, PMT_COUNTER_MTL_DC6_OFFSET, SCOPE_PACKAGE, FORMAT_DELTA, 0, PMT_OPEN_TRY);
10359 }
10360
10361 if (BIC_IS_ENABLED(BIC_CPU_c1e)) {
10362 seq = 0;
10363 offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE;
10364 mod_num = 0; /* Relative module number for current PMT file. */
10365
10366 /* Open the counter for each CPU. */
10367 for (cpu_num = 0; cpu_num < topo.max_cpu_num;) {
10368
10369 if (cpu_is_not_allowed(cpu_num))
10370 goto next_loop_iter;
10371
10372 /*
10373 * Set the scope to CPU, even though CWF report the counter per module.
10374 * CPUs inside the same module will read from the same location, instead of reporting zeros.
10375 *
10376 * CWF with newer firmware might require a PMT_TYPE_XTAL_TIME intead of PMT_TYPE_TCORE_CLOCK.
10377 */
10378 pmt_add_counter(PMT_CWF_MC1E_GUID, seq, "CPU%c1e", PMT_TYPE_TCORE_CLOCK,
10379 PMT_COUNTER_CWF_MC1E_LSB, PMT_COUNTER_CWF_MC1E_MSB, offset, SCOPE_CPU, FORMAT_DELTA, cpu_num, PMT_OPEN_TRY);
10380
10381 /*
10382 * Rather complex logic for each time we go to the next loop iteration,
10383 * so keep it as a label.
10384 */
10385 next_loop_iter:
10386 /*
10387 * Advance the cpu number and check if we should also advance offset to
10388 * the next counter inside the PMT file.
10389 *
10390 * On Clearwater Forest platform, the counter is reported per module,
10391 * so open the same counter for all of the CPUs inside the module.
10392 * That way, reported table show the correct value for all of the CPUs inside the module,
10393 * instead of zeros.
10394 */
10395 ++cpu_num;
10396 if (cpu_num % PMT_COUNTER_CWF_CPUS_PER_MODULE == 0) {
10397 offset += PMT_COUNTER_CWF_MC1E_OFFSET_INCREMENT;
10398 ++mod_num;
10399 }
10400
10401 /*
10402 * There are PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE in each PMT file.
10403 *
10404 * If that number is reached, seq must be incremented to advance to the next file in a sequence.
10405 * Offset inside that file and a module counter has to be reset.
10406 */
10407 if (mod_num == PMT_COUNTER_CWF_MC1E_NUM_MODULES_PER_FILE) {
10408 ++seq;
10409 offset = PMT_COUNTER_CWF_MC1E_OFFSET_BASE;
10410 mod_num = 0;
10411 }
10412 }
10413 }
10414 }
10415
turbostat_init()10416 void turbostat_init()
10417 {
10418 setup_all_buffers(true);
10419 set_master_cpu();
10420 check_msr_access();
10421 check_perf_access();
10422 process_cpuid();
10423 counter_info_init();
10424 probe_pm_features();
10425 msr_perf_init();
10426 linux_perf_init();
10427 rapl_perf_init();
10428 cstate_perf_init();
10429 perf_llc_init();
10430 perf_l2_init();
10431 added_perf_counters_init();
10432 pmt_init();
10433
10434 for_all_cpus(get_cpu_type, ODD_COUNTERS);
10435 for_all_cpus(get_cpu_type, EVEN_COUNTERS);
10436
10437 if (BIC_IS_ENABLED(BIC_IPC) && has_aperf_access && get_instr_count_fd(master_cpu) != -1)
10438 BIC_PRESENT(BIC_IPC);
10439
10440 /*
10441 * If TSC tweak is needed, but couldn't get it,
10442 * disable more BICs, since it can't be reported accurately.
10443 */
10444 if (platform->enable_tsc_tweak && !has_base_hz) {
10445 CLR_BIC(BIC_Busy, &bic_enabled);
10446 CLR_BIC(BIC_Bzy_MHz, &bic_enabled);
10447 }
10448 }
10449
affinitize_child(void)10450 void affinitize_child(void)
10451 {
10452 /* Prefer cpu_possible_set, if available */
10453 if (sched_setaffinity(0, cpu_possible_setsize, cpu_possible_set)) {
10454 warn("sched_setaffinity cpu_possible_set");
10455
10456 /* Otherwise, allow child to run on same cpu set as turbostat */
10457 if (sched_setaffinity(0, cpu_allowed_setsize, cpu_allowed_set))
10458 warn("sched_setaffinity cpu_allowed_set");
10459 }
10460 }
10461
fork_it(char ** argv)10462 int fork_it(char **argv)
10463 {
10464 pid_t child_pid;
10465 int status;
10466
10467 snapshot_proc_sysfs_files();
10468 status = for_all_cpus(get_counters, EVEN_COUNTERS);
10469 first_counter_read = 0;
10470 if (status)
10471 exit(status);
10472 gettimeofday(&tv_even, (struct timezone *)NULL);
10473
10474 child_pid = fork();
10475 if (!child_pid) {
10476 /* child */
10477 affinitize_child();
10478 execvp(argv[0], argv);
10479 err(errno, "exec %s", argv[0]);
10480 } else {
10481
10482 /* parent */
10483 if (child_pid == -1)
10484 err(1, "fork");
10485
10486 signal(SIGINT, SIG_IGN);
10487 signal(SIGQUIT, SIG_IGN);
10488 if (waitpid(child_pid, &status, 0) == -1)
10489 err(status, "waitpid");
10490
10491 if (WIFEXITED(status))
10492 status = WEXITSTATUS(status);
10493 }
10494 /*
10495 * n.b. fork_it() does not check for errors from for_all_cpus()
10496 * because re-starting is problematic when forking
10497 */
10498 snapshot_proc_sysfs_files();
10499 for_all_cpus(get_counters, ODD_COUNTERS);
10500 gettimeofday(&tv_odd, (struct timezone *)NULL);
10501 timersub(&tv_odd, &tv_even, &tv_delta);
10502 if (for_all_cpus_2(delta_cpu, ODD_COUNTERS, EVEN_COUNTERS))
10503 fprintf(outf, "%s: Counter reset detected\n", progname);
10504 delta_platform(&platform_counters_odd, &platform_counters_even);
10505
10506 compute_average(EVEN_COUNTERS);
10507 format_all_counters(EVEN_COUNTERS);
10508
10509 fprintf(outf, "%.6f sec\n", tv_delta.tv_sec + tv_delta.tv_usec / 1000000.0);
10510
10511 flush_output_stderr();
10512
10513 return status;
10514 }
10515
get_and_dump_counters(void)10516 int get_and_dump_counters(void)
10517 {
10518 int status;
10519
10520 snapshot_proc_sysfs_files();
10521 status = for_all_cpus(get_counters, ODD_COUNTERS);
10522 if (status)
10523 return status;
10524
10525 status = for_all_cpus(dump_counters, ODD_COUNTERS);
10526 if (status)
10527 return status;
10528
10529 flush_output_stdout();
10530
10531 return status;
10532 }
10533
print_version()10534 void print_version()
10535 {
10536 fprintf(outf, "turbostat version 2026.02.14 - Len Brown <lenb@kernel.org>\n");
10537 }
10538
10539 #define COMMAND_LINE_SIZE 2048
10540
print_bootcmd(void)10541 void print_bootcmd(void)
10542 {
10543 char bootcmd[COMMAND_LINE_SIZE];
10544 FILE *fp;
10545 int ret;
10546
10547 memset(bootcmd, 0, COMMAND_LINE_SIZE);
10548 fp = fopen("/proc/cmdline", "r");
10549 if (!fp)
10550 return;
10551
10552 ret = fread(bootcmd, sizeof(char), COMMAND_LINE_SIZE - 1, fp);
10553 if (ret) {
10554 bootcmd[ret] = '\0';
10555 /* the last character is already '\n' */
10556 fprintf(outf, "Kernel command line: %s", bootcmd);
10557 }
10558
10559 fclose(fp);
10560 }
10561
find_msrp_by_name(struct msr_counter * head,char * name)10562 struct msr_counter *find_msrp_by_name(struct msr_counter *head, char *name)
10563 {
10564 struct msr_counter *mp;
10565
10566 for (mp = head; mp; mp = mp->next) {
10567 if (debug)
10568 fprintf(stderr, "%s: %s %s\n", __func__, name, mp->name);
10569 if (!strcmp(name, mp->name))
10570 return mp;
10571 }
10572 return NULL;
10573 }
10574
add_counter(unsigned int msr_num,char * path,char * name,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format,int flags,int id)10575 int add_counter(unsigned int msr_num, char *path, char *name,
10576 unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format, int flags, int id)
10577 {
10578 struct msr_counter *msrp;
10579
10580 if (no_msr && msr_num)
10581 errx(1, "Requested MSR counter 0x%x, but in --no-msr mode", msr_num);
10582
10583 if (debug)
10584 fprintf(stderr, "%s(msr%d, %s, %s, width%d, scope%d, type%d, format%d, flags%x, id%d)\n",
10585 __func__, msr_num, path, name, width, scope, type, format, flags, id);
10586
10587 switch (scope) {
10588
10589 case SCOPE_CPU:
10590 msrp = find_msrp_by_name(sys.tp, name);
10591 if (msrp) {
10592 if (debug)
10593 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10594 break;
10595 }
10596 if (sys.added_thread_counters++ >= MAX_ADDED_THREAD_COUNTERS) {
10597 warnx("ignoring thread counter %s", name);
10598 return -1;
10599 }
10600 break;
10601 case SCOPE_CORE:
10602 msrp = find_msrp_by_name(sys.cp, name);
10603 if (msrp) {
10604 if (debug)
10605 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10606 break;
10607 }
10608 if (sys.added_core_counters++ >= MAX_ADDED_CORE_COUNTERS) {
10609 warnx("ignoring core counter %s", name);
10610 return -1;
10611 }
10612 break;
10613 case SCOPE_PACKAGE:
10614 msrp = find_msrp_by_name(sys.pp, name);
10615 if (msrp) {
10616 if (debug)
10617 fprintf(stderr, "%s: %s FOUND\n", __func__, name);
10618 break;
10619 }
10620 if (sys.added_package_counters++ >= MAX_ADDED_PACKAGE_COUNTERS) {
10621 warnx("ignoring package counter %s", name);
10622 return -1;
10623 }
10624 break;
10625 default:
10626 warnx("ignoring counter %s with unknown scope", name);
10627 return -1;
10628 }
10629
10630 if (msrp == NULL) {
10631 msrp = calloc(1, sizeof(struct msr_counter));
10632 if (msrp == NULL)
10633 err(-1, "calloc msr_counter");
10634
10635 msrp->msr_num = msr_num;
10636 strncpy(msrp->name, name, NAME_BYTES - 1);
10637 msrp->width = width;
10638 msrp->type = type;
10639 msrp->format = format;
10640 msrp->flags = flags;
10641
10642 switch (scope) {
10643 case SCOPE_CPU:
10644 msrp->next = sys.tp;
10645 sys.tp = msrp;
10646 break;
10647 case SCOPE_CORE:
10648 msrp->next = sys.cp;
10649 sys.cp = msrp;
10650 break;
10651 case SCOPE_PACKAGE:
10652 msrp->next = sys.pp;
10653 sys.pp = msrp;
10654 break;
10655 }
10656 }
10657
10658 if (path) {
10659 struct sysfs_path *sp;
10660
10661 sp = calloc(1, sizeof(struct sysfs_path));
10662 if (sp == NULL) {
10663 perror("calloc");
10664 exit(1);
10665 }
10666 strncpy(sp->path, path, PATH_BYTES - 1);
10667 sp->id = id;
10668 sp->next = msrp->sp;
10669 msrp->sp = sp;
10670 }
10671
10672 return 0;
10673 }
10674
10675 /*
10676 * Initialize the fields used for identifying and opening the counter.
10677 *
10678 * Defer the initialization of any runtime buffers for actually reading
10679 * the counters for when we initialize all perf counters, so we can later
10680 * easily call re_initialize().
10681 */
make_perf_counter_info(const char * perf_device,const char * perf_event,const char * name,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format)10682 struct perf_counter_info *make_perf_counter_info(const char *perf_device,
10683 const char *perf_event,
10684 const char *name,
10685 unsigned int width, enum counter_scope scope, enum counter_type type, enum counter_format format)
10686 {
10687 struct perf_counter_info *pinfo;
10688
10689 pinfo = calloc(1, sizeof(*pinfo));
10690 if (!pinfo)
10691 errx(1, "%s: Failed to allocate %s/%s\n", __func__, perf_device, perf_event);
10692
10693 strncpy(pinfo->device, perf_device, ARRAY_SIZE(pinfo->device) - 1);
10694 strncpy(pinfo->event, perf_event, ARRAY_SIZE(pinfo->event) - 1);
10695
10696 strncpy(pinfo->name, name, ARRAY_SIZE(pinfo->name) - 1);
10697 pinfo->width = width;
10698 pinfo->scope = scope;
10699 pinfo->type = type;
10700 pinfo->format = format;
10701
10702 return pinfo;
10703 }
10704
add_perf_counter(const char * perf_device,const char * perf_event,const char * name_buffer,unsigned int width,enum counter_scope scope,enum counter_type type,enum counter_format format)10705 int add_perf_counter(const char *perf_device, const char *perf_event, const char *name_buffer, unsigned int width,
10706 enum counter_scope scope, enum counter_type type, enum counter_format format)
10707 {
10708 struct perf_counter_info *pinfo;
10709
10710 switch (scope) {
10711 case SCOPE_CPU:
10712 if (sys.added_thread_perf_counters >= MAX_ADDED_THREAD_COUNTERS) {
10713 warnx("ignoring thread counter perf/%s/%s", perf_device, perf_event);
10714 return -1;
10715 }
10716 break;
10717
10718 case SCOPE_CORE:
10719 if (sys.added_core_perf_counters >= MAX_ADDED_CORE_COUNTERS) {
10720 warnx("ignoring core counter perf/%s/%s", perf_device, perf_event);
10721 return -1;
10722 }
10723 break;
10724
10725 case SCOPE_PACKAGE:
10726 if (sys.added_package_perf_counters >= MAX_ADDED_PACKAGE_COUNTERS) {
10727 warnx("ignoring package counter perf/%s/%s", perf_device, perf_event);
10728 return -1;
10729 }
10730 break;
10731 }
10732
10733 pinfo = make_perf_counter_info(perf_device, perf_event, name_buffer, width, scope, type, format);
10734
10735 if (!pinfo)
10736 return -1;
10737
10738 switch (scope) {
10739 case SCOPE_CPU:
10740 pinfo->next = sys.perf_tp;
10741 sys.perf_tp = pinfo;
10742 ++sys.added_thread_perf_counters;
10743 break;
10744
10745 case SCOPE_CORE:
10746 pinfo->next = sys.perf_cp;
10747 sys.perf_cp = pinfo;
10748 ++sys.added_core_perf_counters;
10749 break;
10750
10751 case SCOPE_PACKAGE:
10752 pinfo->next = sys.perf_pp;
10753 sys.perf_pp = pinfo;
10754 ++sys.added_package_perf_counters;
10755 break;
10756 }
10757
10758 // FIXME: we might not have debug here yet
10759 if (debug)
10760 fprintf(stderr, "%s: %s/%s, name: %s, scope%d\n", __func__, pinfo->device, pinfo->event, pinfo->name, pinfo->scope);
10761
10762 return 0;
10763 }
10764
parse_add_command_msr(char * add_command)10765 void parse_add_command_msr(char *add_command)
10766 {
10767 int msr_num = 0;
10768 char *path = NULL;
10769 char perf_device[PERF_DEV_NAME_BYTES] = "";
10770 char perf_event[PERF_EVT_NAME_BYTES] = "";
10771 char name_buffer[PERF_NAME_BYTES] = "";
10772 int width = 64;
10773 int fail = 0;
10774 enum counter_scope scope = SCOPE_CPU;
10775 enum counter_type type = COUNTER_CYCLES;
10776 enum counter_format format = FORMAT_DELTA;
10777
10778 while (add_command) {
10779
10780 if (sscanf(add_command, "msr0x%x", &msr_num) == 1)
10781 goto next;
10782
10783 if (sscanf(add_command, "msr%d", &msr_num) == 1)
10784 goto next;
10785
10786 BUILD_BUG_ON(ARRAY_SIZE(perf_device) <= 31);
10787 BUILD_BUG_ON(ARRAY_SIZE(perf_event) <= 31);
10788 if (sscanf(add_command, "perf/%31[^/]/%31[^,]", &perf_device[0], &perf_event[0]) == 2)
10789 goto next;
10790
10791 if (*add_command == '/') {
10792 path = add_command;
10793 goto next;
10794 }
10795
10796 if (sscanf(add_command, "u%d", &width) == 1) {
10797 if ((width == 32) || (width == 64))
10798 goto next;
10799 width = 64;
10800 }
10801 if (!strncmp(add_command, "cpu", strlen("cpu"))) {
10802 scope = SCOPE_CPU;
10803 goto next;
10804 }
10805 if (!strncmp(add_command, "core", strlen("core"))) {
10806 scope = SCOPE_CORE;
10807 goto next;
10808 }
10809 if (!strncmp(add_command, "package", strlen("package"))) {
10810 scope = SCOPE_PACKAGE;
10811 goto next;
10812 }
10813 if (!strncmp(add_command, "cycles", strlen("cycles"))) {
10814 type = COUNTER_CYCLES;
10815 goto next;
10816 }
10817 if (!strncmp(add_command, "seconds", strlen("seconds"))) {
10818 type = COUNTER_SECONDS;
10819 goto next;
10820 }
10821 if (!strncmp(add_command, "usec", strlen("usec"))) {
10822 type = COUNTER_USEC;
10823 goto next;
10824 }
10825 if (!strncmp(add_command, "raw", strlen("raw"))) {
10826 format = FORMAT_RAW;
10827 goto next;
10828 }
10829 if (!strncmp(add_command, "average", strlen("average"))) {
10830 format = FORMAT_AVERAGE;
10831 goto next;
10832 }
10833 if (!strncmp(add_command, "delta", strlen("delta"))) {
10834 format = FORMAT_DELTA;
10835 goto next;
10836 }
10837 if (!strncmp(add_command, "percent", strlen("percent"))) {
10838 format = FORMAT_PERCENT;
10839 goto next;
10840 }
10841
10842 BUILD_BUG_ON(ARRAY_SIZE(name_buffer) <= 18);
10843 if (sscanf(add_command, "%18s,%*s", name_buffer) == 1) {
10844 char *eos;
10845
10846 eos = strchr(name_buffer, ',');
10847 if (eos)
10848 *eos = '\0';
10849 goto next;
10850 }
10851
10852 next:
10853 add_command = strchr(add_command, ',');
10854 if (add_command) {
10855 *add_command = '\0';
10856 add_command++;
10857 }
10858
10859 }
10860 if ((msr_num == 0) && (path == NULL) && (perf_device[0] == '\0' || perf_event[0] == '\0')) {
10861 fprintf(stderr, "--add: (msrDDD | msr0xXXX | /path_to_counter | perf/device/event) required\n");
10862 fail++;
10863 }
10864
10865 /* Test for non-empty perf_device and perf_event */
10866 const bool is_perf_counter = perf_device[0] && perf_event[0];
10867
10868 /* generate default column header */
10869 if (*name_buffer == '\0') {
10870 if (is_perf_counter) {
10871 snprintf(name_buffer, ARRAY_SIZE(name_buffer), "perf/%s", perf_event);
10872 } else {
10873 if (width == 32)
10874 sprintf(name_buffer, "M0x%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
10875 else
10876 sprintf(name_buffer, "M0X%x%s", msr_num, format == FORMAT_PERCENT ? "%" : "");
10877 }
10878 }
10879
10880 if (is_perf_counter) {
10881 if (add_perf_counter(perf_device, perf_event, name_buffer, width, scope, type, format))
10882 fail++;
10883 } else {
10884 if (add_counter(msr_num, path, name_buffer, width, scope, type, format, 0, 0))
10885 fail++;
10886 }
10887
10888 if (fail) {
10889 help();
10890 exit(1);
10891 }
10892 }
10893
starts_with(const char * str,const char * prefix)10894 bool starts_with(const char *str, const char *prefix)
10895 {
10896 return strncmp(prefix, str, strlen(prefix)) == 0;
10897 }
10898
pmt_parse_from_path(const char * target_path,unsigned int * out_guid,unsigned int * out_seq)10899 int pmt_parse_from_path(const char *target_path, unsigned int *out_guid, unsigned int *out_seq)
10900 {
10901 struct pmt_diriter_t pmt_iter;
10902 const struct dirent *dirname;
10903 struct stat stat, target_stat;
10904 int fd_telem_dir = -1;
10905 int fd_target_dir;
10906 unsigned int seq = 0;
10907 unsigned long guid, target_guid;
10908 int ret = -1;
10909
10910 fd_target_dir = open(target_path, O_RDONLY | O_DIRECTORY);
10911 if (fd_target_dir == -1) {
10912 return -1;
10913 }
10914
10915 if (fstat(fd_target_dir, &target_stat) == -1) {
10916 fprintf(stderr, "%s: Failed to stat the target: %s", __func__, strerror(errno));
10917 exit(1);
10918 }
10919
10920 if (parse_telem_info_file(fd_target_dir, "guid", "%lx", &target_guid)) {
10921 fprintf(stderr, "%s: Failed to parse the target guid file: %s", __func__, strerror(errno));
10922 exit(1);
10923 }
10924
10925 close(fd_target_dir);
10926
10927 pmt_diriter_init(&pmt_iter);
10928
10929 for (dirname = pmt_diriter_begin(&pmt_iter, SYSFS_TELEM_PATH); dirname != NULL; dirname = pmt_diriter_next(&pmt_iter)) {
10930
10931 fd_telem_dir = openat(dirfd(pmt_iter.dir), dirname->d_name, O_RDONLY | O_DIRECTORY);
10932 if (fd_telem_dir == -1)
10933 continue;
10934
10935 if (parse_telem_info_file(fd_telem_dir, "guid", "%lx", &guid)) {
10936 fprintf(stderr, "%s: Failed to parse the guid file: %s", __func__, strerror(errno));
10937 continue;
10938 }
10939
10940 if (fstat(fd_telem_dir, &stat) == -1) {
10941 fprintf(stderr, "%s: Failed to stat %s directory: %s", __func__, dirname->d_name, strerror(errno));
10942 continue;
10943 }
10944
10945 /*
10946 * If reached the same directory as target, exit the loop.
10947 * Seq has the correct value now.
10948 */
10949 if (stat.st_dev == target_stat.st_dev && stat.st_ino == target_stat.st_ino) {
10950 ret = 0;
10951 break;
10952 }
10953
10954 /*
10955 * If reached directory with the same guid,
10956 * but it's not the target directory yet,
10957 * increment seq and continue the search.
10958 */
10959 if (guid == target_guid)
10960 ++seq;
10961
10962 close(fd_telem_dir);
10963 fd_telem_dir = -1;
10964 }
10965
10966 pmt_diriter_remove(&pmt_iter);
10967
10968 if (fd_telem_dir != -1)
10969 close(fd_telem_dir);
10970
10971 if (!ret) {
10972 *out_guid = target_guid;
10973 *out_seq = seq;
10974 }
10975
10976 return ret;
10977 }
10978
parse_add_command_pmt(char * add_command)10979 void parse_add_command_pmt(char *add_command)
10980 {
10981 char *name = NULL;
10982 char *type_name = NULL;
10983 char *format_name = NULL;
10984 char *direct_path = NULL;
10985 static const char direct_path_prefix[] = "path=";
10986 unsigned int offset;
10987 unsigned int lsb;
10988 unsigned int msb;
10989 unsigned int guid;
10990 unsigned int seq = 0; /* By default, pick first file in a sequence with a given GUID. */
10991 unsigned int domain_id;
10992 enum counter_scope scope = 0;
10993 enum pmt_datatype type = PMT_TYPE_RAW;
10994 enum counter_format format = FORMAT_RAW;
10995 bool has_offset = false;
10996 bool has_lsb = false;
10997 bool has_msb = false;
10998 bool has_format = true; /* Format has a default value. */
10999 bool has_guid = false;
11000 bool has_scope = false;
11001 bool has_type = true; /* Type has a default value. */
11002
11003 /* Consume the "pmt," prefix. */
11004 add_command = strchr(add_command, ',');
11005 if (!add_command) {
11006 help();
11007 exit(1);
11008 }
11009 ++add_command;
11010
11011 while (add_command) {
11012 if (starts_with(add_command, "name=")) {
11013 name = add_command + strlen("name=");
11014 goto next;
11015 }
11016
11017 if (starts_with(add_command, "type=")) {
11018 type_name = add_command + strlen("type=");
11019 goto next;
11020 }
11021
11022 if (starts_with(add_command, "domain=")) {
11023 const size_t prefix_len = strlen("domain=");
11024
11025 if (sscanf(add_command + prefix_len, "cpu%u", &domain_id) == 1) {
11026 scope = SCOPE_CPU;
11027 has_scope = true;
11028 } else if (sscanf(add_command + prefix_len, "core%u", &domain_id) == 1) {
11029 scope = SCOPE_CORE;
11030 has_scope = true;
11031 } else if (sscanf(add_command + prefix_len, "package%u", &domain_id) == 1) {
11032 scope = SCOPE_PACKAGE;
11033 has_scope = true;
11034 }
11035
11036 if (!has_scope) {
11037 printf("%s: invalid value for scope. Expected cpu%%u, core%%u or package%%u.\n", __func__);
11038 exit(1);
11039 }
11040
11041 goto next;
11042 }
11043
11044 if (starts_with(add_command, "format=")) {
11045 format_name = add_command + strlen("format=");
11046 goto next;
11047 }
11048
11049 if (sscanf(add_command, "offset=%u", &offset) == 1) {
11050 has_offset = true;
11051 goto next;
11052 }
11053
11054 if (sscanf(add_command, "lsb=%u", &lsb) == 1) {
11055 has_lsb = true;
11056 goto next;
11057 }
11058
11059 if (sscanf(add_command, "msb=%u", &msb) == 1) {
11060 has_msb = true;
11061 goto next;
11062 }
11063
11064 if (sscanf(add_command, "guid=%x", &guid) == 1) {
11065 has_guid = true;
11066 goto next;
11067 }
11068
11069 if (sscanf(add_command, "seq=%x", &seq) == 1)
11070 goto next;
11071
11072 if (strncmp(add_command, direct_path_prefix, strlen(direct_path_prefix)) == 0) {
11073 direct_path = add_command + strlen(direct_path_prefix);
11074 goto next;
11075 }
11076 next:
11077 add_command = strchr(add_command, ',');
11078 if (add_command) {
11079 *add_command = '\0';
11080 add_command++;
11081 }
11082 }
11083
11084 if (!name) {
11085 printf("%s: missing %s\n", __func__, "name");
11086 exit(1);
11087 }
11088
11089 if (strlen(name) >= PMT_COUNTER_NAME_SIZE_BYTES) {
11090 printf("%s: name has to be at most %d characters long\n", __func__, PMT_COUNTER_NAME_SIZE_BYTES);
11091 exit(1);
11092 }
11093
11094 if (format_name) {
11095 has_format = false;
11096
11097 if (strcmp("raw", format_name) == 0) {
11098 format = FORMAT_RAW;
11099 has_format = true;
11100 }
11101
11102 if (strcmp("average", format_name) == 0) {
11103 format = FORMAT_AVERAGE;
11104 has_format = true;
11105 }
11106
11107 if (strcmp("delta", format_name) == 0) {
11108 format = FORMAT_DELTA;
11109 has_format = true;
11110 }
11111
11112 if (!has_format) {
11113 fprintf(stderr, "%s: Invalid format %s. Expected raw, average or delta\n", __func__, format_name);
11114 exit(1);
11115 }
11116 }
11117
11118 if (type_name) {
11119 has_type = false;
11120
11121 if (strcmp("raw", type_name) == 0) {
11122 type = PMT_TYPE_RAW;
11123 has_type = true;
11124 }
11125
11126 if (strcmp("txtal_time", type_name) == 0) {
11127 type = PMT_TYPE_XTAL_TIME;
11128 has_type = true;
11129 }
11130
11131 if (strcmp("tcore_clock", type_name) == 0) {
11132 type = PMT_TYPE_TCORE_CLOCK;
11133 has_type = true;
11134 }
11135
11136 if (!has_type) {
11137 printf("%s: invalid %s: %s\n", __func__, "type", type_name);
11138 exit(1);
11139 }
11140 }
11141
11142 if (!has_offset) {
11143 printf("%s : missing %s\n", __func__, "offset");
11144 exit(1);
11145 }
11146
11147 if (!has_lsb) {
11148 printf("%s: missing %s\n", __func__, "lsb");
11149 exit(1);
11150 }
11151
11152 if (!has_msb) {
11153 printf("%s: missing %s\n", __func__, "msb");
11154 exit(1);
11155 }
11156
11157 if (direct_path && has_guid) {
11158 printf("%s: path and guid+seq parameters are mutually exclusive\nnotice: passed guid=0x%x and path=%s\n", __func__, guid, direct_path);
11159 exit(1);
11160 }
11161
11162 if (direct_path) {
11163 if (pmt_parse_from_path(direct_path, &guid, &seq)) {
11164 printf("%s: failed to parse PMT file from %s\n", __func__, direct_path);
11165 exit(1);
11166 }
11167
11168 /* GUID was just infered from the direct path. */
11169 has_guid = true;
11170 }
11171
11172 if (!has_guid) {
11173 printf("%s: missing %s\n", __func__, "guid or path");
11174 exit(1);
11175 }
11176
11177 if (!has_scope) {
11178 printf("%s: missing %s\n", __func__, "scope");
11179 exit(1);
11180 }
11181
11182 if (lsb > msb) {
11183 printf("%s: lsb > msb doesn't make sense\n", __func__);
11184 exit(1);
11185 }
11186
11187 pmt_add_counter(guid, seq, name, type, lsb, msb, offset, scope, format, domain_id, PMT_OPEN_REQUIRED);
11188 }
11189
parse_add_command(char * add_command)11190 void parse_add_command(char *add_command)
11191 {
11192 if (strncmp(add_command, "pmt", strlen("pmt")) == 0)
11193 return parse_add_command_pmt(add_command);
11194 return parse_add_command_msr(add_command);
11195 }
11196
is_deferred_add(char * name)11197 int is_deferred_add(char *name)
11198 {
11199 int i;
11200
11201 for (i = 0; i < deferred_add_index; ++i)
11202 if (!strcmp(name, deferred_add_names[i])) {
11203 deferred_add_consumed |= (1 << i);
11204 return 1;
11205 }
11206 return 0;
11207 }
11208
is_deferred_skip(char * name)11209 int is_deferred_skip(char *name)
11210 {
11211 int i;
11212
11213 for (i = 0; i < deferred_skip_index; ++i)
11214 if (!strcmp(name, deferred_skip_names[i])) {
11215 deferred_skip_consumed |= (1 << i);
11216 return 1;
11217 }
11218 return 0;
11219 }
11220
verify_deferred_consumed(void)11221 void verify_deferred_consumed(void)
11222 {
11223 int i;
11224 int fail = 0;
11225
11226 for (i = 0; i < deferred_add_index; ++i) {
11227 if (!(deferred_add_consumed & (1 << i))) {
11228 warnx("Counter '%s' can not be added.", deferred_add_names[i]);
11229 fail++;
11230 }
11231 }
11232 for (i = 0; i < deferred_skip_index; ++i) {
11233 if (!(deferred_skip_consumed & (1 << i))) {
11234 warnx("Counter '%s' can not be skipped.", deferred_skip_names[i]);
11235 fail++;
11236 }
11237 }
11238 if (fail)
11239 exit(-EINVAL);
11240 }
11241
probe_cpuidle_residency(void)11242 void probe_cpuidle_residency(void)
11243 {
11244 char path[64];
11245 char name_buf[16];
11246 FILE *input;
11247 int state;
11248 int min_state = 1024, max_state = 0;
11249 char *sp;
11250
11251 for (state = 10; state >= 0; --state) {
11252
11253 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state);
11254 input = fopen(path, "r");
11255 if (input == NULL)
11256 continue;
11257 if (!fgets(name_buf, sizeof(name_buf), input))
11258 err(1, "%s: failed to read file", path);
11259
11260 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
11261 sp = strchr(name_buf, '-');
11262 if (!sp)
11263 sp = strchrnul(name_buf, '\n');
11264 *sp = '%';
11265 *(sp + 1) = '\0';
11266
11267 remove_underbar(name_buf);
11268
11269 fclose(input);
11270
11271 sprintf(path, "cpuidle/state%d/time", state);
11272
11273 if (!DO_BIC(BIC_pct_idle) && !is_deferred_add(name_buf))
11274 continue;
11275
11276 if (is_deferred_skip(name_buf))
11277 continue;
11278
11279 add_counter(0, path, name_buf, 32, SCOPE_CPU, COUNTER_USEC, FORMAT_PERCENT, SYSFS_PERCPU, 0);
11280
11281 if (state > max_state)
11282 max_state = state;
11283 if (state < min_state)
11284 min_state = state;
11285 }
11286 }
11287
cpuidle_counter_wanted(char * name)11288 static bool cpuidle_counter_wanted(char *name)
11289 {
11290 if (is_deferred_skip(name))
11291 return false;
11292
11293 return DO_BIC(BIC_cpuidle) || is_deferred_add(name);
11294 }
11295
probe_cpuidle_counts(void)11296 void probe_cpuidle_counts(void)
11297 {
11298 char path[64];
11299 char name_buf[16];
11300 FILE *input;
11301 int state;
11302 int min_state = 1024, max_state = 0;
11303 char *sp;
11304
11305 if (!DO_BIC(BIC_cpuidle) && !deferred_add_index)
11306 return;
11307
11308 for (state = 10; state >= 0; --state) {
11309
11310 sprintf(path, "/sys/devices/system/cpu/cpu%d/cpuidle/state%d/name", master_cpu, state);
11311 input = fopen(path, "r");
11312 if (input == NULL)
11313 continue;
11314 if (!fgets(name_buf, sizeof(name_buf), input))
11315 err(1, "%s: failed to read file", path);
11316 fclose(input);
11317
11318 remove_underbar(name_buf);
11319
11320 /* truncate "C1-HSW\n" to "C1", or truncate "C1\n" to "C1" */
11321 sp = strchr(name_buf, '-');
11322 if (!sp)
11323 sp = strchrnul(name_buf, '\n');
11324
11325 /*
11326 * The 'below' sysfs file always contains 0 for the deepest state (largest index),
11327 * do not add it.
11328 */
11329 if (state != max_state) {
11330 /*
11331 * Add 'C1+' for C1, and so on. The 'below' sysfs file always contains 0 for
11332 * the last state, so do not add it.
11333 */
11334 *sp = '+';
11335 *(sp + 1) = '\0';
11336 if (cpuidle_counter_wanted(name_buf)) {
11337 sprintf(path, "cpuidle/state%d/below", state);
11338 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
11339 }
11340 }
11341
11342 *sp = '\0';
11343 if (cpuidle_counter_wanted(name_buf)) {
11344 sprintf(path, "cpuidle/state%d/usage", state);
11345 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
11346 }
11347
11348 /*
11349 * The 'above' sysfs file always contains 0 for the shallowest state (smallest
11350 * index), do not add it.
11351 */
11352 if (state != min_state) {
11353 *sp = '-';
11354 *(sp + 1) = '\0';
11355 if (cpuidle_counter_wanted(name_buf)) {
11356 sprintf(path, "cpuidle/state%d/above", state);
11357 add_counter(0, path, name_buf, 64, SCOPE_CPU, COUNTER_ITEMS, FORMAT_DELTA, SYSFS_PERCPU, 0);
11358 }
11359 }
11360 }
11361 }
11362
11363 /*
11364 * parse cpuset with following syntax
11365 * 1,2,4..6,8-10 and set bits in cpu_subset
11366 */
parse_cpu_command(char * optarg)11367 void parse_cpu_command(char *optarg)
11368 {
11369 if (!strcmp(optarg, "core")) {
11370 if (cpu_subset)
11371 goto error;
11372 show_core_only++;
11373 return;
11374 }
11375 if (!strcmp(optarg, "package")) {
11376 if (cpu_subset)
11377 goto error;
11378 show_pkg_only++;
11379 return;
11380 }
11381 if (show_core_only || show_pkg_only)
11382 goto error;
11383
11384 cpu_subset = CPU_ALLOC(CPU_SUBSET_MAXCPUS);
11385 if (cpu_subset == NULL)
11386 err(3, "CPU_ALLOC");
11387 cpu_subset_size = CPU_ALLOC_SIZE(CPU_SUBSET_MAXCPUS);
11388
11389 CPU_ZERO_S(cpu_subset_size, cpu_subset);
11390
11391 if (parse_cpu_str(optarg, cpu_subset, cpu_subset_size))
11392 goto error;
11393
11394 return;
11395
11396 error:
11397 fprintf(stderr, "\"--cpu %s\" malformed\n", optarg);
11398 help();
11399 exit(-1);
11400 }
11401
cmdline(int argc,char ** argv)11402 void cmdline(int argc, char **argv)
11403 {
11404 int opt;
11405 int option_index = 0;
11406 static struct option long_options[] = {
11407 { "add", required_argument, 0, 'a' },
11408 { "cpu", required_argument, 0, 'c' },
11409 { "Dump", no_argument, 0, 'D' },
11410 { "debug", no_argument, 0, 'd' }, /* internal, not documented */
11411 { "enable", required_argument, 0, 'e' },
11412 { "force", no_argument, 0, 'f' },
11413 { "interval", required_argument, 0, 'i' },
11414 { "IPC", no_argument, 0, 'I' },
11415 { "num_iterations", required_argument, 0, 'n' },
11416 { "header_iterations", required_argument, 0, 'N' },
11417 { "help", no_argument, 0, 'h' },
11418 { "hide", required_argument, 0, 'H' }, // meh, -h taken by --help
11419 { "Joules", no_argument, 0, 'J' },
11420 { "list", no_argument, 0, 'l' },
11421 { "out", required_argument, 0, 'o' },
11422 { "quiet", no_argument, 0, 'q' },
11423 { "no-msr", no_argument, 0, 'M' },
11424 { "no-perf", no_argument, 0, 'P' },
11425 { "show", required_argument, 0, 's' },
11426 { "Summary", no_argument, 0, 'S' },
11427 { "TCC", required_argument, 0, 'T' },
11428 { "version", no_argument, 0, 'v' },
11429 { 0, 0, 0, 0 }
11430 };
11431
11432 progname = argv[0];
11433
11434 /*
11435 * Parse some options early, because they may make other options invalid,
11436 * like adding the MSR counter with --add and at the same time using --no-msr.
11437 */
11438 while ((opt = getopt_long_only(argc, argv, "+:MP", long_options, &option_index)) != -1) {
11439 switch (opt) {
11440 case 'M':
11441 no_msr = 1;
11442 break;
11443 case 'P':
11444 no_perf = 1;
11445 break;
11446 default:
11447 break;
11448 }
11449 }
11450 optind = 0;
11451
11452 while ((opt = getopt_long_only(argc, argv, "+C:c:Dde:hi:Jn:N:o:qMST:v", long_options, &option_index)) != -1) {
11453 switch (opt) {
11454 case 'a':
11455 parse_add_command(optarg);
11456 break;
11457 case 'c':
11458 parse_cpu_command(optarg);
11459 break;
11460 case 'D':
11461 dump_only++;
11462 /*
11463 * Force the no_perf early to prevent using it as a source.
11464 * User asks for raw values, but perf returns them relative
11465 * to the opening of the file descriptor.
11466 */
11467 no_perf = 1;
11468 break;
11469 case 'e':
11470 /* --enable specified counter, without clearning existing list */
11471 bic_lookup(&bic_enabled, optarg, SHOW_LIST);
11472 break;
11473 case 'f':
11474 force_load++;
11475 break;
11476 case 'd':
11477 debug++;
11478 bic_set_all(&bic_enabled);
11479 break;
11480 case 'H':
11481 /*
11482 * --hide: do not show those specified
11483 * multiple invocations simply clear more bits in enabled mask
11484 */
11485 {
11486 cpu_set_t bic_group_hide;
11487
11488 BIC_INIT(&bic_group_hide);
11489
11490 bic_lookup(&bic_group_hide, optarg, HIDE_LIST);
11491 bic_clear_bits(&bic_enabled, &bic_group_hide);
11492 }
11493 break;
11494 case 'h':
11495 help();
11496 exit(1);
11497 case 'i':
11498 {
11499 double interval = strtod(optarg, NULL);
11500
11501 if (interval < 0.001) {
11502 fprintf(outf, "interval %f seconds is too small\n", interval);
11503 exit(2);
11504 }
11505
11506 interval_tv.tv_sec = interval_ts.tv_sec = interval;
11507 interval_tv.tv_usec = (interval - interval_tv.tv_sec) * 1000000;
11508 interval_ts.tv_nsec = (interval - interval_ts.tv_sec) * 1000000000;
11509 }
11510 break;
11511 case 'J':
11512 rapl_joules++;
11513 break;
11514 case 'l':
11515 bic_set_all(&bic_enabled);
11516 list_header_only++;
11517 quiet++;
11518 break;
11519 case 'o':
11520 outf = fopen_or_die(optarg, "w");
11521 break;
11522 case 'q':
11523 quiet = 1;
11524 break;
11525 case 'M':
11526 case 'P':
11527 /* Parsed earlier */
11528 break;
11529 case 'n':
11530 num_iterations = strtoul(optarg, NULL, 0);
11531 errno = 0;
11532
11533 if (errno || num_iterations == 0)
11534 errx(-1, "invalid iteration count: %s", optarg);
11535 break;
11536 case 'N':
11537 header_iterations = strtoul(optarg, NULL, 0);
11538 errno = 0;
11539
11540 if (errno || header_iterations == 0)
11541 errx(-1, "invalid header iteration count: %s", optarg);
11542 break;
11543 case 's':
11544 /*
11545 * --show: show only those specified
11546 * The 1st invocation will clear and replace the enabled mask
11547 * subsequent invocations can add to it.
11548 */
11549 if (shown == 0)
11550 BIC_INIT(&bic_enabled);
11551 bic_lookup(&bic_enabled, optarg, SHOW_LIST);
11552 shown = 1;
11553 break;
11554 case 'S':
11555 summary_only++;
11556 break;
11557 case 'T':
11558 tj_max_override = atoi(optarg);
11559 break;
11560 case 'v':
11561 print_version();
11562 exit(0);
11563 break;
11564 default:
11565 help();
11566 exit(1);
11567 }
11568 }
11569 }
11570
set_rlimit(void)11571 void set_rlimit(void)
11572 {
11573 struct rlimit limit;
11574
11575 if (getrlimit(RLIMIT_NOFILE, &limit) < 0)
11576 err(1, "Failed to get rlimit");
11577
11578 if (limit.rlim_max < MAX_NOFILE)
11579 limit.rlim_max = MAX_NOFILE;
11580 if (limit.rlim_cur < MAX_NOFILE)
11581 limit.rlim_cur = MAX_NOFILE;
11582
11583 if (setrlimit(RLIMIT_NOFILE, &limit) < 0)
11584 err(1, "Failed to set rlimit");
11585 }
11586
main(int argc,char ** argv)11587 int main(int argc, char **argv)
11588 {
11589 int fd, ret;
11590
11591 bic_groups_init();
11592
11593 fd = open("/sys/fs/cgroup/cgroup.procs", O_WRONLY);
11594 if (fd < 0)
11595 goto skip_cgroup_setting;
11596
11597 ret = write(fd, "0\n", 2);
11598 if (ret == -1)
11599 perror("Can't update cgroup\n");
11600
11601 close(fd);
11602
11603 skip_cgroup_setting:
11604 outf = stderr;
11605 cmdline(argc, argv);
11606
11607 if (!quiet) {
11608 print_version();
11609 print_bootcmd();
11610 }
11611
11612 probe_cpuidle_residency();
11613 probe_cpuidle_counts();
11614
11615 verify_deferred_consumed();
11616
11617 if (!getuid())
11618 set_rlimit();
11619
11620 turbostat_init();
11621
11622 if (!no_msr)
11623 msr_sum_record();
11624
11625 /* dump counters and exit */
11626 if (dump_only)
11627 return get_and_dump_counters();
11628
11629 /* list header and exit */
11630 if (list_header_only) {
11631 print_header(",");
11632 flush_output_stdout();
11633 return 0;
11634 }
11635
11636 /*
11637 * if any params left, it must be a command to fork
11638 */
11639 if (argc - optind)
11640 return fork_it(argv + optind);
11641 else
11642 turbostat_loop();
11643
11644 return 0;
11645 }
11646