1 // SPDX-License-Identifier: MIT
2 /*
3  * Copyright © 2019 Intel Corporation
4  */
5 
6 #include <linux/string_helpers.h>
7 
8 #include "i915_drv.h"
9 #include "i915_perf_types.h"
10 #include "intel_engine_regs.h"
11 #include "intel_gt_regs.h"
12 #include "intel_sseu.h"
13 
intel_sseu_set_info(struct sseu_dev_info * sseu,u8 max_slices,u8 max_subslices,u8 max_eus_per_subslice)14 void intel_sseu_set_info(struct sseu_dev_info *sseu, u8 max_slices,
15 			 u8 max_subslices, u8 max_eus_per_subslice)
16 {
17 	sseu->max_slices = max_slices;
18 	sseu->max_subslices = max_subslices;
19 	sseu->max_eus_per_subslice = max_eus_per_subslice;
20 }
21 
22 unsigned int
intel_sseu_subslice_total(const struct sseu_dev_info * sseu)23 intel_sseu_subslice_total(const struct sseu_dev_info *sseu)
24 {
25 	unsigned int i, total = 0;
26 
27 	if (sseu->has_xehp_dss)
28 		return bitmap_weight(sseu->subslice_mask.xehp,
29 				     XEHP_BITMAP_BITS(sseu->subslice_mask));
30 
31 	for (i = 0; i < ARRAY_SIZE(sseu->subslice_mask.hsw); i++)
32 		total += hweight8(sseu->subslice_mask.hsw[i]);
33 
34 	return total;
35 }
36 
37 unsigned int
intel_sseu_get_hsw_subslices(const struct sseu_dev_info * sseu,u8 slice)38 intel_sseu_get_hsw_subslices(const struct sseu_dev_info *sseu, u8 slice)
39 {
40 	WARN_ON(sseu->has_xehp_dss);
41 	if (WARN_ON(slice >= sseu->max_slices))
42 		return 0;
43 
44 	return sseu->subslice_mask.hsw[slice];
45 }
46 
sseu_get_eus(const struct sseu_dev_info * sseu,int slice,int subslice)47 static u16 sseu_get_eus(const struct sseu_dev_info *sseu, int slice,
48 			int subslice)
49 {
50 	if (sseu->has_xehp_dss) {
51 		WARN_ON(slice > 0);
52 		return sseu->eu_mask.xehp[subslice];
53 	} else {
54 		return sseu->eu_mask.hsw[slice][subslice];
55 	}
56 }
57 
sseu_set_eus(struct sseu_dev_info * sseu,int slice,int subslice,u16 eu_mask)58 static void sseu_set_eus(struct sseu_dev_info *sseu, int slice, int subslice,
59 			 u16 eu_mask)
60 {
61 	GEM_WARN_ON(eu_mask && __fls(eu_mask) >= sseu->max_eus_per_subslice);
62 	if (sseu->has_xehp_dss) {
63 		GEM_WARN_ON(slice > 0);
64 		sseu->eu_mask.xehp[subslice] = eu_mask;
65 	} else {
66 		sseu->eu_mask.hsw[slice][subslice] = eu_mask;
67 	}
68 }
69 
compute_eu_total(const struct sseu_dev_info * sseu)70 static u16 compute_eu_total(const struct sseu_dev_info *sseu)
71 {
72 	int s, ss, total = 0;
73 
74 	for (s = 0; s < sseu->max_slices; s++)
75 		for (ss = 0; ss < sseu->max_subslices; ss++)
76 			if (sseu->has_xehp_dss)
77 				total += hweight16(sseu->eu_mask.xehp[ss]);
78 			else
79 				total += hweight16(sseu->eu_mask.hsw[s][ss]);
80 
81 	return total;
82 }
83 
84 /**
85  * intel_sseu_copy_eumask_to_user - Copy EU mask into a userspace buffer
86  * @to: Pointer to userspace buffer to copy to
87  * @sseu: SSEU structure containing EU mask to copy
88  *
89  * Copies the EU mask to a userspace buffer in the format expected by
90  * the query ioctl's topology queries.
91  *
92  * Returns the result of the copy_to_user() operation.
93  */
intel_sseu_copy_eumask_to_user(void __user * to,const struct sseu_dev_info * sseu)94 int intel_sseu_copy_eumask_to_user(void __user *to,
95 				   const struct sseu_dev_info *sseu)
96 {
97 	u8 eu_mask[GEN_SS_MASK_SIZE * GEN_MAX_EU_STRIDE] = {};
98 	int eu_stride = GEN_SSEU_STRIDE(sseu->max_eus_per_subslice);
99 	int len = sseu->max_slices * sseu->max_subslices * eu_stride;
100 	int s, ss, i;
101 
102 	for (s = 0; s < sseu->max_slices; s++) {
103 		for (ss = 0; ss < sseu->max_subslices; ss++) {
104 			int uapi_offset =
105 				s * sseu->max_subslices * eu_stride +
106 				ss * eu_stride;
107 			u16 mask = sseu_get_eus(sseu, s, ss);
108 
109 			for (i = 0; i < eu_stride; i++)
110 				eu_mask[uapi_offset + i] =
111 					(mask >> (BITS_PER_BYTE * i)) & 0xff;
112 		}
113 	}
114 
115 	return copy_to_user(to, eu_mask, len);
116 }
117 
118 /**
119  * intel_sseu_copy_ssmask_to_user - Copy subslice mask into a userspace buffer
120  * @to: Pointer to userspace buffer to copy to
121  * @sseu: SSEU structure containing subslice mask to copy
122  *
123  * Copies the subslice mask to a userspace buffer in the format expected by
124  * the query ioctl's topology queries.
125  *
126  * Returns the result of the copy_to_user() operation.
127  */
intel_sseu_copy_ssmask_to_user(void __user * to,const struct sseu_dev_info * sseu)128 int intel_sseu_copy_ssmask_to_user(void __user *to,
129 				   const struct sseu_dev_info *sseu)
130 {
131 	u8 ss_mask[GEN_SS_MASK_SIZE] = {};
132 	int ss_stride = GEN_SSEU_STRIDE(sseu->max_subslices);
133 	int len = sseu->max_slices * ss_stride;
134 	int s, ss, i;
135 
136 	for (s = 0; s < sseu->max_slices; s++) {
137 		for (ss = 0; ss < sseu->max_subslices; ss++) {
138 			i = s * ss_stride * BITS_PER_BYTE + ss;
139 
140 			if (!intel_sseu_has_subslice(sseu, s, ss))
141 				continue;
142 
143 			ss_mask[i / BITS_PER_BYTE] |= BIT(i % BITS_PER_BYTE);
144 		}
145 	}
146 
147 	return copy_to_user(to, ss_mask, len);
148 }
149 
gen11_compute_sseu_info(struct sseu_dev_info * sseu,u32 ss_en,u16 eu_en)150 static void gen11_compute_sseu_info(struct sseu_dev_info *sseu,
151 				    u32 ss_en, u16 eu_en)
152 {
153 	u32 valid_ss_mask = GENMASK(sseu->max_subslices - 1, 0);
154 	int ss;
155 
156 	sseu->slice_mask |= BIT(0);
157 	sseu->subslice_mask.hsw[0] = ss_en & valid_ss_mask;
158 
159 	for (ss = 0; ss < sseu->max_subslices; ss++)
160 		if (intel_sseu_has_subslice(sseu, 0, ss))
161 			sseu_set_eus(sseu, 0, ss, eu_en);
162 
163 	sseu->eu_per_subslice = hweight16(eu_en);
164 	sseu->eu_total = compute_eu_total(sseu);
165 }
166 
xehp_compute_sseu_info(struct sseu_dev_info * sseu,u16 eu_en)167 static void xehp_compute_sseu_info(struct sseu_dev_info *sseu,
168 				   u16 eu_en)
169 {
170 	int ss;
171 
172 	sseu->slice_mask |= BIT(0);
173 
174 	bitmap_or(sseu->subslice_mask.xehp,
175 		  sseu->compute_subslice_mask.xehp,
176 		  sseu->geometry_subslice_mask.xehp,
177 		  XEHP_BITMAP_BITS(sseu->subslice_mask));
178 
179 	for (ss = 0; ss < sseu->max_subslices; ss++)
180 		if (intel_sseu_has_subslice(sseu, 0, ss))
181 			sseu_set_eus(sseu, 0, ss, eu_en);
182 
183 	sseu->eu_per_subslice = hweight16(eu_en);
184 	sseu->eu_total = compute_eu_total(sseu);
185 }
186 
187 static void
xehp_load_dss_mask(struct intel_uncore * uncore,intel_sseu_ss_mask_t * ssmask,int numregs,...)188 xehp_load_dss_mask(struct intel_uncore *uncore,
189 		   intel_sseu_ss_mask_t *ssmask,
190 		   int numregs,
191 		   ...)
192 {
193 	va_list argp;
194 	u32 fuse_val[I915_MAX_SS_FUSE_REGS] = {};
195 	int i;
196 
197 	if (WARN_ON(numregs > I915_MAX_SS_FUSE_REGS))
198 		numregs = I915_MAX_SS_FUSE_REGS;
199 
200 	va_start(argp, numregs);
201 	for (i = 0; i < numregs; i++)
202 		fuse_val[i] = intel_uncore_read(uncore, va_arg(argp, i915_reg_t));
203 	va_end(argp);
204 
205 	bitmap_from_arr32(ssmask->xehp, fuse_val, numregs * 32);
206 }
207 
xehp_sseu_info_init(struct intel_gt * gt)208 static void xehp_sseu_info_init(struct intel_gt *gt)
209 {
210 	struct sseu_dev_info *sseu = &gt->info.sseu;
211 	struct intel_uncore *uncore = gt->uncore;
212 	u16 eu_en = 0;
213 	u8 eu_en_fuse;
214 	int num_compute_regs, num_geometry_regs;
215 	int eu;
216 
217 	num_geometry_regs = 1;
218 	num_compute_regs = 1;
219 
220 	/*
221 	 * The concept of slice has been removed in Xe_HP.  To be compatible
222 	 * with prior generations, assume a single slice across the entire
223 	 * device. Then calculate out the DSS for each workload type within
224 	 * that software slice.
225 	 */
226 	intel_sseu_set_info(sseu, 1,
227 			    32 * max(num_geometry_regs, num_compute_regs),
228 			    HAS_ONE_EU_PER_FUSE_BIT(gt->i915) ? 8 : 16);
229 	sseu->has_xehp_dss = 1;
230 
231 	xehp_load_dss_mask(uncore, &sseu->geometry_subslice_mask,
232 			   num_geometry_regs,
233 			   GEN12_GT_GEOMETRY_DSS_ENABLE);
234 	xehp_load_dss_mask(uncore, &sseu->compute_subslice_mask,
235 			   num_compute_regs,
236 			   GEN12_GT_COMPUTE_DSS_ENABLE,
237 			   XEHPC_GT_COMPUTE_DSS_ENABLE_EXT);
238 
239 	eu_en_fuse = REG_FIELD_GET(XEHP_EU_ENA_MASK,
240 				   intel_uncore_read(uncore, XEHP_EU_ENABLE));
241 
242 	if (HAS_ONE_EU_PER_FUSE_BIT(gt->i915))
243 		eu_en = eu_en_fuse;
244 	else
245 		for (eu = 0; eu < sseu->max_eus_per_subslice / 2; eu++)
246 			if (eu_en_fuse & BIT(eu))
247 				eu_en |= BIT(eu * 2) | BIT(eu * 2 + 1);
248 
249 	xehp_compute_sseu_info(sseu, eu_en);
250 }
251 
gen12_sseu_info_init(struct intel_gt * gt)252 static void gen12_sseu_info_init(struct intel_gt *gt)
253 {
254 	struct sseu_dev_info *sseu = &gt->info.sseu;
255 	struct intel_uncore *uncore = gt->uncore;
256 	u32 g_dss_en;
257 	u16 eu_en = 0;
258 	u8 eu_en_fuse;
259 	u8 s_en;
260 	int eu;
261 
262 	/*
263 	 * Gen12 has Dual-Subslices, which behave similarly to 2 gen11 SS.
264 	 * Instead of splitting these, provide userspace with an array
265 	 * of DSS to more closely represent the hardware resource.
266 	 */
267 	intel_sseu_set_info(sseu, 1, 6, 16);
268 
269 	/*
270 	 * Although gen12 architecture supported multiple slices, TGL, RKL,
271 	 * DG1, and ADL only had a single slice.
272 	 */
273 	s_en = REG_FIELD_GET(GEN11_GT_S_ENA_MASK,
274 			     intel_uncore_read(uncore, GEN11_GT_SLICE_ENABLE));
275 	drm_WARN_ON(&gt->i915->drm, s_en != 0x1);
276 
277 	g_dss_en = intel_uncore_read(uncore, GEN12_GT_GEOMETRY_DSS_ENABLE);
278 
279 	/* one bit per pair of EUs */
280 	eu_en_fuse = ~REG_FIELD_GET(GEN11_EU_DIS_MASK,
281 				    intel_uncore_read(uncore, GEN11_EU_DISABLE));
282 
283 	for (eu = 0; eu < sseu->max_eus_per_subslice / 2; eu++)
284 		if (eu_en_fuse & BIT(eu))
285 			eu_en |= BIT(eu * 2) | BIT(eu * 2 + 1);
286 
287 	gen11_compute_sseu_info(sseu, g_dss_en, eu_en);
288 
289 	/* TGL only supports slice-level power gating */
290 	sseu->has_slice_pg = 1;
291 }
292 
gen11_sseu_info_init(struct intel_gt * gt)293 static void gen11_sseu_info_init(struct intel_gt *gt)
294 {
295 	struct sseu_dev_info *sseu = &gt->info.sseu;
296 	struct intel_uncore *uncore = gt->uncore;
297 	u32 ss_en;
298 	u8 eu_en;
299 	u8 s_en;
300 
301 	if (IS_JASPERLAKE(gt->i915) || IS_ELKHARTLAKE(gt->i915))
302 		intel_sseu_set_info(sseu, 1, 4, 8);
303 	else
304 		intel_sseu_set_info(sseu, 1, 8, 8);
305 
306 	/*
307 	 * Although gen11 architecture supported multiple slices, ICL and
308 	 * EHL/JSL only had a single slice in practice.
309 	 */
310 	s_en = REG_FIELD_GET(GEN11_GT_S_ENA_MASK,
311 			     intel_uncore_read(uncore, GEN11_GT_SLICE_ENABLE));
312 	drm_WARN_ON(&gt->i915->drm, s_en != 0x1);
313 
314 	ss_en = ~intel_uncore_read(uncore, GEN11_GT_SUBSLICE_DISABLE);
315 
316 	eu_en = ~REG_FIELD_GET(GEN11_EU_DIS_MASK,
317 			       intel_uncore_read(uncore, GEN11_EU_DISABLE));
318 
319 	gen11_compute_sseu_info(sseu, ss_en, eu_en);
320 
321 	/* ICL has no power gating restrictions. */
322 	sseu->has_slice_pg = 1;
323 	sseu->has_subslice_pg = 1;
324 	sseu->has_eu_pg = 1;
325 }
326 
cherryview_sseu_info_init(struct intel_gt * gt)327 static void cherryview_sseu_info_init(struct intel_gt *gt)
328 {
329 	struct sseu_dev_info *sseu = &gt->info.sseu;
330 	u32 fuse;
331 
332 	fuse = intel_uncore_read(gt->uncore, CHV_FUSE_GT);
333 
334 	sseu->slice_mask = BIT(0);
335 	intel_sseu_set_info(sseu, 1, 2, 8);
336 
337 	if (!(fuse & CHV_FGT_DISABLE_SS0)) {
338 		u8 disabled_mask =
339 			REG_FIELD_GET(CHV_FGT_EU_DIS_SS0_R0_MASK, fuse) |
340 			REG_FIELD_GET(CHV_FGT_EU_DIS_SS0_R1_MASK, fuse) << hweight32(CHV_FGT_EU_DIS_SS0_R0_MASK);
341 
342 		sseu->subslice_mask.hsw[0] |= BIT(0);
343 		sseu_set_eus(sseu, 0, 0, ~disabled_mask & 0xFF);
344 	}
345 
346 	if (!(fuse & CHV_FGT_DISABLE_SS1)) {
347 		u8 disabled_mask =
348 			REG_FIELD_GET(CHV_FGT_EU_DIS_SS1_R0_MASK, fuse) |
349 			REG_FIELD_GET(CHV_FGT_EU_DIS_SS1_R1_MASK, fuse) << hweight32(CHV_FGT_EU_DIS_SS1_R0_MASK);
350 
351 		sseu->subslice_mask.hsw[0] |= BIT(1);
352 		sseu_set_eus(sseu, 0, 1, ~disabled_mask & 0xFF);
353 	}
354 
355 	sseu->eu_total = compute_eu_total(sseu);
356 
357 	/*
358 	 * CHV expected to always have a uniform distribution of EU
359 	 * across subslices.
360 	 */
361 	sseu->eu_per_subslice = intel_sseu_subslice_total(sseu) ?
362 		sseu->eu_total /
363 		intel_sseu_subslice_total(sseu) :
364 		0;
365 	/*
366 	 * CHV supports subslice power gating on devices with more than
367 	 * one subslice, and supports EU power gating on devices with
368 	 * more than one EU pair per subslice.
369 	 */
370 	sseu->has_slice_pg = 0;
371 	sseu->has_subslice_pg = intel_sseu_subslice_total(sseu) > 1;
372 	sseu->has_eu_pg = (sseu->eu_per_subslice > 2);
373 }
374 
gen9_sseu_info_init(struct intel_gt * gt)375 static void gen9_sseu_info_init(struct intel_gt *gt)
376 {
377 	struct drm_i915_private *i915 = gt->i915;
378 	struct sseu_dev_info *sseu = &gt->info.sseu;
379 	struct intel_uncore *uncore = gt->uncore;
380 	u32 fuse2, eu_disable, subslice_mask;
381 	const u8 eu_mask = 0xff;
382 	int s, ss;
383 
384 	fuse2 = intel_uncore_read(uncore, GEN8_FUSE2);
385 	sseu->slice_mask = REG_FIELD_GET(GEN8_F2_S_ENA_MASK, fuse2);
386 
387 	/* BXT has a single slice and at most 3 subslices. */
388 	intel_sseu_set_info(sseu, IS_GEN9_LP(i915) ? 1 : 3,
389 			    IS_GEN9_LP(i915) ? 3 : 4, 8);
390 
391 	/*
392 	 * The subslice disable field is global, i.e. it applies
393 	 * to each of the enabled slices.
394 	 */
395 	subslice_mask = (1 << sseu->max_subslices) - 1;
396 	subslice_mask &= ~REG_FIELD_GET(GEN9_F2_SS_DIS_MASK, fuse2);
397 
398 	/*
399 	 * Iterate through enabled slices and subslices to
400 	 * count the total enabled EU.
401 	 */
402 	for (s = 0; s < sseu->max_slices; s++) {
403 		if (!(sseu->slice_mask & BIT(s)))
404 			/* skip disabled slice */
405 			continue;
406 
407 		sseu->subslice_mask.hsw[s] = subslice_mask;
408 
409 		eu_disable = intel_uncore_read(uncore, GEN9_EU_DISABLE(s));
410 		for (ss = 0; ss < sseu->max_subslices; ss++) {
411 			int eu_per_ss;
412 			u8 eu_disabled_mask;
413 
414 			if (!intel_sseu_has_subslice(sseu, s, ss))
415 				/* skip disabled subslice */
416 				continue;
417 
418 			eu_disabled_mask = (eu_disable >> (ss * 8)) & eu_mask;
419 
420 			sseu_set_eus(sseu, s, ss, ~eu_disabled_mask & eu_mask);
421 
422 			eu_per_ss = sseu->max_eus_per_subslice -
423 				hweight8(eu_disabled_mask);
424 
425 			/*
426 			 * Record which subslice(s) has(have) 7 EUs. we
427 			 * can tune the hash used to spread work among
428 			 * subslices if they are unbalanced.
429 			 */
430 			if (eu_per_ss == 7)
431 				sseu->subslice_7eu[s] |= BIT(ss);
432 		}
433 	}
434 
435 	sseu->eu_total = compute_eu_total(sseu);
436 
437 	/*
438 	 * SKL is expected to always have a uniform distribution
439 	 * of EU across subslices with the exception that any one
440 	 * EU in any one subslice may be fused off for die
441 	 * recovery. BXT is expected to be perfectly uniform in EU
442 	 * distribution.
443 	 */
444 	sseu->eu_per_subslice =
445 		intel_sseu_subslice_total(sseu) ?
446 		DIV_ROUND_UP(sseu->eu_total, intel_sseu_subslice_total(sseu)) :
447 		0;
448 
449 	/*
450 	 * SKL+ supports slice power gating on devices with more than
451 	 * one slice, and supports EU power gating on devices with
452 	 * more than one EU pair per subslice. BXT+ supports subslice
453 	 * power gating on devices with more than one subslice, and
454 	 * supports EU power gating on devices with more than one EU
455 	 * pair per subslice.
456 	 */
457 	sseu->has_slice_pg =
458 		!IS_GEN9_LP(i915) && hweight8(sseu->slice_mask) > 1;
459 	sseu->has_subslice_pg =
460 		IS_GEN9_LP(i915) && intel_sseu_subslice_total(sseu) > 1;
461 	sseu->has_eu_pg = sseu->eu_per_subslice > 2;
462 
463 	if (IS_GEN9_LP(i915)) {
464 #define IS_SS_DISABLED(ss)	(!(sseu->subslice_mask.hsw[0] & BIT(ss)))
465 		RUNTIME_INFO(i915)->has_pooled_eu = hweight8(sseu->subslice_mask.hsw[0]) == 3;
466 
467 		sseu->min_eu_in_pool = 0;
468 		if (HAS_POOLED_EU(i915)) {
469 			if (IS_SS_DISABLED(2) || IS_SS_DISABLED(0))
470 				sseu->min_eu_in_pool = 3;
471 			else if (IS_SS_DISABLED(1))
472 				sseu->min_eu_in_pool = 6;
473 			else
474 				sseu->min_eu_in_pool = 9;
475 		}
476 #undef IS_SS_DISABLED
477 	}
478 }
479 
bdw_sseu_info_init(struct intel_gt * gt)480 static void bdw_sseu_info_init(struct intel_gt *gt)
481 {
482 	struct sseu_dev_info *sseu = &gt->info.sseu;
483 	struct intel_uncore *uncore = gt->uncore;
484 	int s, ss;
485 	u32 fuse2, subslice_mask, eu_disable[3]; /* s_max */
486 	u32 eu_disable0, eu_disable1, eu_disable2;
487 
488 	fuse2 = intel_uncore_read(uncore, GEN8_FUSE2);
489 	sseu->slice_mask = REG_FIELD_GET(GEN8_F2_S_ENA_MASK, fuse2);
490 	intel_sseu_set_info(sseu, 3, 3, 8);
491 
492 	/*
493 	 * The subslice disable field is global, i.e. it applies
494 	 * to each of the enabled slices.
495 	 */
496 	subslice_mask = GENMASK(sseu->max_subslices - 1, 0);
497 	subslice_mask &= ~REG_FIELD_GET(GEN8_F2_SS_DIS_MASK, fuse2);
498 	eu_disable0 = intel_uncore_read(uncore, GEN8_EU_DISABLE0);
499 	eu_disable1 = intel_uncore_read(uncore, GEN8_EU_DISABLE1);
500 	eu_disable2 = intel_uncore_read(uncore, GEN8_EU_DISABLE2);
501 	eu_disable[0] =
502 		REG_FIELD_GET(GEN8_EU_DIS0_S0_MASK, eu_disable0);
503 	eu_disable[1] =
504 		REG_FIELD_GET(GEN8_EU_DIS0_S1_MASK, eu_disable0) |
505 		REG_FIELD_GET(GEN8_EU_DIS1_S1_MASK, eu_disable1) << hweight32(GEN8_EU_DIS0_S1_MASK);
506 	eu_disable[2] =
507 		REG_FIELD_GET(GEN8_EU_DIS1_S2_MASK, eu_disable1) |
508 		REG_FIELD_GET(GEN8_EU_DIS2_S2_MASK, eu_disable2) << hweight32(GEN8_EU_DIS1_S2_MASK);
509 
510 	/*
511 	 * Iterate through enabled slices and subslices to
512 	 * count the total enabled EU.
513 	 */
514 	for (s = 0; s < sseu->max_slices; s++) {
515 		if (!(sseu->slice_mask & BIT(s)))
516 			/* skip disabled slice */
517 			continue;
518 
519 		sseu->subslice_mask.hsw[s] = subslice_mask;
520 
521 		for (ss = 0; ss < sseu->max_subslices; ss++) {
522 			u8 eu_disabled_mask;
523 			u32 n_disabled;
524 
525 			if (!intel_sseu_has_subslice(sseu, s, ss))
526 				/* skip disabled subslice */
527 				continue;
528 
529 			eu_disabled_mask =
530 				eu_disable[s] >> (ss * sseu->max_eus_per_subslice);
531 
532 			sseu_set_eus(sseu, s, ss, ~eu_disabled_mask & 0xFF);
533 
534 			n_disabled = hweight8(eu_disabled_mask);
535 
536 			/*
537 			 * Record which subslices have 7 EUs.
538 			 */
539 			if (sseu->max_eus_per_subslice - n_disabled == 7)
540 				sseu->subslice_7eu[s] |= 1 << ss;
541 		}
542 	}
543 
544 	sseu->eu_total = compute_eu_total(sseu);
545 
546 	/*
547 	 * BDW is expected to always have a uniform distribution of EU across
548 	 * subslices with the exception that any one EU in any one subslice may
549 	 * be fused off for die recovery.
550 	 */
551 	sseu->eu_per_subslice =
552 		intel_sseu_subslice_total(sseu) ?
553 		DIV_ROUND_UP(sseu->eu_total, intel_sseu_subslice_total(sseu)) :
554 		0;
555 
556 	/*
557 	 * BDW supports slice power gating on devices with more than
558 	 * one slice.
559 	 */
560 	sseu->has_slice_pg = hweight8(sseu->slice_mask) > 1;
561 	sseu->has_subslice_pg = 0;
562 	sseu->has_eu_pg = 0;
563 }
564 
hsw_sseu_info_init(struct intel_gt * gt)565 static void hsw_sseu_info_init(struct intel_gt *gt)
566 {
567 	struct drm_i915_private *i915 = gt->i915;
568 	struct sseu_dev_info *sseu = &gt->info.sseu;
569 	u32 fuse1;
570 	u8 subslice_mask = 0;
571 	int s, ss;
572 
573 	/*
574 	 * There isn't a register to tell us how many slices/subslices. We
575 	 * work off the PCI-ids here.
576 	 */
577 	switch (INTEL_INFO(i915)->gt) {
578 	default:
579 		MISSING_CASE(INTEL_INFO(i915)->gt);
580 		fallthrough;
581 	case 1:
582 		sseu->slice_mask = BIT(0);
583 		subslice_mask = BIT(0);
584 		break;
585 	case 2:
586 		sseu->slice_mask = BIT(0);
587 		subslice_mask = BIT(0) | BIT(1);
588 		break;
589 	case 3:
590 		sseu->slice_mask = BIT(0) | BIT(1);
591 		subslice_mask = BIT(0) | BIT(1);
592 		break;
593 	}
594 
595 	fuse1 = intel_uncore_read(gt->uncore, HSW_PAVP_FUSE1);
596 	switch (REG_FIELD_GET(HSW_F1_EU_DIS_MASK, fuse1)) {
597 	default:
598 		MISSING_CASE(REG_FIELD_GET(HSW_F1_EU_DIS_MASK, fuse1));
599 		fallthrough;
600 	case HSW_F1_EU_DIS_10EUS:
601 		sseu->eu_per_subslice = 10;
602 		break;
603 	case HSW_F1_EU_DIS_8EUS:
604 		sseu->eu_per_subslice = 8;
605 		break;
606 	case HSW_F1_EU_DIS_6EUS:
607 		sseu->eu_per_subslice = 6;
608 		break;
609 	}
610 
611 	intel_sseu_set_info(sseu, hweight8(sseu->slice_mask),
612 			    hweight8(subslice_mask),
613 			    sseu->eu_per_subslice);
614 
615 	for (s = 0; s < sseu->max_slices; s++) {
616 		sseu->subslice_mask.hsw[s] = subslice_mask;
617 
618 		for (ss = 0; ss < sseu->max_subslices; ss++) {
619 			sseu_set_eus(sseu, s, ss,
620 				     (1UL << sseu->eu_per_subslice) - 1);
621 		}
622 	}
623 
624 	sseu->eu_total = compute_eu_total(sseu);
625 
626 	/* No powergating for you. */
627 	sseu->has_slice_pg = 0;
628 	sseu->has_subslice_pg = 0;
629 	sseu->has_eu_pg = 0;
630 }
631 
intel_sseu_info_init(struct intel_gt * gt)632 void intel_sseu_info_init(struct intel_gt *gt)
633 {
634 	struct drm_i915_private *i915 = gt->i915;
635 
636 	if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
637 		xehp_sseu_info_init(gt);
638 	else if (GRAPHICS_VER(i915) >= 12)
639 		gen12_sseu_info_init(gt);
640 	else if (GRAPHICS_VER(i915) >= 11)
641 		gen11_sseu_info_init(gt);
642 	else if (GRAPHICS_VER(i915) >= 9)
643 		gen9_sseu_info_init(gt);
644 	else if (IS_BROADWELL(i915))
645 		bdw_sseu_info_init(gt);
646 	else if (IS_CHERRYVIEW(i915))
647 		cherryview_sseu_info_init(gt);
648 	else if (IS_HASWELL(i915))
649 		hsw_sseu_info_init(gt);
650 }
651 
intel_sseu_make_rpcs(struct intel_gt * gt,const struct intel_sseu * req_sseu)652 u32 intel_sseu_make_rpcs(struct intel_gt *gt,
653 			 const struct intel_sseu *req_sseu)
654 {
655 	struct drm_i915_private *i915 = gt->i915;
656 	const struct sseu_dev_info *sseu = &gt->info.sseu;
657 	bool subslice_pg = sseu->has_subslice_pg;
658 	u8 slices, subslices;
659 	u32 rpcs = 0;
660 
661 	/*
662 	 * No explicit RPCS request is needed to ensure full
663 	 * slice/subslice/EU enablement prior to Gen9.
664 	 */
665 	if (GRAPHICS_VER(i915) < 9)
666 		return 0;
667 
668 	/*
669 	 * If i915/perf is active, we want a stable powergating configuration
670 	 * on the system. Use the configuration pinned by i915/perf.
671 	 */
672 	if (gt->perf.group && gt->perf.group[PERF_GROUP_OAG].exclusive_stream)
673 		req_sseu = &gt->perf.sseu;
674 
675 	slices = hweight8(req_sseu->slice_mask);
676 	subslices = hweight8(req_sseu->subslice_mask);
677 
678 	/*
679 	 * Since the SScount bitfield in GEN8_R_PWR_CLK_STATE is only three bits
680 	 * wide and Icelake has up to eight subslices, specfial programming is
681 	 * needed in order to correctly enable all subslices.
682 	 *
683 	 * According to documentation software must consider the configuration
684 	 * as 2x4x8 and hardware will translate this to 1x8x8.
685 	 *
686 	 * Furthermore, even though SScount is three bits, maximum documented
687 	 * value for it is four. From this some rules/restrictions follow:
688 	 *
689 	 * 1.
690 	 * If enabled subslice count is greater than four, two whole slices must
691 	 * be enabled instead.
692 	 *
693 	 * 2.
694 	 * When more than one slice is enabled, hardware ignores the subslice
695 	 * count altogether.
696 	 *
697 	 * From these restrictions it follows that it is not possible to enable
698 	 * a count of subslices between the SScount maximum of four restriction,
699 	 * and the maximum available number on a particular SKU. Either all
700 	 * subslices are enabled, or a count between one and four on the first
701 	 * slice.
702 	 */
703 	if (GRAPHICS_VER(i915) == 11 &&
704 	    slices == 1 &&
705 	    subslices > min_t(u8, 4, hweight8(sseu->subslice_mask.hsw[0]) / 2)) {
706 		GEM_BUG_ON(subslices & 1);
707 
708 		subslice_pg = false;
709 		slices *= 2;
710 	}
711 
712 	/*
713 	 * Starting in Gen9, render power gating can leave
714 	 * slice/subslice/EU in a partially enabled state. We
715 	 * must make an explicit request through RPCS for full
716 	 * enablement.
717 	 */
718 	if (sseu->has_slice_pg) {
719 		u32 mask, val = slices;
720 
721 		if (GRAPHICS_VER(i915) >= 11) {
722 			mask = GEN11_RPCS_S_CNT_MASK;
723 			val <<= GEN11_RPCS_S_CNT_SHIFT;
724 		} else {
725 			mask = GEN8_RPCS_S_CNT_MASK;
726 			val <<= GEN8_RPCS_S_CNT_SHIFT;
727 		}
728 
729 		GEM_BUG_ON(val & ~mask);
730 		val &= mask;
731 
732 		rpcs |= GEN8_RPCS_ENABLE | GEN8_RPCS_S_CNT_ENABLE | val;
733 	}
734 
735 	if (subslice_pg) {
736 		u32 val = subslices;
737 
738 		val <<= GEN8_RPCS_SS_CNT_SHIFT;
739 
740 		GEM_BUG_ON(val & ~GEN8_RPCS_SS_CNT_MASK);
741 		val &= GEN8_RPCS_SS_CNT_MASK;
742 
743 		rpcs |= GEN8_RPCS_ENABLE | GEN8_RPCS_SS_CNT_ENABLE | val;
744 	}
745 
746 	if (sseu->has_eu_pg) {
747 		u32 val;
748 
749 		val = req_sseu->min_eus_per_subslice << GEN8_RPCS_EU_MIN_SHIFT;
750 		GEM_BUG_ON(val & ~GEN8_RPCS_EU_MIN_MASK);
751 		val &= GEN8_RPCS_EU_MIN_MASK;
752 
753 		rpcs |= val;
754 
755 		val = req_sseu->max_eus_per_subslice << GEN8_RPCS_EU_MAX_SHIFT;
756 		GEM_BUG_ON(val & ~GEN8_RPCS_EU_MAX_MASK);
757 		val &= GEN8_RPCS_EU_MAX_MASK;
758 
759 		rpcs |= val;
760 
761 		rpcs |= GEN8_RPCS_ENABLE;
762 	}
763 
764 	return rpcs;
765 }
766 
intel_sseu_dump(const struct sseu_dev_info * sseu,struct drm_printer * p)767 void intel_sseu_dump(const struct sseu_dev_info *sseu, struct drm_printer *p)
768 {
769 	int s;
770 
771 	if (sseu->has_xehp_dss) {
772 		drm_printf(p, "subslice total: %u\n",
773 			   intel_sseu_subslice_total(sseu));
774 		drm_printf(p, "geometry dss mask=%*pb\n",
775 			   XEHP_BITMAP_BITS(sseu->geometry_subslice_mask),
776 			   sseu->geometry_subslice_mask.xehp);
777 		drm_printf(p, "compute dss mask=%*pb\n",
778 			   XEHP_BITMAP_BITS(sseu->compute_subslice_mask),
779 			   sseu->compute_subslice_mask.xehp);
780 	} else {
781 		drm_printf(p, "slice total: %u, mask=%04x\n",
782 			   hweight8(sseu->slice_mask), sseu->slice_mask);
783 		drm_printf(p, "subslice total: %u\n",
784 			   intel_sseu_subslice_total(sseu));
785 
786 		for (s = 0; s < sseu->max_slices; s++) {
787 			u8 ss_mask = sseu->subslice_mask.hsw[s];
788 
789 			drm_printf(p, "slice%d: %u subslices, mask=%08x\n",
790 				   s, hweight8(ss_mask), ss_mask);
791 		}
792 	}
793 
794 	drm_printf(p, "EU total: %u\n", sseu->eu_total);
795 	drm_printf(p, "EU per subslice: %u\n", sseu->eu_per_subslice);
796 	drm_printf(p, "has slice power gating: %s\n",
797 		   str_yes_no(sseu->has_slice_pg));
798 	drm_printf(p, "has subslice power gating: %s\n",
799 		   str_yes_no(sseu->has_subslice_pg));
800 	drm_printf(p, "has EU power gating: %s\n",
801 		   str_yes_no(sseu->has_eu_pg));
802 }
803 
sseu_print_hsw_topology(const struct sseu_dev_info * sseu,struct drm_printer * p)804 static void sseu_print_hsw_topology(const struct sseu_dev_info *sseu,
805 				    struct drm_printer *p)
806 {
807 	int s, ss;
808 
809 	for (s = 0; s < sseu->max_slices; s++) {
810 		u8 ss_mask = sseu->subslice_mask.hsw[s];
811 
812 		drm_printf(p, "slice%d: %u subslice(s) (0x%08x):\n",
813 			   s, hweight8(ss_mask), ss_mask);
814 
815 		for (ss = 0; ss < sseu->max_subslices; ss++) {
816 			u16 enabled_eus = sseu_get_eus(sseu, s, ss);
817 
818 			drm_printf(p, "\tsubslice%d: %u EUs (0x%hx)\n",
819 				   ss, hweight16(enabled_eus), enabled_eus);
820 		}
821 	}
822 }
823 
sseu_print_xehp_topology(const struct sseu_dev_info * sseu,struct drm_printer * p)824 static void sseu_print_xehp_topology(const struct sseu_dev_info *sseu,
825 				     struct drm_printer *p)
826 {
827 	int dss;
828 
829 	for (dss = 0; dss < sseu->max_subslices; dss++) {
830 		u16 enabled_eus = sseu_get_eus(sseu, 0, dss);
831 
832 		drm_printf(p, "DSS_%02d: G:%3s C:%3s, %2u EUs (0x%04hx)\n", dss,
833 			   str_yes_no(test_bit(dss, sseu->geometry_subslice_mask.xehp)),
834 			   str_yes_no(test_bit(dss, sseu->compute_subslice_mask.xehp)),
835 			   hweight16(enabled_eus), enabled_eus);
836 	}
837 }
838 
intel_sseu_print_topology(struct drm_i915_private * i915,const struct sseu_dev_info * sseu,struct drm_printer * p)839 void intel_sseu_print_topology(struct drm_i915_private *i915,
840 			       const struct sseu_dev_info *sseu,
841 			       struct drm_printer *p)
842 {
843 	if (sseu->max_slices == 0)
844 		drm_printf(p, "Unavailable\n");
845 	else if (GRAPHICS_VER_FULL(i915) >= IP_VER(12, 55))
846 		sseu_print_xehp_topology(sseu, p);
847 	else
848 		sseu_print_hsw_topology(sseu, p);
849 }
850 
intel_sseu_print_ss_info(const char * type,const struct sseu_dev_info * sseu,struct seq_file * m)851 void intel_sseu_print_ss_info(const char *type,
852 			      const struct sseu_dev_info *sseu,
853 			      struct seq_file *m)
854 {
855 	int s;
856 
857 	if (sseu->has_xehp_dss) {
858 		seq_printf(m, "  %s Geometry DSS: %u\n", type,
859 			   bitmap_weight(sseu->geometry_subslice_mask.xehp,
860 					 XEHP_BITMAP_BITS(sseu->geometry_subslice_mask)));
861 		seq_printf(m, "  %s Compute DSS: %u\n", type,
862 			   bitmap_weight(sseu->compute_subslice_mask.xehp,
863 					 XEHP_BITMAP_BITS(sseu->compute_subslice_mask)));
864 	} else {
865 		for (s = 0; s < fls(sseu->slice_mask); s++)
866 			seq_printf(m, "  %s Slice%i subslices: %u\n", type,
867 				   s, hweight8(sseu->subslice_mask.hsw[s]));
868 	}
869 }
870 
intel_slicemask_from_xehp_dssmask(intel_sseu_ss_mask_t dss_mask,int dss_per_slice)871 u16 intel_slicemask_from_xehp_dssmask(intel_sseu_ss_mask_t dss_mask,
872 				      int dss_per_slice)
873 {
874 	intel_sseu_ss_mask_t per_slice_mask = {};
875 	unsigned long slice_mask = 0;
876 	int i;
877 
878 	WARN_ON(DIV_ROUND_UP(XEHP_BITMAP_BITS(dss_mask), dss_per_slice) >
879 		8 * sizeof(slice_mask));
880 
881 	bitmap_fill(per_slice_mask.xehp, dss_per_slice);
882 	for (i = 0; !bitmap_empty(dss_mask.xehp, XEHP_BITMAP_BITS(dss_mask)); i++) {
883 		if (bitmap_intersects(dss_mask.xehp, per_slice_mask.xehp, dss_per_slice))
884 			slice_mask |= BIT(i);
885 
886 		bitmap_shift_right(dss_mask.xehp, dss_mask.xehp, dss_per_slice,
887 				   XEHP_BITMAP_BITS(dss_mask));
888 	}
889 
890 	return slice_mask;
891 }
892