1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * uncore-frquency-tpmi: Uncore frequency scaling using TPMI
4  *
5  * Copyright (c) 2023, Intel Corporation.
6  * All Rights Reserved.
7  *
8  * The hardware interface to read/write is basically substitution of
9  * MSR 0x620 and 0x621.
10  * There are specific MMIO offset and bits to get/set minimum and
11  * maximum uncore ratio, similar to MSRs.
12  * The scope of the uncore MSRs was package scope. But TPMI allows
13  * new gen CPUs to have multiple uncore controls at uncore-cluster
14  * level. Each package can have multiple power domains which further
15  * can have multiple clusters.
16  * Here number of power domains = number of resources in this aux
17  * device. There are offsets and bits to discover number of clusters
18  * and offset for each cluster level controls.
19  *
20  */
21 
22 #include <linux/auxiliary_bus.h>
23 #include <linux/bitfield.h>
24 #include <linux/bits.h>
25 #include <linux/io.h>
26 #include <linux/module.h>
27 #include <linux/intel_tpmi.h>
28 
29 #include "uncore-frequency-common.h"
30 
31 #define	UNCORE_MAJOR_VERSION		0
32 #define	UNCORE_MINOR_VERSION		1
33 #define UNCORE_HEADER_INDEX		0
34 #define UNCORE_FABRIC_CLUSTER_OFFSET	8
35 
36 /* status + control + adv_ctl1 + adv_ctl2 */
37 #define UNCORE_FABRIC_CLUSTER_SIZE	(4 * 8)
38 
39 #define UNCORE_STATUS_INDEX		0
40 #define UNCORE_CONTROL_INDEX		8
41 
42 #define UNCORE_FREQ_KHZ_MULTIPLIER	100000
43 
44 struct tpmi_uncore_struct;
45 
46 /* Information for each cluster */
47 struct tpmi_uncore_cluster_info {
48 	bool root_domain;
49 	u8 __iomem *cluster_base;
50 	struct uncore_data uncore_data;
51 	struct tpmi_uncore_struct *uncore_root;
52 };
53 
54 /* Information for each power domain */
55 struct tpmi_uncore_power_domain_info {
56 	u8 __iomem *uncore_base;
57 	int ufs_header_ver;
58 	int cluster_count;
59 	struct tpmi_uncore_cluster_info *cluster_infos;
60 };
61 
62 /* Information for all power domains in a package */
63 struct tpmi_uncore_struct {
64 	int power_domain_count;
65 	int max_ratio;
66 	int min_ratio;
67 	struct tpmi_uncore_power_domain_info *pd_info;
68 	struct tpmi_uncore_cluster_info root_cluster;
69 	bool write_blocked;
70 };
71 
72 #define UNCORE_GENMASK_MIN_RATIO	GENMASK_ULL(21, 15)
73 #define UNCORE_GENMASK_MAX_RATIO	GENMASK_ULL(14, 8)
74 #define UNCORE_GENMASK_CURRENT_RATIO	GENMASK_ULL(6, 0)
75 
76 /* Helper function to read MMIO offset for max/min control frequency */
read_control_freq(struct tpmi_uncore_cluster_info * cluster_info,unsigned int * min,unsigned int * max)77 static void read_control_freq(struct tpmi_uncore_cluster_info *cluster_info,
78 			     unsigned int *min, unsigned int *max)
79 {
80 	u64 control;
81 
82 	control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
83 	*max = FIELD_GET(UNCORE_GENMASK_MAX_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
84 	*min = FIELD_GET(UNCORE_GENMASK_MIN_RATIO, control) * UNCORE_FREQ_KHZ_MULTIPLIER;
85 }
86 
87 #define UNCORE_MAX_RATIO	FIELD_MAX(UNCORE_GENMASK_MAX_RATIO)
88 
89 /* Callback for sysfs read for max/min frequencies. Called under mutex locks */
uncore_read_control_freq(struct uncore_data * data,unsigned int * min,unsigned int * max)90 static int uncore_read_control_freq(struct uncore_data *data, unsigned int *min,
91 				    unsigned int *max)
92 {
93 	struct tpmi_uncore_cluster_info *cluster_info;
94 
95 	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
96 
97 	if (cluster_info->root_domain) {
98 		struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
99 		int i, _min = 0, _max = 0;
100 
101 		*min = UNCORE_MAX_RATIO * UNCORE_FREQ_KHZ_MULTIPLIER;
102 		*max = 0;
103 
104 		/*
105 		 * Get the max/min by looking at each cluster. Get the lowest
106 		 * min and highest max.
107 		 */
108 		for (i = 0; i < uncore_root->power_domain_count; ++i) {
109 			int j;
110 
111 			for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j) {
112 				read_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
113 						  &_min, &_max);
114 				if (*min > _min)
115 					*min = _min;
116 				if (*max < _max)
117 					*max = _max;
118 			}
119 		}
120 		return 0;
121 	}
122 
123 	read_control_freq(cluster_info, min, max);
124 
125 	return 0;
126 }
127 
128 /* Helper function to write MMIO offset for max/min control frequency */
write_control_freq(struct tpmi_uncore_cluster_info * cluster_info,unsigned int input,unsigned int min_max)129 static void write_control_freq(struct tpmi_uncore_cluster_info *cluster_info, unsigned int input,
130 			      unsigned int min_max)
131 {
132 	u64 control;
133 
134 	control = readq(cluster_info->cluster_base + UNCORE_CONTROL_INDEX);
135 
136 	if (min_max) {
137 		control &= ~UNCORE_GENMASK_MAX_RATIO;
138 		control |= FIELD_PREP(UNCORE_GENMASK_MAX_RATIO, input);
139 	} else {
140 		control &= ~UNCORE_GENMASK_MIN_RATIO;
141 		control |= FIELD_PREP(UNCORE_GENMASK_MIN_RATIO, input);
142 	}
143 
144 	writeq(control, (cluster_info->cluster_base + UNCORE_CONTROL_INDEX));
145 }
146 
147 /* Callback for sysfs write for max/min frequencies. Called under mutex locks */
uncore_write_control_freq(struct uncore_data * data,unsigned int input,unsigned int min_max)148 static int uncore_write_control_freq(struct uncore_data *data, unsigned int input,
149 				     unsigned int min_max)
150 {
151 	struct tpmi_uncore_cluster_info *cluster_info;
152 	struct tpmi_uncore_struct *uncore_root;
153 
154 	input /= UNCORE_FREQ_KHZ_MULTIPLIER;
155 	if (!input || input > UNCORE_MAX_RATIO)
156 		return -EINVAL;
157 
158 	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
159 	uncore_root = cluster_info->uncore_root;
160 
161 	if (uncore_root->write_blocked)
162 		return -EPERM;
163 
164 	/* Update each cluster in a package */
165 	if (cluster_info->root_domain) {
166 		struct tpmi_uncore_struct *uncore_root = cluster_info->uncore_root;
167 		int i;
168 
169 		for (i = 0; i < uncore_root->power_domain_count; ++i) {
170 			int j;
171 
172 			for (j = 0; j < uncore_root->pd_info[i].cluster_count; ++j)
173 				write_control_freq(&uncore_root->pd_info[i].cluster_infos[j],
174 						  input, min_max);
175 		}
176 
177 		if (min_max)
178 			uncore_root->max_ratio = input;
179 		else
180 			uncore_root->min_ratio = input;
181 
182 		return 0;
183 	}
184 
185 	if (min_max && uncore_root->max_ratio && uncore_root->max_ratio < input)
186 		return -EINVAL;
187 
188 	if (!min_max && uncore_root->min_ratio && uncore_root->min_ratio > input)
189 		return -EINVAL;
190 
191 	write_control_freq(cluster_info, input, min_max);
192 
193 	return 0;
194 }
195 
196 /* Callback for sysfs read for the current uncore frequency. Called under mutex locks */
uncore_read_freq(struct uncore_data * data,unsigned int * freq)197 static int uncore_read_freq(struct uncore_data *data, unsigned int *freq)
198 {
199 	struct tpmi_uncore_cluster_info *cluster_info;
200 	u64 status;
201 
202 	cluster_info = container_of(data, struct tpmi_uncore_cluster_info, uncore_data);
203 	if (cluster_info->root_domain)
204 		return -ENODATA;
205 
206 	status = readq((u8 __iomem *)cluster_info->cluster_base + UNCORE_STATUS_INDEX);
207 	*freq = FIELD_GET(UNCORE_GENMASK_CURRENT_RATIO, status) * UNCORE_FREQ_KHZ_MULTIPLIER;
208 
209 	return 0;
210 }
211 
remove_cluster_entries(struct tpmi_uncore_struct * tpmi_uncore)212 static void remove_cluster_entries(struct tpmi_uncore_struct *tpmi_uncore)
213 {
214 	int i;
215 
216 	for (i = 0; i < tpmi_uncore->power_domain_count; ++i) {
217 		struct tpmi_uncore_power_domain_info *pd_info;
218 		int j;
219 
220 		pd_info = &tpmi_uncore->pd_info[i];
221 		if (!pd_info->uncore_base)
222 			continue;
223 
224 		for (j = 0; j < pd_info->cluster_count; ++j) {
225 			struct tpmi_uncore_cluster_info *cluster_info;
226 
227 			cluster_info = &pd_info->cluster_infos[j];
228 			uncore_freq_remove_die_entry(&cluster_info->uncore_data);
229 		}
230 	}
231 }
232 
233 #define UNCORE_VERSION_MASK			GENMASK_ULL(7, 0)
234 #define UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK	GENMASK_ULL(15, 8)
235 #define UNCORE_CLUSTER_OFF_MASK			GENMASK_ULL(7, 0)
236 #define UNCORE_MAX_CLUSTER_PER_DOMAIN		8
237 
uncore_probe(struct auxiliary_device * auxdev,const struct auxiliary_device_id * id)238 static int uncore_probe(struct auxiliary_device *auxdev, const struct auxiliary_device_id *id)
239 {
240 	bool read_blocked = 0, write_blocked = 0;
241 	struct intel_tpmi_plat_info *plat_info;
242 	struct tpmi_uncore_struct *tpmi_uncore;
243 	int ret, i, pkg = 0;
244 	int num_resources;
245 
246 	ret = tpmi_get_feature_status(auxdev, TPMI_ID_UNCORE, &read_blocked, &write_blocked);
247 	if (ret)
248 		dev_info(&auxdev->dev, "Can't read feature status: ignoring blocked status\n");
249 
250 	if (read_blocked) {
251 		dev_info(&auxdev->dev, "Firmware has blocked reads, exiting\n");
252 		return -ENODEV;
253 	}
254 
255 	/* Get number of power domains, which is equal to number of resources */
256 	num_resources = tpmi_get_resource_count(auxdev);
257 	if (!num_resources)
258 		return -EINVAL;
259 
260 	/* Register callbacks to uncore core */
261 	ret = uncore_freq_common_init(uncore_read_control_freq, uncore_write_control_freq,
262 				      uncore_read_freq);
263 	if (ret)
264 		return ret;
265 
266 	/* Allocate uncore instance per package */
267 	tpmi_uncore = devm_kzalloc(&auxdev->dev, sizeof(*tpmi_uncore), GFP_KERNEL);
268 	if (!tpmi_uncore) {
269 		ret = -ENOMEM;
270 		goto err_rem_common;
271 	}
272 
273 	/* Allocate memory for all power domains in a package */
274 	tpmi_uncore->pd_info = devm_kcalloc(&auxdev->dev, num_resources,
275 					    sizeof(*tpmi_uncore->pd_info),
276 					    GFP_KERNEL);
277 	if (!tpmi_uncore->pd_info) {
278 		ret = -ENOMEM;
279 		goto err_rem_common;
280 	}
281 
282 	tpmi_uncore->power_domain_count = num_resources;
283 	tpmi_uncore->write_blocked = write_blocked;
284 
285 	/* Get the package ID from the TPMI core */
286 	plat_info = tpmi_get_platform_data(auxdev);
287 	if (plat_info)
288 		pkg = plat_info->package_id;
289 	else
290 		dev_info(&auxdev->dev, "Platform information is NULL\n");
291 
292 	for (i = 0; i < num_resources; ++i) {
293 		struct tpmi_uncore_power_domain_info *pd_info;
294 		struct resource *res;
295 		u64 cluster_offset;
296 		u8 cluster_mask;
297 		int mask, j;
298 		u64 header;
299 
300 		res = tpmi_get_resource_at_index(auxdev, i);
301 		if (!res)
302 			continue;
303 
304 		pd_info = &tpmi_uncore->pd_info[i];
305 
306 		pd_info->uncore_base = devm_ioremap_resource(&auxdev->dev, res);
307 		if (IS_ERR(pd_info->uncore_base)) {
308 			ret = PTR_ERR(pd_info->uncore_base);
309 			/*
310 			 * Set to NULL so that clean up can still remove other
311 			 * entries already created if any by
312 			 * remove_cluster_entries()
313 			 */
314 			pd_info->uncore_base = NULL;
315 			goto remove_clusters;
316 		}
317 
318 		/* Check for version and skip this resource if there is mismatch */
319 		header = readq(pd_info->uncore_base);
320 		pd_info->ufs_header_ver = header & UNCORE_VERSION_MASK;
321 
322 		if (pd_info->ufs_header_ver == TPMI_VERSION_INVALID)
323 			continue;
324 
325 		if (TPMI_MAJOR_VERSION(pd_info->ufs_header_ver) != UNCORE_MAJOR_VERSION) {
326 			dev_err(&auxdev->dev, "Uncore: Unsupported major version:%lx\n",
327 				TPMI_MAJOR_VERSION(pd_info->ufs_header_ver));
328 			ret = -ENODEV;
329 			goto remove_clusters;
330 		}
331 
332 		if (TPMI_MINOR_VERSION(pd_info->ufs_header_ver) != UNCORE_MINOR_VERSION)
333 			dev_info(&auxdev->dev, "Uncore: Ignore: Unsupported minor version:%lx\n",
334 				 TPMI_MINOR_VERSION(pd_info->ufs_header_ver));
335 
336 		/* Get Cluster ID Mask */
337 		cluster_mask = FIELD_GET(UNCORE_LOCAL_FABRIC_CLUSTER_ID_MASK, header);
338 		if (!cluster_mask) {
339 			dev_info(&auxdev->dev, "Uncore: Invalid cluster mask:%x\n", cluster_mask);
340 			continue;
341 		}
342 
343 		/* Find out number of clusters in this resource */
344 		pd_info->cluster_count = hweight8(cluster_mask);
345 
346 		pd_info->cluster_infos = devm_kcalloc(&auxdev->dev, pd_info->cluster_count,
347 						      sizeof(struct tpmi_uncore_cluster_info),
348 						      GFP_KERNEL);
349 		if (!pd_info->cluster_infos) {
350 			ret = -ENOMEM;
351 			goto remove_clusters;
352 		}
353 		/*
354 		 * Each byte in the register point to status and control
355 		 * registers belonging to cluster id 0-8.
356 		 */
357 		cluster_offset = readq(pd_info->uncore_base +
358 					UNCORE_FABRIC_CLUSTER_OFFSET);
359 
360 		for (j = 0; j < pd_info->cluster_count; ++j) {
361 			struct tpmi_uncore_cluster_info *cluster_info;
362 
363 			/* Get the offset for this cluster */
364 			mask = (cluster_offset & UNCORE_CLUSTER_OFF_MASK);
365 			/* Offset in QWORD, so change to bytes */
366 			mask <<= 3;
367 
368 			cluster_info = &pd_info->cluster_infos[j];
369 
370 			cluster_info->cluster_base = pd_info->uncore_base + mask;
371 
372 			cluster_info->uncore_data.package_id = pkg;
373 			/* There are no dies like Cascade Lake */
374 			cluster_info->uncore_data.die_id = 0;
375 			cluster_info->uncore_data.domain_id = i;
376 			cluster_info->uncore_data.cluster_id = j;
377 
378 			cluster_info->uncore_root = tpmi_uncore;
379 
380 			ret = uncore_freq_add_entry(&cluster_info->uncore_data, 0);
381 			if (ret) {
382 				cluster_info->cluster_base = NULL;
383 				goto remove_clusters;
384 			}
385 			/* Point to next cluster offset */
386 			cluster_offset >>= UNCORE_MAX_CLUSTER_PER_DOMAIN;
387 		}
388 	}
389 
390 	auxiliary_set_drvdata(auxdev, tpmi_uncore);
391 
392 	tpmi_uncore->root_cluster.root_domain = true;
393 	tpmi_uncore->root_cluster.uncore_root = tpmi_uncore;
394 
395 	tpmi_uncore->root_cluster.uncore_data.package_id = pkg;
396 	tpmi_uncore->root_cluster.uncore_data.domain_id = UNCORE_DOMAIN_ID_INVALID;
397 	ret = uncore_freq_add_entry(&tpmi_uncore->root_cluster.uncore_data, 0);
398 	if (ret)
399 		goto remove_clusters;
400 
401 	return 0;
402 
403 remove_clusters:
404 	remove_cluster_entries(tpmi_uncore);
405 err_rem_common:
406 	uncore_freq_common_exit();
407 
408 	return ret;
409 }
410 
uncore_remove(struct auxiliary_device * auxdev)411 static void uncore_remove(struct auxiliary_device *auxdev)
412 {
413 	struct tpmi_uncore_struct *tpmi_uncore = auxiliary_get_drvdata(auxdev);
414 
415 	uncore_freq_remove_die_entry(&tpmi_uncore->root_cluster.uncore_data);
416 	remove_cluster_entries(tpmi_uncore);
417 
418 	uncore_freq_common_exit();
419 }
420 
421 static const struct auxiliary_device_id intel_uncore_id_table[] = {
422 	{ .name = "intel_vsec.tpmi-uncore" },
423 	{}
424 };
425 MODULE_DEVICE_TABLE(auxiliary, intel_uncore_id_table);
426 
427 static struct auxiliary_driver intel_uncore_aux_driver = {
428 	.id_table       = intel_uncore_id_table,
429 	.remove         = uncore_remove,
430 	.probe          = uncore_probe,
431 };
432 
433 module_auxiliary_driver(intel_uncore_aux_driver);
434 
435 MODULE_IMPORT_NS(INTEL_TPMI);
436 MODULE_IMPORT_NS(INTEL_UNCORE_FREQUENCY);
437 MODULE_DESCRIPTION("Intel TPMI UFS Driver");
438 MODULE_LICENSE("GPL");
439