1 // SPDX-License-Identifier: GPL-2.0
3  * Copyright (c) 2009-2010 Intel Corporation
10  * Some Intel Ibex Peak based platforms support so-called "intelligent
34  *   - dual MCP configs
37  *   - handle CPU hotplug
38  *   - provide turbo enable/disable api
41  *   - CDI 403777, 403778 - Auburndale EDS vol 1 & 2
42  *   - CDI 401376 - Ibex Peak EDS
43  *   - ref 26037, 26641 - IPS BIOS spec
44  *   - ref 26489 - Nehalem BIOS writer's guide
45  *   - ref 26921 - Ibex Peak BIOS Specification
67 #include <linux/io-64-nonatomic-lo-hi.h>
228 #define thm_readb(off) readb(ips->regmap + (off))
229 #define thm_readw(off) readw(ips->regmap + (off))
230 #define thm_readl(off) readl(ips->regmap + (off))
231 #define thm_readq(off) readq(ips->regmap + (off))
233 #define thm_writeb(off, val) writeb((val), ips->regmap + (off))
234 #define thm_writew(off, val) writew((val), ips->regmap + (off))
235 #define thm_writel(off, val) writel((val), ips->regmap + (off))
242 static const int IPS_SAMPLE_WINDOW = 5000; /* 5s moving window of samples */
245 /* Per-SKU limits */
254 /* Max temps are -10 degrees C to avoid PROCHOT# */
338  * ips_cpu_busy - is CPU busy?
355  * ips_cpu_raise - raise CPU power clamp
369 	if (!ips->cpu_turbo_enabled)  in ips_cpu_raise()
378 	if (((new_tdp_limit * 10) / 8) > ips->core_power_limit)  in ips_cpu_raise()
393  * ips_cpu_lower - lower CPU power clamp
410 	new_limit = cur_limit - 8; /* 1W decrease */  in ips_cpu_lower()
413 	if (new_limit  < (ips->orig_turbo_limit & TURBO_TDP_MASK))  in ips_cpu_lower()
414 		new_limit = ips->orig_turbo_limit & TURBO_TDP_MASK;  in ips_cpu_lower()
428  * do_enable_cpu_turbo - internal turbo enable function
447  * ips_enable_cpu_turbo - enable turbo mode on all CPUs
456 	if (ips->__cpu_turbo_on)  in ips_enable_cpu_turbo()
459 	if (ips->turbo_toggle_allowed)  in ips_enable_cpu_turbo()
462 	ips->__cpu_turbo_on = true;  in ips_enable_cpu_turbo()
466  * do_disable_cpu_turbo - internal turbo disable function
485  * ips_disable_cpu_turbo - disable turbo mode on all CPUs
494 	if (!ips->__cpu_turbo_on)  in ips_disable_cpu_turbo()
497 	if (ips->turbo_toggle_allowed)  in ips_disable_cpu_turbo()
500 	ips->__cpu_turbo_on = false;  in ips_disable_cpu_turbo()
504  * ips_gpu_busy - is GPU busy?
518 	return ips->gpu_busy();  in ips_gpu_busy()
522  * ips_gpu_raise - raise GPU power clamp
533 	if (!ips->gpu_raise())  in ips_gpu_raise()
534 		ips->gpu_turbo_enabled = false;  in ips_gpu_raise()
540  * ips_gpu_lower - lower GPU power clamp
550 	if (!ips->gpu_lower())  in ips_gpu_lower()
551 		ips->gpu_turbo_enabled = false;  in ips_gpu_lower()
557  * ips_enable_gpu_turbo - notify the gfx driver turbo is available
565 	if (ips->__gpu_turbo_on)  in ips_enable_gpu_turbo()
567 	ips->__gpu_turbo_on = true;  in ips_enable_gpu_turbo()
571  * ips_disable_gpu_turbo - notify the gfx driver to disable turbo mode
579 	if (!ips->__gpu_turbo_on)  in ips_disable_gpu_turbo()
582 	if (!ips->gpu_turbo_disable())  in ips_disable_gpu_turbo()
583 		dev_err(ips->dev, "failed to disable graphics turbo\n");  in ips_disable_gpu_turbo()
585 		ips->__gpu_turbo_on = false;  in ips_disable_gpu_turbo()
589  * mcp_exceeded - check whether we're outside our thermal & power limits
603 	spin_lock_irqsave(&ips->turbo_status_lock, flags);  in mcp_exceeded()
605 	temp_limit = ips->mcp_temp_limit * 100;  in mcp_exceeded()
606 	if (ips->mcp_avg_temp > temp_limit)  in mcp_exceeded()
609 	avg_power = ips->cpu_avg_power + ips->mch_avg_power;  in mcp_exceeded()
610 	if (avg_power > ips->mcp_power_limit)  in mcp_exceeded()
613 	spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in mcp_exceeded()
619  * cpu_exceeded - check whether a CPU core is outside its limits
630 	int avg;  in cpu_exceeded()  local
633 	spin_lock_irqsave(&ips->turbo_status_lock, flags);  in cpu_exceeded()
634 	avg = cpu ? ips->ctv2_avg_temp : ips->ctv1_avg_temp;  in cpu_exceeded()
635 	if (avg > (ips->limits->core_temp_limit * 100))  in cpu_exceeded()
637 	if (ips->cpu_avg_power > ips->core_power_limit * 100)  in cpu_exceeded()
639 	spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in cpu_exceeded()
642 		dev_info(ips->dev, "CPU power or thermal limit exceeded\n");  in cpu_exceeded()
648  * mch_exceeded - check whether the GPU is over budget
660 	spin_lock_irqsave(&ips->turbo_status_lock, flags);  in mch_exceeded()
661 	if (ips->mch_avg_temp > (ips->limits->mch_temp_limit * 100))  in mch_exceeded()
663 	if (ips->mch_avg_power > ips->mch_power_limit)  in mch_exceeded()
665 	spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in mch_exceeded()
671  * verify_limits - verify BIOS provided limits
674  * BIOS can optionally provide non-default limits for power and temp.  Check
680 	if (ips->mcp_power_limit < ips->limits->mcp_power_limit ||  in verify_limits()
681 	    ips->mcp_power_limit > 35000)  in verify_limits()
682 		ips->mcp_power_limit = ips->limits->mcp_power_limit;  in verify_limits()
684 	if (ips->mcp_temp_limit < ips->limits->core_temp_limit ||  in verify_limits()
685 	    ips->mcp_temp_limit < ips->limits->mch_temp_limit ||  in verify_limits()
686 	    ips->mcp_temp_limit > 150)  in verify_limits()
687 		ips->mcp_temp_limit = min(ips->limits->core_temp_limit,  in verify_limits()
688 					  ips->limits->mch_temp_limit);  in verify_limits()
692  * update_turbo_limits - get various limits & settings from regs
699  * the regs for updates (as a result of AC->DC transition for example).
708 	ips->cpu_turbo_enabled = !(hts & HTS_PCTD_DIS);  in update_turbo_limits()
713 	ips->cpu_turbo_enabled = false;  in update_turbo_limits()
715 	if (ips->gpu_busy)  in update_turbo_limits()
716 		ips->gpu_turbo_enabled = !(hts & HTS_GTD_DIS);  in update_turbo_limits()
718 	ips->core_power_limit = thm_readw(THM_MPCPC);  in update_turbo_limits()
719 	ips->mch_power_limit = thm_readw(THM_MMGPC);  in update_turbo_limits()
720 	ips->mcp_temp_limit = thm_readw(THM_PTL);  in update_turbo_limits()
721 	ips->mcp_power_limit = thm_readw(THM_MPPC);  in update_turbo_limits()
728  * ips_adjust - adjust power clamp based on thermal state
734  *   - do we need to adjust up or down?
735  *   - is CPU busy?
736  *   - is GPU busy?
737  *   - is CPU in turbo?
738  *   - is GPU in turbo?
739  *   - is CPU or GPU preferred? (CPU is default)
742  *   - up (TDP available)
743  *     - CPU not busy, GPU not busy - nothing
744  *     - CPU busy, GPU not busy - adjust CPU up
745  *     - CPU not busy, GPU busy - adjust GPU up
746  *     - CPU busy, GPU busy - adjust preferred unit up, taking headroom from
747  *       non-preferred unit if necessary
748  *   - down (at TDP limit)
749  *     - adjust both CPU and GPU down if possible
751  *              |cpu+ gpu+      cpu+gpu-        cpu-gpu+        cpu-gpu-
753  * cpu < gpu >= |cpu+gpu-(mcp<) cpu+gpu-(mcp<)  gpu-            gpu-
754  * cpu >= gpu < |cpu-gpu+(mcp<) cpu-            cpu-gpu+(mcp<)  cpu-
755  * cpu >= gpu >=|cpu-gpu-       cpu-gpu-        cpu-gpu-        cpu-gpu-
764 	dev_dbg(ips->dev, "starting ips-adjust thread\n");  in ips_adjust()
774 		spin_lock_irqsave(&ips->turbo_status_lock, flags);  in ips_adjust()
775 		if (ips->poll_turbo_status)  in ips_adjust()
777 		spin_unlock_irqrestore(&ips->turbo_status_lock, flags);  in ips_adjust()
780 		if (ips->cpu_turbo_enabled)  in ips_adjust()
785 		if (ips->gpu_turbo_enabled)  in ips_adjust()
811 	dev_dbg(ips->dev, "ips-adjust thread stopped\n");  in ips_adjust()
825 	u16 avg;  in calc_avg_temp()  local
832 	avg = (u16)total;  in calc_avg_temp()
834 	return avg;  in calc_avg_temp()
890 	ret = (((val - *last) * 1000) / period);  in get_cpu_power()
898 static u16 update_average_temp(u16 avg, u16 val)  in update_average_temp()  argument
904 		(((temp_decay_factor - 1) * avg) / temp_decay_factor);  in update_average_temp()
909 static u16 update_average_power(u32 avg, u32 val)  in update_average_power()  argument
914 		(((power_decay_factor - 1) * avg) / power_decay_factor);  in update_average_power()
922 	u32 avg;  in calc_avg_power()  local
929 	avg = (u32)total;  in calc_avg_power()
931 	return avg;  in calc_avg_power()
937 	wake_up_process(ips->monitor);  in monitor_timeout()
941  * ips_monitor - temp/power monitoring thread
951  * Returns: %0 on success or -errno on error
970 		dev_err(ips->dev,  in ips_monitor()
978 		return -ENOMEM;  in ips_monitor()
1008 		if (ips->read_mch_val) {  in ips_monitor()
1009 			mchp = ips->read_mch_val();  in ips_monitor()
1018 	ips->mcp_avg_temp = calc_avg_temp(ips, mcp_samples);  in ips_monitor()
1019 	ips->ctv1_avg_temp = calc_avg_temp(ips, ctv1_samples);  in ips_monitor()
1020 	ips->ctv2_avg_temp = calc_avg_temp(ips, ctv2_samples);  in ips_monitor()
1021 	ips->mch_avg_temp = calc_avg_temp(ips, mch_samples);  in ips_monitor()
1022 	ips->cpu_avg_power = calc_avg_power(ips, cpu_samples);  in ips_monitor()
1023 	ips->mch_avg_power = calc_avg_power(ips, mchp_samples);  in ips_monitor()
1032 	wake_up_process(ips->adjust);  in ips_monitor()
1035 	 * Ok, now we have an initial avg.  From here on out, we track the  in ips_monitor()
1036 	 * running avg using a decaying average calculation.  This allows  in ips_monitor()
1043 	timer_setup(&ips->timer, monitor_timeout, TIMER_DEFERRABLE);  in ips_monitor()
1050 		ips->mcp_avg_temp = update_average_temp(ips->mcp_avg_temp, val);  in ips_monitor()
1054 		ips->ctv1_avg_temp =  in ips_monitor()
1055 			update_average_temp(ips->ctv1_avg_temp, val);  in ips_monitor()
1059 		ips->cpu_avg_power =  in ips_monitor()
1060 			update_average_power(ips->cpu_avg_power, cpu_val);  in ips_monitor()
1062 		if (ips->second_cpu) {  in ips_monitor()
1065 			ips->ctv2_avg_temp =  in ips_monitor()
1066 				update_average_temp(ips->ctv2_avg_temp, val);  in ips_monitor()
1071 		ips->mch_avg_temp = update_average_temp(ips->mch_avg_temp, val);  in ips_monitor()
1073 		if (ips->read_mch_val) {  in ips_monitor()
1074 			mch_val = ips->read_mch_val();  in ips_monitor()
1075 			ips->mch_avg_power =  in ips_monitor()
1076 				update_average_power(ips->mch_avg_power,  in ips_monitor()
1090 			dev_warn(ips->dev,  in ips_monitor()
1101 		mod_timer(&ips->timer, expire);  in ips_monitor()
1105 		last_sample_period = jiffies_to_msecs(jiffies) - last_msecs;  in ips_monitor()
1110 	del_timer_sync(&ips->timer);  in ips_monitor()
1112 	dev_dbg(ips->dev, "ips-monitor thread stopped\n");  in ips_monitor()
1118  * ips_irq_handler - handle temperature triggers and other IPS events
1137 	dev_info(ips->dev, "TSES: 0x%02x\n", tses);  in ips_irq_handler()
1138 	dev_info(ips->dev, "TES: 0x%02x\n", tes);  in ips_irq_handler()
1148 			spin_lock(&ips->turbo_status_lock);  in ips_irq_handler()
1149 			ips->core_power_limit = (sts & STS_PCPL_MASK) >>  in ips_irq_handler()
1151 			ips->mch_power_limit = (sts & STS_GPL_MASK) >>  in ips_irq_handler()
1154 			ips->cpu_turbo_enabled = !(sts & STS_PCTD_DIS);  in ips_irq_handler()
1159 			ips->cpu_turbo_enabled = false;  in ips_irq_handler()
1160 			if (ips->gpu_busy)  in ips_irq_handler()
1161 				ips->gpu_turbo_enabled = !(sts & STS_GTD_DIS);  in ips_irq_handler()
1162 			ips->mcp_temp_limit = (sts & STS_PTL_MASK) >>  in ips_irq_handler()
1164 			ips->mcp_power_limit = (tc1 & STS_PPL_MASK) >>  in ips_irq_handler()
1167 			spin_unlock(&ips->turbo_status_lock);  in ips_irq_handler()
1176 		dev_warn(ips->dev, "thermal trip occurred, tses: 0x%04x\n",  in ips_irq_handler()
1193 	struct ips_driver *ips = m->private;  in cpu_temp_show()
1195 	seq_printf(m, "%d.%02d\n", ips->ctv1_avg_temp / 100,  in cpu_temp_show()
1196 		   ips->ctv1_avg_temp % 100);  in cpu_temp_show()
1204 	struct ips_driver *ips = m->private;  in cpu_power_show()
1206 	seq_printf(m, "%dmW\n", ips->cpu_avg_power);  in cpu_power_show()
1236 	struct ips_driver *ips = m->private;  in mch_temp_show()
1238 	seq_printf(m, "%d.%02d\n", ips->mch_avg_temp / 100,  in mch_temp_show()
1239 		   ips->mch_avg_temp % 100);  in mch_temp_show()
1247 	struct ips_driver *ips = m->private;  in mch_power_show()
1249 	seq_printf(m, "%dmW\n", ips->mch_avg_power);  in mch_power_show()
1257 	debugfs_remove_recursive(ips->debug_root);  in ips_debugfs_cleanup()
1262 	ips->debug_root = debugfs_create_dir("ips", NULL);  in ips_debugfs_init()
1264 	debugfs_create_file("cpu_temp", 0444, ips->debug_root, ips, &cpu_temp_fops);  in ips_debugfs_init()
1265 	debugfs_create_file("cpu_power", 0444, ips->debug_root, ips, &cpu_power_fops);  in ips_debugfs_init()
1266 	debugfs_create_file("cpu_clamp", 0444, ips->debug_root, ips, &cpu_clamp_fops);  in ips_debugfs_init()
1267 	debugfs_create_file("mch_temp", 0444, ips->debug_root, ips, &mch_temp_fops);  in ips_debugfs_init()
1268 	debugfs_create_file("mch_power", 0444, ips->debug_root, ips, &mch_power_fops);  in ips_debugfs_init()
1273  * ips_detect_cpu - detect whether CPU supports IPS
1288 		dev_info(ips->dev, "Non-IPS CPU detected.\n");  in ips_detect_cpu()
1299 		ips->turbo_toggle_allowed = true;  in ips_detect_cpu()
1301 		ips->turbo_toggle_allowed = false;  in ips_detect_cpu()
1310 		dev_info(ips->dev, "No CPUID match found.\n");  in ips_detect_cpu()
1318 	if (limits->core_power_limit != (tdp / 8) * 1000) {  in ips_detect_cpu()
1319 		dev_info(ips->dev,  in ips_detect_cpu()
1321 			 tdp / 8, limits->core_power_limit / 1000);  in ips_detect_cpu()
1322 		limits->core_power_limit = (tdp / 8) * 1000;  in ips_detect_cpu()
1329  * ips_get_i915_syms - try to get GPU control methods from i915 driver
1341 	ips->read_mch_val = symbol_get(i915_read_mch_val);  in ips_get_i915_syms()
1342 	if (!ips->read_mch_val)  in ips_get_i915_syms()
1344 	ips->gpu_raise = symbol_get(i915_gpu_raise);  in ips_get_i915_syms()
1345 	if (!ips->gpu_raise)  in ips_get_i915_syms()
1347 	ips->gpu_lower = symbol_get(i915_gpu_lower);  in ips_get_i915_syms()
1348 	if (!ips->gpu_lower)  in ips_get_i915_syms()
1350 	ips->gpu_busy = symbol_get(i915_gpu_busy);  in ips_get_i915_syms()
1351 	if (!ips->gpu_busy)  in ips_get_i915_syms()
1353 	ips->gpu_turbo_disable = symbol_get(i915_gpu_turbo_disable);  in ips_get_i915_syms()
1354 	if (!ips->gpu_turbo_disable)  in ips_get_i915_syms()
1374 	if (!ips->gpu_busy && late_i915_load) {  in ips_gpu_turbo_enabled()
1376 			dev_info(ips->dev,  in ips_gpu_turbo_enabled()
1378 			ips->gpu_turbo_enabled = !(thm_readl(THM_HTS) & HTS_GTD_DIS);  in ips_gpu_turbo_enabled()
1382 	return ips->gpu_turbo_enabled;  in ips_gpu_turbo_enabled()
1405 	pr_info("Blacklisted intel_ips for %s\n", id->ident);  in ips_blacklist_callback()
1414 			DMI_MATCH(DMI_SYS_VENDOR, "Hewlett-Packard"),
1431 		return -ENODEV;  in ips_probe()
1433 	ips = devm_kzalloc(&dev->dev, sizeof(*ips), GFP_KERNEL);  in ips_probe()
1435 		return -ENOMEM;  in ips_probe()
1437 	spin_lock_init(&ips->turbo_status_lock);  in ips_probe()
1438 	ips->dev = &dev->dev;  in ips_probe()
1440 	ips->limits = ips_detect_cpu(ips);  in ips_probe()
1441 	if (!ips->limits) {  in ips_probe()
1442 		dev_info(&dev->dev, "IPS not supported on this CPU\n");  in ips_probe()
1443 		return -ENXIO;  in ips_probe()
1448 		dev_err(&dev->dev, "can't enable PCI device, aborting\n");  in ips_probe()
1454 		dev_err(&dev->dev, "failed to map thermal regs, aborting\n");  in ips_probe()
1457 	ips->regmap = pcim_iomap_table(dev)[0];  in ips_probe()
1463 		dev_err(&dev->dev, "thermal device not enabled (0x%02x), aborting\n", tse);  in ips_probe()
1464 		return -ENXIO;  in ips_probe()
1470 		dev_err(&dev->dev, "thermal reporting for required devices not enabled, aborting\n");  in ips_probe()
1471 		return -ENXIO;  in ips_probe()
1475 		ips->second_cpu = true;  in ips_probe()
1478 	dev_dbg(&dev->dev, "max cpu power clamp: %dW\n",  in ips_probe()
1479 		ips->mcp_power_limit / 10);  in ips_probe()
1480 	dev_dbg(&dev->dev, "max core power clamp: %dW\n",  in ips_probe()
1481 		ips->core_power_limit / 10);  in ips_probe()
1484 		ips->poll_turbo_status = true;  in ips_probe()
1487 		dev_info(&dev->dev, "failed to get i915 symbols, graphics turbo disabled until i915 loads\n");  in ips_probe()
1488 		ips->gpu_turbo_enabled = false;  in ips_probe()
1490 		dev_dbg(&dev->dev, "graphics turbo enabled\n");  in ips_probe()
1491 		ips->gpu_turbo_enabled = true;  in ips_probe()
1500 		dev_err(&dev->dev, "platform indicates TDP override unavailable, aborting\n");  in ips_probe()
1501 		return -ENODEV;  in ips_probe()
1512 	ips->irq = pci_irq_vector(dev, 0);  in ips_probe()
1514 	ret = request_irq(ips->irq, ips_irq_handler, IRQF_SHARED, "ips", ips);  in ips_probe()
1516 		dev_err(&dev->dev, "request irq failed, aborting\n");  in ips_probe()
1526 	ips->cta_val = thm_readw(THM_CTA);  in ips_probe()
1527 	ips->pta_val = thm_readw(THM_PTA);  in ips_probe()
1528 	ips->mgta_val = thm_readw(THM_MGTA);  in ips_probe()
1531 	rdmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit);  in ips_probe()
1534 	ips->cpu_turbo_enabled = false;  in ips_probe()
1537 	ips->adjust = kthread_create(ips_adjust, ips, "ips-adjust");  in ips_probe()
1538 	if (IS_ERR(ips->adjust)) {  in ips_probe()
1539 		dev_err(&dev->dev,  in ips_probe()
1541 		ret = -ENOMEM;  in ips_probe()
1550 	ips->monitor = kthread_run(ips_monitor, ips, "ips-monitor");  in ips_probe()
1551 	if (IS_ERR(ips->monitor)) {  in ips_probe()
1552 		dev_err(&dev->dev,  in ips_probe()
1554 		ret = -ENOMEM;  in ips_probe()
1558 	hts = (ips->core_power_limit << HTS_PCPL_SHIFT) |  in ips_probe()
1559 		(ips->mcp_temp_limit << HTS_PTL_SHIFT) | HTS_NVV;  in ips_probe()
1567 	dev_info(&dev->dev, "IPS driver initialized, MCP temp limit %d\n",  in ips_probe()
1568 		 ips->mcp_temp_limit);  in ips_probe()
1572 	kthread_stop(ips->adjust);  in ips_probe()
1574 	free_irq(ips->irq, ips);  in ips_probe()
1587 	if (ips->read_mch_val)  in ips_remove()
1589 	if (ips->gpu_raise)  in ips_remove()
1591 	if (ips->gpu_lower)  in ips_remove()
1593 	if (ips->gpu_busy)  in ips_remove()
1595 	if (ips->gpu_turbo_disable)  in ips_remove()
1601 	wrmsrl(TURBO_POWER_CURRENT_LIMIT, ips->orig_turbo_limit);  in ips_remove()
1603 	free_irq(ips->irq, ips);  in ips_remove()
1605 	if (ips->adjust)  in ips_remove()
1606 		kthread_stop(ips->adjust);  in ips_remove()
1607 	if (ips->monitor)  in ips_remove()
1608 		kthread_stop(ips->monitor);  in ips_remove()
1609 	dev_dbg(&dev->dev, "IPS driver removed\n");  in ips_remove()