1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2020 - 2024 Intel Corporation
4 */
5
6 #include "ivpu_drv.h"
7 #include "ivpu_hw.h"
8 #include "ivpu_hw_btrs.h"
9 #include "ivpu_hw_ip.h"
10
11 #include <linux/dmi.h>
12 #include <linux/fault-inject.h>
13 #include <linux/pm_runtime.h>
14
15 #ifdef CONFIG_FAULT_INJECTION
16 DECLARE_FAULT_ATTR(ivpu_hw_failure);
17
18 static char *ivpu_fail_hw;
19 module_param_named_unsafe(fail_hw, ivpu_fail_hw, charp, 0444);
20 MODULE_PARM_DESC(fail_hw, "<interval>,<probability>,<space>,<times>");
21 #endif
22
platform_to_str(u32 platform)23 static char *platform_to_str(u32 platform)
24 {
25 switch (platform) {
26 case IVPU_PLATFORM_SILICON:
27 return "SILICON";
28 case IVPU_PLATFORM_SIMICS:
29 return "SIMICS";
30 case IVPU_PLATFORM_FPGA:
31 return "FPGA";
32 case IVPU_PLATFORM_HSLE:
33 return "HSLE";
34 default:
35 return "Invalid platform";
36 }
37 }
38
platform_init(struct ivpu_device * vdev)39 static void platform_init(struct ivpu_device *vdev)
40 {
41 int platform = ivpu_hw_btrs_platform_read(vdev);
42
43 ivpu_dbg(vdev, MISC, "Platform type: %s (%d)\n", platform_to_str(platform), platform);
44
45 switch (platform) {
46 case IVPU_PLATFORM_SILICON:
47 case IVPU_PLATFORM_SIMICS:
48 case IVPU_PLATFORM_FPGA:
49 case IVPU_PLATFORM_HSLE:
50 vdev->platform = platform;
51 break;
52
53 default:
54 ivpu_err(vdev, "Invalid platform type: %d\n", platform);
55 break;
56 }
57 }
58
wa_init(struct ivpu_device * vdev)59 static void wa_init(struct ivpu_device *vdev)
60 {
61 vdev->wa.punit_disabled = false;
62 vdev->wa.clear_runtime_mem = false;
63
64 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
65 vdev->wa.interrupt_clear_with_0 = ivpu_hw_btrs_irqs_clear_with_0_mtl(vdev);
66
67 if (ivpu_device_id(vdev) == PCI_DEVICE_ID_LNL &&
68 ivpu_revision(vdev) < IVPU_HW_IP_REV_LNL_B0)
69 vdev->wa.disable_clock_relinquish = true;
70
71 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_ENABLE)
72 vdev->wa.disable_clock_relinquish = false;
73
74 if (ivpu_test_mode & IVPU_TEST_MODE_CLK_RELINQ_DISABLE)
75 vdev->wa.disable_clock_relinquish = true;
76
77 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
78 vdev->wa.wp0_during_power_up = true;
79
80 if (ivpu_test_mode & IVPU_TEST_MODE_D0I2_DISABLE)
81 vdev->wa.disable_d0i2 = true;
82
83 IVPU_PRINT_WA(punit_disabled);
84 IVPU_PRINT_WA(clear_runtime_mem);
85 IVPU_PRINT_WA(interrupt_clear_with_0);
86 IVPU_PRINT_WA(disable_clock_relinquish);
87 IVPU_PRINT_WA(wp0_during_power_up);
88 IVPU_PRINT_WA(disable_d0i2);
89 }
90
timeouts_init(struct ivpu_device * vdev)91 static void timeouts_init(struct ivpu_device *vdev)
92 {
93 if (ivpu_test_mode & IVPU_TEST_MODE_DISABLE_TIMEOUTS) {
94 vdev->timeout.boot = -1;
95 vdev->timeout.jsm = -1;
96 vdev->timeout.tdr = -1;
97 vdev->timeout.inference = -1;
98 vdev->timeout.autosuspend = -1;
99 vdev->timeout.d0i3_entry_msg = -1;
100 } else if (ivpu_is_fpga(vdev)) {
101 vdev->timeout.boot = 50;
102 vdev->timeout.jsm = 15000;
103 vdev->timeout.tdr = 30000;
104 vdev->timeout.inference = 900000;
105 vdev->timeout.autosuspend = -1;
106 vdev->timeout.d0i3_entry_msg = 500;
107 vdev->timeout.state_dump_msg = 10000;
108 } else if (ivpu_is_simics(vdev)) {
109 vdev->timeout.boot = 50;
110 vdev->timeout.jsm = 500;
111 vdev->timeout.tdr = 10000;
112 vdev->timeout.inference = 300000;
113 vdev->timeout.autosuspend = 100;
114 vdev->timeout.d0i3_entry_msg = 100;
115 vdev->timeout.state_dump_msg = 10;
116 } else {
117 vdev->timeout.boot = 1000;
118 vdev->timeout.jsm = 500;
119 vdev->timeout.tdr = 2000;
120 vdev->timeout.inference = 60000;
121 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
122 vdev->timeout.autosuspend = 10;
123 else
124 vdev->timeout.autosuspend = 100;
125 vdev->timeout.d0i3_entry_msg = 5;
126 vdev->timeout.state_dump_msg = 100;
127 }
128 }
129
priority_bands_init(struct ivpu_device * vdev)130 static void priority_bands_init(struct ivpu_device *vdev)
131 {
132 /* Idle */
133 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 0;
134 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 50000;
135 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_IDLE] = 160000;
136 /* Normal */
137 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000;
138 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 50000;
139 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_NORMAL] = 300000;
140 /* Focus */
141 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000;
142 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 50000;
143 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_FOCUS] = 200000;
144 /* Realtime */
145 vdev->hw->hws.grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 0;
146 vdev->hw->hws.process_grace_period[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 50000;
147 vdev->hw->hws.process_quantum[VPU_JOB_SCHEDULING_PRIORITY_BAND_REALTIME] = 200000;
148 }
149
memory_ranges_init(struct ivpu_device * vdev)150 static void memory_ranges_init(struct ivpu_device *vdev)
151 {
152 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
153 ivpu_hw_range_init(&vdev->hw->ranges.global, 0x80000000, SZ_512M);
154 ivpu_hw_range_init(&vdev->hw->ranges.user, 0x88000000, 511 * SZ_1M);
155 ivpu_hw_range_init(&vdev->hw->ranges.shave, 0x180000000, SZ_2G);
156 ivpu_hw_range_init(&vdev->hw->ranges.dma, 0x200000000, SZ_128G);
157 } else {
158 ivpu_hw_range_init(&vdev->hw->ranges.global, 0x80000000, SZ_512M);
159 ivpu_hw_range_init(&vdev->hw->ranges.shave, 0x80000000, SZ_2G);
160 ivpu_hw_range_init(&vdev->hw->ranges.user, 0x100000000, SZ_256G);
161 vdev->hw->ranges.dma = vdev->hw->ranges.user;
162 }
163 }
164
wp_enable(struct ivpu_device * vdev)165 static int wp_enable(struct ivpu_device *vdev)
166 {
167 return ivpu_hw_btrs_wp_drive(vdev, true);
168 }
169
wp_disable(struct ivpu_device * vdev)170 static int wp_disable(struct ivpu_device *vdev)
171 {
172 return ivpu_hw_btrs_wp_drive(vdev, false);
173 }
174
ivpu_hw_power_up(struct ivpu_device * vdev)175 int ivpu_hw_power_up(struct ivpu_device *vdev)
176 {
177 int ret;
178
179 if (IVPU_WA(wp0_during_power_up)) {
180 /* WP requests may fail when powering down, so issue WP 0 here */
181 ret = wp_disable(vdev);
182 if (ret)
183 ivpu_warn(vdev, "Failed to disable workpoint: %d\n", ret);
184 }
185
186 ret = ivpu_hw_btrs_d0i3_disable(vdev);
187 if (ret)
188 ivpu_warn(vdev, "Failed to disable D0I3: %d\n", ret);
189
190 ret = wp_enable(vdev);
191 if (ret) {
192 ivpu_err(vdev, "Failed to enable workpoint: %d\n", ret);
193 return ret;
194 }
195
196 if (ivpu_hw_btrs_gen(vdev) >= IVPU_HW_BTRS_LNL) {
197 if (IVPU_WA(disable_clock_relinquish))
198 ivpu_hw_btrs_clock_relinquish_disable_lnl(vdev);
199 ivpu_hw_btrs_profiling_freq_reg_set_lnl(vdev);
200 ivpu_hw_btrs_ats_print_lnl(vdev);
201 }
202
203 ret = ivpu_hw_ip_host_ss_configure(vdev);
204 if (ret) {
205 ivpu_err(vdev, "Failed to configure host SS: %d\n", ret);
206 return ret;
207 }
208
209 ivpu_hw_ip_idle_gen_disable(vdev);
210
211 ret = ivpu_hw_btrs_wait_for_clock_res_own_ack(vdev);
212 if (ret) {
213 ivpu_err(vdev, "Timed out waiting for clock resource own ACK\n");
214 return ret;
215 }
216
217 ret = ivpu_hw_ip_pwr_domain_enable(vdev);
218 if (ret) {
219 ivpu_err(vdev, "Failed to enable power domain: %d\n", ret);
220 return ret;
221 }
222
223 ret = ivpu_hw_ip_host_ss_axi_enable(vdev);
224 if (ret) {
225 ivpu_err(vdev, "Failed to enable AXI: %d\n", ret);
226 return ret;
227 }
228
229 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_LNL)
230 ivpu_hw_btrs_set_port_arbitration_weights_lnl(vdev);
231
232 ret = ivpu_hw_ip_top_noc_enable(vdev);
233 if (ret)
234 ivpu_err(vdev, "Failed to enable TOP NOC: %d\n", ret);
235
236 return ret;
237 }
238
save_d0i3_entry_timestamp(struct ivpu_device * vdev)239 static void save_d0i3_entry_timestamp(struct ivpu_device *vdev)
240 {
241 vdev->hw->d0i3_entry_host_ts = ktime_get_boottime();
242 vdev->hw->d0i3_entry_vpu_ts = ivpu_hw_ip_read_perf_timer_counter(vdev);
243 }
244
ivpu_hw_reset(struct ivpu_device * vdev)245 int ivpu_hw_reset(struct ivpu_device *vdev)
246 {
247 int ret = 0;
248
249 if (ivpu_hw_btrs_ip_reset(vdev)) {
250 ivpu_err(vdev, "Failed to reset NPU IP\n");
251 ret = -EIO;
252 }
253
254 if (wp_disable(vdev)) {
255 ivpu_err(vdev, "Failed to disable workpoint\n");
256 ret = -EIO;
257 }
258
259 return ret;
260 }
261
ivpu_hw_power_down(struct ivpu_device * vdev)262 int ivpu_hw_power_down(struct ivpu_device *vdev)
263 {
264 int ret = 0;
265
266 save_d0i3_entry_timestamp(vdev);
267
268 if (!ivpu_hw_is_idle(vdev))
269 ivpu_warn(vdev, "NPU not idle during power down\n");
270
271 if (ivpu_hw_reset(vdev)) {
272 ivpu_err(vdev, "Failed to reset NPU\n");
273 ret = -EIO;
274 }
275
276 if (ivpu_hw_btrs_d0i3_enable(vdev)) {
277 ivpu_err(vdev, "Failed to enter D0I3\n");
278 ret = -EIO;
279 }
280
281 return ret;
282 }
283
ivpu_hw_init(struct ivpu_device * vdev)284 int ivpu_hw_init(struct ivpu_device *vdev)
285 {
286 ivpu_hw_btrs_info_init(vdev);
287 ivpu_hw_btrs_freq_ratios_init(vdev);
288 priority_bands_init(vdev);
289 memory_ranges_init(vdev);
290 platform_init(vdev);
291 wa_init(vdev);
292 timeouts_init(vdev);
293 atomic_set(&vdev->hw->firewall_irq_counter, 0);
294
295 #ifdef CONFIG_FAULT_INJECTION
296 if (ivpu_fail_hw)
297 setup_fault_attr(&ivpu_hw_failure, ivpu_fail_hw);
298 #endif
299
300 return 0;
301 }
302
ivpu_hw_boot_fw(struct ivpu_device * vdev)303 int ivpu_hw_boot_fw(struct ivpu_device *vdev)
304 {
305 int ret;
306
307 ivpu_hw_ip_snoop_disable(vdev);
308 ivpu_hw_ip_tbu_mmu_enable(vdev);
309 ret = ivpu_hw_ip_soc_cpu_boot(vdev);
310 if (ret)
311 ivpu_err(vdev, "Failed to boot SOC CPU: %d\n", ret);
312
313 return ret;
314 }
315
ivpu_hw_profiling_freq_drive(struct ivpu_device * vdev,bool enable)316 void ivpu_hw_profiling_freq_drive(struct ivpu_device *vdev, bool enable)
317 {
318 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX) {
319 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
320 return;
321 }
322
323 if (enable)
324 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_HIGH;
325 else
326 vdev->hw->pll.profiling_freq = PLL_PROFILING_FREQ_DEFAULT;
327 }
328
ivpu_irq_handlers_init(struct ivpu_device * vdev)329 void ivpu_irq_handlers_init(struct ivpu_device *vdev)
330 {
331 if (ivpu_hw_ip_gen(vdev) == IVPU_HW_IP_37XX)
332 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_37xx;
333 else
334 vdev->hw->irq.ip_irq_handler = ivpu_hw_ip_irq_handler_40xx;
335
336 if (ivpu_hw_btrs_gen(vdev) == IVPU_HW_BTRS_MTL)
337 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_mtl;
338 else
339 vdev->hw->irq.btrs_irq_handler = ivpu_hw_btrs_irq_handler_lnl;
340 }
341
ivpu_hw_irq_enable(struct ivpu_device * vdev)342 void ivpu_hw_irq_enable(struct ivpu_device *vdev)
343 {
344 ivpu_hw_ip_irq_enable(vdev);
345 ivpu_hw_btrs_irq_enable(vdev);
346 }
347
ivpu_hw_irq_disable(struct ivpu_device * vdev)348 void ivpu_hw_irq_disable(struct ivpu_device *vdev)
349 {
350 ivpu_hw_btrs_irq_disable(vdev);
351 ivpu_hw_ip_irq_disable(vdev);
352 }
353
ivpu_hw_irq_handler(int irq,void * ptr)354 irqreturn_t ivpu_hw_irq_handler(int irq, void *ptr)
355 {
356 struct ivpu_device *vdev = ptr;
357 bool ip_handled, btrs_handled;
358
359 ivpu_hw_btrs_global_int_disable(vdev);
360
361 btrs_handled = ivpu_hw_btrs_irq_handler(vdev, irq);
362 if (!ivpu_hw_is_idle((vdev)) || !btrs_handled)
363 ip_handled = ivpu_hw_ip_irq_handler(vdev, irq);
364 else
365 ip_handled = false;
366
367 /* Re-enable global interrupts to re-trigger MSI for pending interrupts */
368 ivpu_hw_btrs_global_int_enable(vdev);
369
370 if (!ip_handled && !btrs_handled)
371 return IRQ_NONE;
372
373 pm_runtime_mark_last_busy(vdev->drm.dev);
374 return IRQ_HANDLED;
375 }
376