Lines Matching +full:cs +full:- +full:x
1 // SPDX-License-Identifier: MIT
22 #define CPU_LATENCY 0 /* -1 to disable pm_qos, 0 to disable cstates */
33 return -1; in cmp_u64()
45 return -1; in cmp_u32()
64 #define CS_GPR(x) GEN8_RING_CS_GPR(engine->mmio_base, x) in create_spin_counter() argument
68 u32 *base, *cs; in create_spin_counter() local
72 obj = i915_gem_object_create_internal(vm->i915, 64 << 10); in create_spin_counter()
76 end = obj->base.size / sizeof(u32) - 1; in create_spin_counter()
95 cs = base; in create_spin_counter()
97 *cs++ = MI_LOAD_REGISTER_IMM(__NGPR__ * 2); in create_spin_counter()
99 *cs++ = i915_mmio_reg_offset(CS_GPR(i)); in create_spin_counter()
100 *cs++ = 0; in create_spin_counter()
101 *cs++ = i915_mmio_reg_offset(CS_GPR(i)) + 4; in create_spin_counter()
102 *cs++ = 0; in create_spin_counter()
105 *cs++ = MI_LOAD_REGISTER_IMM(1); in create_spin_counter()
106 *cs++ = i915_mmio_reg_offset(CS_GPR(INC)); in create_spin_counter()
107 *cs++ = 1; in create_spin_counter()
109 loop = cs - base; in create_spin_counter()
113 *cs++ = MI_MATH(4); in create_spin_counter()
114 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCA, MI_MATH_REG(COUNT)); in create_spin_counter()
115 *cs++ = MI_MATH_LOAD(MI_MATH_REG_SRCB, MI_MATH_REG(INC)); in create_spin_counter()
116 *cs++ = MI_MATH_ADD; in create_spin_counter()
117 *cs++ = MI_MATH_STORE(MI_MATH_REG(COUNT), MI_MATH_REG_ACCU); in create_spin_counter()
120 *cs++ = MI_STORE_REGISTER_MEM_GEN8; in create_spin_counter()
121 *cs++ = i915_mmio_reg_offset(CS_GPR(COUNT)); in create_spin_counter()
122 *cs++ = lower_32_bits(vma->node.start + end * sizeof(*cs)); in create_spin_counter()
123 *cs++ = upper_32_bits(vma->node.start + end * sizeof(*cs)); in create_spin_counter()
127 *cs++ = MI_BATCH_BUFFER_START_GEN8; in create_spin_counter()
128 *cs++ = lower_32_bits(vma->node.start + loop * sizeof(*cs)); in create_spin_counter()
129 *cs++ = upper_32_bits(vma->node.start + loop * sizeof(*cs)); in create_spin_counter()
130 GEM_BUG_ON(cs - base > end); in create_spin_counter()
186 mutex_lock(&rps->lock); in rps_set_check()
189 GEM_BUG_ON(rps->last_freq != freq); in rps_set_check()
190 mutex_unlock(&rps->lock); in rps_set_check()
200 pr_info("P_STATE_CAP[%x]: 0x%08x\n", in show_pstate_limits()
205 pr_info("P_STATE_LIMITS[%x]: 0x%08x\n", in show_pstate_limits()
215 struct intel_rps *rps = >->rps; in live_rps_clock_interval()
226 return -ENOMEM; in live_rps_clock_interval()
229 saved_work = rps->work.func; in live_rps_clock_interval()
230 rps->work.func = dummy_rps_work; in live_rps_clock_interval()
233 intel_rps_disable(>->rps); in live_rps_clock_interval()
248 engine->kernel_context, in live_rps_clock_interval()
260 engine->name); in live_rps_clock_interval()
263 intel_gt_set_wedged(engine->gt); in live_rps_clock_interval()
264 err = -EIO; in live_rps_clock_interval()
268 intel_uncore_forcewake_get(gt->uncore, FORCEWAKE_ALL); in live_rps_clock_interval()
270 intel_uncore_write_fw(gt->uncore, GEN6_RP_CUR_UP_EI, 0); in live_rps_clock_interval()
273 intel_uncore_write_fw(gt->uncore, in live_rps_clock_interval()
275 intel_uncore_write_fw(gt->uncore, in live_rps_clock_interval()
278 intel_uncore_write_fw(gt->uncore, GEN6_RP_CONTROL, in live_rps_clock_interval()
281 if (wait_for(intel_uncore_read_fw(gt->uncore, in live_rps_clock_interval()
286 engine->name); in live_rps_clock_interval()
287 err = -ENODEV; in live_rps_clock_interval()
297 cycles_[i] = -intel_uncore_read_fw(gt->uncore, GEN6_RP_CUR_UP_EI); in live_rps_clock_interval()
302 cycles_[i] += intel_uncore_read_fw(gt->uncore, GEN6_RP_CUR_UP_EI); in live_rps_clock_interval()
314 intel_uncore_write_fw(gt->uncore, GEN6_RP_CONTROL, 0); in live_rps_clock_interval()
315 intel_uncore_forcewake_put(gt->uncore, FORCEWAKE_ALL); in live_rps_clock_interval()
326 engine->name, cycles, time, dt, expected, in live_rps_clock_interval()
327 gt->clock_frequency / 1000); in live_rps_clock_interval()
332 engine->name); in live_rps_clock_interval()
333 err = -EINVAL; in live_rps_clock_interval()
339 engine->name); in live_rps_clock_interval()
340 err = -EINVAL; in live_rps_clock_interval()
344 if (igt_flush_test(gt->i915)) in live_rps_clock_interval()
345 err = -EIO; in live_rps_clock_interval()
350 intel_rps_enable(>->rps); in live_rps_clock_interval()
356 rps->work.func = saved_work; in live_rps_clock_interval()
358 if (err == -ENODEV) /* skipped, don't report a fail */ in live_rps_clock_interval()
367 struct intel_rps *rps = >->rps; in live_rps_control()
384 if (IS_CHERRYVIEW(gt->i915)) /* XXX fragile PCU */ in live_rps_control()
388 return -ENOMEM; in live_rps_control()
391 saved_work = rps->work.func; in live_rps_control()
392 rps->work.func = dummy_rps_work; in live_rps_control()
407 engine->kernel_context, in live_rps_control()
418 engine->name); in live_rps_control()
421 intel_gt_set_wedged(engine->gt); in live_rps_control()
422 err = -EIO; in live_rps_control()
426 if (rps_set_check(rps, rps->min_freq) != rps->min_freq) { in live_rps_control()
427 pr_err("%s: could not set minimum frequency [%x], only %x!\n", in live_rps_control()
428 engine->name, rps->min_freq, read_cagf(rps)); in live_rps_control()
432 err = -EINVAL; in live_rps_control()
436 for (f = rps->min_freq + 1; f < rps->max_freq; f++) { in live_rps_control()
443 if (rps_set_check(rps, rps->min_freq) != rps->min_freq) { in live_rps_control()
444 pr_err("%s: could not restore minimum frequency [%x], only %x!\n", in live_rps_control()
445 engine->name, rps->min_freq, read_cagf(rps)); in live_rps_control()
449 err = -EINVAL; in live_rps_control()
458 min = rps_set_check(rps, rps->min_freq); in live_rps_control()
464 pr_info("%s: range:[%x:%uMHz, %x:%uMHz] limit:[%x:%uMHz], %x:%x response %lluns:%lluns\n", in live_rps_control()
465 engine->name, in live_rps_control()
466 rps->min_freq, intel_gpu_freq(rps, rps->min_freq), in live_rps_control()
467 rps->max_freq, intel_gpu_freq(rps, rps->max_freq), in live_rps_control()
471 if (limit == rps->min_freq) { in live_rps_control()
473 engine->name); in live_rps_control()
475 err = -ENODEV; in live_rps_control()
479 if (igt_flush_test(gt->i915)) { in live_rps_control()
480 err = -EIO; in live_rps_control()
489 rps->work.func = saved_work; in live_rps_control()
504 min_gpu_freq = rps->min_freq; in show_pcu_config()
505 max_gpu_freq = rps->max_freq; in show_pcu_config()
512 wakeref = intel_runtime_pm_get(rps_to_uncore(rps)->rpm); in show_pcu_config()
528 intel_runtime_pm_put(rps_to_uncore(rps)->rpm, wakeref); in show_pcu_config()
538 dc = READ_ONCE(*cntr) - dc; in __measure_frequency()
539 dt = ktime_get() - dt; in __measure_frequency()
546 u64 x[5]; in measure_frequency_at() local
551 x[i] = __measure_frequency(cntr, 2); in measure_frequency_at()
555 sort(x, 5, sizeof(*x), cmp_u64, NULL); in measure_frequency_at()
556 return div_u64(x[1] + 2 * x[2] + x[3], 4); in measure_frequency_at()
565 dc = intel_uncore_read_fw(engine->uncore, CS_GPR(0)); in __measure_cs_frequency()
567 dc = intel_uncore_read_fw(engine->uncore, CS_GPR(0)) - dc; in __measure_cs_frequency()
568 dt = ktime_get() - dt; in __measure_cs_frequency()
577 u64 x[5]; in measure_cs_frequency_at() local
582 x[i] = __measure_cs_frequency(engine, 2); in measure_cs_frequency_at()
586 sort(x, 5, sizeof(*x), cmp_u64, NULL); in measure_cs_frequency_at()
587 return div_u64(x[1] + 2 * x[2] + x[3], 4); in measure_cs_frequency_at()
590 static bool scaled_within(u64 x, u64 y, u32 f_n, u32 f_d) in scaled_within() argument
592 return f_d * x > f_n * y && f_n * x < f_d * y; in scaled_within()
599 struct intel_rps *rps = >->rps; in live_rps_frequency_cs()
614 if (INTEL_GEN(gt->i915) < 8) /* for CS simplicity */ in live_rps_frequency_cs()
621 saved_work = rps->work.func; in live_rps_frequency_cs()
622 rps->work.func = dummy_rps_work; in live_rps_frequency_cs()
636 engine->kernel_context->vm, false, in live_rps_frequency_cs()
650 err = i915_request_await_object(rq, vma->obj, false); in live_rps_frequency_cs()
654 err = rq->engine->emit_bb_start(rq, in live_rps_frequency_cs()
655 vma->node.start, in live_rps_frequency_cs()
661 if (wait_for(intel_uncore_read(engine->uncore, CS_GPR(0)), in live_rps_frequency_cs()
664 engine->name); in live_rps_frequency_cs()
668 min.freq = rps->min_freq; in live_rps_frequency_cs()
671 max.freq = rps->max_freq; in live_rps_frequency_cs()
675 engine->name, in live_rps_frequency_cs()
686 pr_err("%s: CS did not scale with frequency! scaled min:%llu, max:%llu\n", in live_rps_frequency_cs()
687 engine->name, in live_rps_frequency_cs()
692 for (f = min.freq + 1; f <= rps->max_freq; f++) { in live_rps_frequency_cs()
700 pr_info("%s: %x:%uMHz: %lluKHz [%d%%]\n", in live_rps_frequency_cs()
701 engine->name, in live_rps_frequency_cs()
709 err = -EINTR; /* ignore error, continue on with test */ in live_rps_frequency_cs()
714 i915_gem_object_flush_map(vma->obj); in live_rps_frequency_cs()
715 i915_gem_object_unpin_map(vma->obj); in live_rps_frequency_cs()
721 if (igt_flush_test(gt->i915)) in live_rps_frequency_cs()
722 err = -EIO; in live_rps_frequency_cs()
728 rps->work.func = saved_work; in live_rps_frequency_cs()
740 struct intel_rps *rps = >->rps; in live_rps_frequency_srm()
755 if (INTEL_GEN(gt->i915) < 8) /* for CS simplicity */ in live_rps_frequency_srm()
762 saved_work = rps->work.func; in live_rps_frequency_srm()
763 rps->work.func = dummy_rps_work; in live_rps_frequency_srm()
777 engine->kernel_context->vm, true, in live_rps_frequency_srm()
791 err = i915_request_await_object(rq, vma->obj, false); in live_rps_frequency_srm()
795 err = rq->engine->emit_bb_start(rq, in live_rps_frequency_srm()
796 vma->node.start, in live_rps_frequency_srm()
804 engine->name); in live_rps_frequency_srm()
808 min.freq = rps->min_freq; in live_rps_frequency_srm()
811 max.freq = rps->max_freq; in live_rps_frequency_srm()
815 engine->name, in live_rps_frequency_srm()
826 pr_err("%s: CS did not scale with frequency! scaled min:%llu, max:%llu\n", in live_rps_frequency_srm()
827 engine->name, in live_rps_frequency_srm()
832 for (f = min.freq + 1; f <= rps->max_freq; f++) { in live_rps_frequency_srm()
840 pr_info("%s: %x:%uMHz: %lluKHz [%d%%]\n", in live_rps_frequency_srm()
841 engine->name, in live_rps_frequency_srm()
849 err = -EINTR; /* ignore error, continue on with test */ in live_rps_frequency_srm()
854 i915_gem_object_flush_map(vma->obj); in live_rps_frequency_srm()
855 i915_gem_object_unpin_map(vma->obj); in live_rps_frequency_srm()
861 if (igt_flush_test(gt->i915)) in live_rps_frequency_srm()
862 err = -EIO; in live_rps_frequency_srm()
868 rps->work.func = saved_work; in live_rps_frequency_srm()
883 GEM_BUG_ON(rps->pm_iir); in sleep_for_ei()
894 struct intel_uncore *uncore = engine->uncore; in __rps_up_interrupt()
901 rps_set_check(rps, rps->min_freq); in __rps_up_interrupt()
903 rq = igt_spinner_create_request(spin, engine->kernel_context, MI_NOOP); in __rps_up_interrupt()
912 engine->name); in __rps_up_interrupt()
914 intel_gt_set_wedged(engine->gt); in __rps_up_interrupt()
915 return -EIO; in __rps_up_interrupt()
920 engine->name); in __rps_up_interrupt()
923 return -EINVAL; in __rps_up_interrupt()
926 if (!(rps->pm_events & GEN6_PM_RP_UP_THRESHOLD)) { in __rps_up_interrupt()
928 engine->name); in __rps_up_interrupt()
930 return -EINVAL; in __rps_up_interrupt()
933 if (rps->last_freq != rps->min_freq) { in __rps_up_interrupt()
935 engine->name); in __rps_up_interrupt()
937 return -EINVAL; in __rps_up_interrupt()
941 timeout = intel_gt_pm_interval_to_ns(engine->gt, timeout); in __rps_up_interrupt()
950 if (rps->cur_freq != rps->min_freq) { in __rps_up_interrupt()
952 engine->name, intel_rps_read_actual_frequency(rps)); in __rps_up_interrupt()
953 return -EINVAL; in __rps_up_interrupt()
956 if (!(rps->pm_iir & GEN6_PM_RP_UP_THRESHOLD)) { in __rps_up_interrupt()
957 …err("%s: UP interrupt not recorded for spinner, pm_iir:%x, prev_up:%x, up_threshold:%x, up_ei:%x\n… in __rps_up_interrupt()
958 engine->name, rps->pm_iir, in __rps_up_interrupt()
962 return -EINVAL; in __rps_up_interrupt()
971 struct intel_uncore *uncore = engine->uncore; in __rps_down_interrupt()
974 rps_set_check(rps, rps->max_freq); in __rps_down_interrupt()
976 if (!(rps->pm_events & GEN6_PM_RP_DOWN_THRESHOLD)) { in __rps_down_interrupt()
978 engine->name); in __rps_down_interrupt()
979 return -EINVAL; in __rps_down_interrupt()
982 if (rps->last_freq != rps->max_freq) { in __rps_down_interrupt()
984 engine->name); in __rps_down_interrupt()
985 return -EINVAL; in __rps_down_interrupt()
989 timeout = intel_gt_pm_interval_to_ns(engine->gt, timeout); in __rps_down_interrupt()
994 if (rps->cur_freq != rps->max_freq) { in __rps_down_interrupt()
996 engine->name, in __rps_down_interrupt()
998 return -EINVAL; in __rps_down_interrupt()
1001 if (!(rps->pm_iir & (GEN6_PM_RP_DOWN_THRESHOLD | GEN6_PM_RP_DOWN_TIMEOUT))) { in __rps_down_interrupt()
1002 …pt not recorded for idle, pm_iir:%x, prev_down:%x, down_threshold:%x, down_ei:%x [prev_up:%x, up_t… in __rps_down_interrupt()
1003 engine->name, rps->pm_iir, in __rps_down_interrupt()
1010 return -EINVAL; in __rps_down_interrupt()
1019 struct intel_rps *rps = >->rps; in live_rps_interrupt()
1035 pm_events = rps->pm_events; in live_rps_interrupt()
1039 return -ENODEV; in live_rps_interrupt()
1043 return -ENOMEM; in live_rps_interrupt()
1046 saved_work = rps->work.func; in live_rps_interrupt()
1047 rps->work.func = dummy_rps_work; in live_rps_interrupt()
1052 intel_gt_pm_wait_for_idle(engine->gt); in live_rps_interrupt()
1063 intel_gt_pm_wait_for_idle(engine->gt); in live_rps_interrupt()
1069 intel_rc6_disable(>->rc6); in live_rps_interrupt()
1073 intel_rc6_enable(>->rc6); in live_rps_interrupt()
1081 if (igt_flush_test(gt->i915)) in live_rps_interrupt()
1082 err = -EIO; in live_rps_interrupt()
1087 rps->work.func = saved_work; in live_rps_interrupt()
1099 dE = librapl_energy_uJ() - dE; in __measure_power()
1100 dt = ktime_get() - dt; in __measure_power()
1107 u64 x[5]; in measure_power_at() local
1112 x[i] = __measure_power(5); in measure_power_at()
1116 sort(x, 5, sizeof(*x), cmp_u64, NULL); in measure_power_at()
1117 return div_u64(x[1] + 2 * x[2] + x[3], 4); in measure_power_at()
1123 struct intel_rps *rps = >->rps; in live_rps_power()
1143 return -ENOMEM; in live_rps_power()
1146 saved_work = rps->work.func; in live_rps_power()
1147 rps->work.func = dummy_rps_work; in live_rps_power()
1162 engine->kernel_context, in live_rps_power()
1174 engine->name); in live_rps_power()
1177 intel_gt_set_wedged(engine->gt); in live_rps_power()
1178 err = -EIO; in live_rps_power()
1182 max.freq = rps->max_freq; in live_rps_power()
1185 min.freq = rps->min_freq; in live_rps_power()
1192 engine->name, in live_rps_power()
1205 engine->name); in live_rps_power()
1206 err = -EINVAL; in live_rps_power()
1210 if (igt_flush_test(gt->i915)) { in live_rps_power()
1211 err = -EIO; in live_rps_power()
1219 rps->work.func = saved_work; in live_rps_power()
1227 struct intel_rps *rps = >->rps; in live_rps_dynamic()
1244 return -ENOMEM; in live_rps_dynamic()
1263 rps->cur_freq = rps->min_freq; in live_rps_dynamic()
1266 intel_rc6_disable(>->rc6); in live_rps_dynamic()
1267 GEM_BUG_ON(rps->last_freq != rps->min_freq); in live_rps_dynamic()
1270 engine->kernel_context, in live_rps_dynamic()
1280 max.freq = wait_for_freq(rps, rps->max_freq, 500); in live_rps_dynamic()
1286 min.freq = wait_for_freq(rps, rps->min_freq, 2000); in live_rps_dynamic()
1290 engine->name, in live_rps_dynamic()
1297 engine->name); in live_rps_dynamic()
1298 err = -EINVAL; in live_rps_dynamic()
1302 intel_rc6_enable(>->rc6); in live_rps_dynamic()
1305 if (igt_flush_test(gt->i915)) in live_rps_dynamic()
1306 err = -EIO; in live_rps_dynamic()