1 // SPDX-License-Identifier: MIT
2 /*
3 * Copyright © 2021 Intel Corporation
4 */
5
6 #include "gem/i915_gem_domain.h"
7 #include "gem/i915_gem_internal.h"
8 #include "gem/i915_gem_lmem.h"
9 #include "gt/gen8_ppgtt.h"
10
11 #include "i915_drv.h"
12 #include "intel_display_types.h"
13 #include "intel_dpt.h"
14 #include "intel_fb.h"
15
16 struct i915_dpt {
17 struct i915_address_space vm;
18
19 struct drm_i915_gem_object *obj;
20 struct i915_vma *vma;
21 void __iomem *iomem;
22 };
23
24 #define i915_is_dpt(vm) ((vm)->is_dpt)
25
26 static inline struct i915_dpt *
i915_vm_to_dpt(struct i915_address_space * vm)27 i915_vm_to_dpt(struct i915_address_space *vm)
28 {
29 BUILD_BUG_ON(offsetof(struct i915_dpt, vm));
30 drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm));
31 return container_of(vm, struct i915_dpt, vm);
32 }
33
34 #define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT)
35
gen8_set_pte(void __iomem * addr,gen8_pte_t pte)36 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte)
37 {
38 writeq(pte, addr);
39 }
40
dpt_insert_page(struct i915_address_space * vm,dma_addr_t addr,u64 offset,unsigned int pat_index,u32 flags)41 static void dpt_insert_page(struct i915_address_space *vm,
42 dma_addr_t addr,
43 u64 offset,
44 unsigned int pat_index,
45 u32 flags)
46 {
47 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
48 gen8_pte_t __iomem *base = dpt->iomem;
49
50 gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE,
51 vm->pte_encode(addr, pat_index, flags));
52 }
53
dpt_insert_entries(struct i915_address_space * vm,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)54 static void dpt_insert_entries(struct i915_address_space *vm,
55 struct i915_vma_resource *vma_res,
56 unsigned int pat_index,
57 u32 flags)
58 {
59 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
60 gen8_pte_t __iomem *base = dpt->iomem;
61 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags);
62 struct sgt_iter sgt_iter;
63 dma_addr_t addr;
64 int i;
65
66 /*
67 * Note that we ignore PTE_READ_ONLY here. The caller must be careful
68 * not to allow the user to override access to a read only page.
69 */
70
71 i = vma_res->start / I915_GTT_PAGE_SIZE;
72 for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages)
73 gen8_set_pte(&base[i++], pte_encode | addr);
74 }
75
dpt_clear_range(struct i915_address_space * vm,u64 start,u64 length)76 static void dpt_clear_range(struct i915_address_space *vm,
77 u64 start, u64 length)
78 {
79 }
80
dpt_bind_vma(struct i915_address_space * vm,struct i915_vm_pt_stash * stash,struct i915_vma_resource * vma_res,unsigned int pat_index,u32 flags)81 static void dpt_bind_vma(struct i915_address_space *vm,
82 struct i915_vm_pt_stash *stash,
83 struct i915_vma_resource *vma_res,
84 unsigned int pat_index,
85 u32 flags)
86 {
87 u32 pte_flags;
88
89 if (vma_res->bound_flags)
90 return;
91
92 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */
93 pte_flags = 0;
94 if (vm->has_read_only && vma_res->bi.readonly)
95 pte_flags |= PTE_READ_ONLY;
96 if (vma_res->bi.lmem)
97 pte_flags |= PTE_LM;
98
99 vm->insert_entries(vm, vma_res, pat_index, pte_flags);
100
101 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE;
102
103 /*
104 * Without aliasing PPGTT there's no difference between
105 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally
106 * upgrade to both bound if we bind either to avoid double-binding.
107 */
108 vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND;
109 }
110
dpt_unbind_vma(struct i915_address_space * vm,struct i915_vma_resource * vma_res)111 static void dpt_unbind_vma(struct i915_address_space *vm,
112 struct i915_vma_resource *vma_res)
113 {
114 vm->clear_range(vm, vma_res->start, vma_res->vma_size);
115 }
116
dpt_cleanup(struct i915_address_space * vm)117 static void dpt_cleanup(struct i915_address_space *vm)
118 {
119 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
120
121 i915_gem_object_put(dpt->obj);
122 }
123
intel_dpt_pin_to_ggtt(struct i915_address_space * vm,unsigned int alignment)124 struct i915_vma *intel_dpt_pin_to_ggtt(struct i915_address_space *vm,
125 unsigned int alignment)
126 {
127 struct drm_i915_private *i915 = vm->i915;
128 struct intel_display *display = &i915->display;
129 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
130 intel_wakeref_t wakeref;
131 struct i915_vma *vma;
132 void __iomem *iomem;
133 struct i915_gem_ww_ctx ww;
134 u64 pin_flags = 0;
135 int err;
136
137 if (i915_gem_object_is_stolen(dpt->obj))
138 pin_flags |= PIN_MAPPABLE;
139
140 wakeref = intel_runtime_pm_get(&i915->runtime_pm);
141 atomic_inc(&display->restore.pending_fb_pin);
142
143 for_i915_gem_ww(&ww, err, true) {
144 err = i915_gem_object_lock(dpt->obj, &ww);
145 if (err)
146 continue;
147
148 vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0,
149 alignment, pin_flags);
150 if (IS_ERR(vma)) {
151 err = PTR_ERR(vma);
152 continue;
153 }
154
155 iomem = i915_vma_pin_iomap(vma);
156 i915_vma_unpin(vma);
157
158 if (IS_ERR(iomem)) {
159 err = PTR_ERR(iomem);
160 continue;
161 }
162
163 dpt->vma = vma;
164 dpt->iomem = iomem;
165
166 i915_vma_get(vma);
167 }
168
169 dpt->obj->mm.dirty = true;
170
171 atomic_dec(&display->restore.pending_fb_pin);
172 intel_runtime_pm_put(&i915->runtime_pm, wakeref);
173
174 return err ? ERR_PTR(err) : vma;
175 }
176
intel_dpt_unpin_from_ggtt(struct i915_address_space * vm)177 void intel_dpt_unpin_from_ggtt(struct i915_address_space *vm)
178 {
179 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
180
181 i915_vma_unpin_iomap(dpt->vma);
182 i915_vma_put(dpt->vma);
183 }
184
185 /**
186 * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume
187 * @display: display device instance
188 *
189 * Restore the memory mapping during system resume for all framebuffers which
190 * are mapped to HW via a GGTT->DPT page table. The content of these page
191 * tables are not stored in the hibernation image during S4 and S3RST->S4
192 * transitions, so here we reprogram the PTE entries in those tables.
193 *
194 * This function must be called after the mappings in GGTT have been restored calling
195 * i915_ggtt_resume().
196 */
intel_dpt_resume(struct intel_display * display)197 void intel_dpt_resume(struct intel_display *display)
198 {
199 struct drm_framebuffer *drm_fb;
200
201 if (!HAS_DISPLAY(display))
202 return;
203
204 mutex_lock(&display->drm->mode_config.fb_lock);
205 drm_for_each_fb(drm_fb, display->drm) {
206 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
207
208 if (fb->dpt_vm)
209 i915_ggtt_resume_vm(fb->dpt_vm, true);
210 }
211 mutex_unlock(&display->drm->mode_config.fb_lock);
212 }
213
214 /**
215 * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend
216 * @display: display device instance
217 *
218 * Suspend the memory mapping during system suspend for all framebuffers which
219 * are mapped to HW via a GGTT->DPT page table.
220 *
221 * This function must be called before the mappings in GGTT are suspended calling
222 * i915_ggtt_suspend().
223 */
intel_dpt_suspend(struct intel_display * display)224 void intel_dpt_suspend(struct intel_display *display)
225 {
226 struct drm_framebuffer *drm_fb;
227
228 if (!HAS_DISPLAY(display))
229 return;
230
231 mutex_lock(&display->drm->mode_config.fb_lock);
232
233 drm_for_each_fb(drm_fb, display->drm) {
234 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb);
235
236 if (fb->dpt_vm)
237 i915_ggtt_suspend_vm(fb->dpt_vm, true);
238 }
239
240 mutex_unlock(&display->drm->mode_config.fb_lock);
241 }
242
243 struct i915_address_space *
intel_dpt_create(struct intel_framebuffer * fb)244 intel_dpt_create(struct intel_framebuffer *fb)
245 {
246 struct drm_gem_object *obj = intel_fb_bo(&fb->base);
247 struct drm_i915_private *i915 = to_i915(obj->dev);
248 struct drm_i915_gem_object *dpt_obj;
249 struct i915_address_space *vm;
250 struct i915_dpt *dpt;
251 size_t size;
252 int ret;
253
254 if (intel_fb_needs_pot_stride_remap(fb))
255 size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped);
256 else
257 size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE);
258
259 size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE);
260
261 dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS);
262 if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt))
263 dpt_obj = i915_gem_object_create_stolen(i915, size);
264 if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) {
265 drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n");
266 dpt_obj = i915_gem_object_create_shmem(i915, size);
267 }
268 if (IS_ERR(dpt_obj))
269 return ERR_CAST(dpt_obj);
270
271 ret = i915_gem_object_lock_interruptible(dpt_obj, NULL);
272 if (!ret) {
273 ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE);
274 i915_gem_object_unlock(dpt_obj);
275 }
276 if (ret) {
277 i915_gem_object_put(dpt_obj);
278 return ERR_PTR(ret);
279 }
280
281 dpt = kzalloc(sizeof(*dpt), GFP_KERNEL);
282 if (!dpt) {
283 i915_gem_object_put(dpt_obj);
284 return ERR_PTR(-ENOMEM);
285 }
286
287 vm = &dpt->vm;
288
289 vm->gt = to_gt(i915);
290 vm->i915 = i915;
291 vm->dma = i915->drm.dev;
292 vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE;
293 vm->is_dpt = true;
294
295 i915_address_space_init(vm, VM_CLASS_DPT);
296
297 vm->insert_page = dpt_insert_page;
298 vm->clear_range = dpt_clear_range;
299 vm->insert_entries = dpt_insert_entries;
300 vm->cleanup = dpt_cleanup;
301
302 vm->vma_ops.bind_vma = dpt_bind_vma;
303 vm->vma_ops.unbind_vma = dpt_unbind_vma;
304
305 vm->pte_encode = vm->gt->ggtt->vm.pte_encode;
306
307 dpt->obj = dpt_obj;
308 dpt->obj->is_dpt = true;
309
310 return &dpt->vm;
311 }
312
intel_dpt_destroy(struct i915_address_space * vm)313 void intel_dpt_destroy(struct i915_address_space *vm)
314 {
315 struct i915_dpt *dpt = i915_vm_to_dpt(vm);
316
317 dpt->obj->is_dpt = false;
318 i915_vm_put(&dpt->vm);
319 }
320
intel_dpt_offset(struct i915_vma * dpt_vma)321 u64 intel_dpt_offset(struct i915_vma *dpt_vma)
322 {
323 return dpt_vma->node.start;
324 }
325