1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2021 Intel Corporation 4 */ 5 6 #include "gem/i915_gem_domain.h" 7 #include "gem/i915_gem_internal.h" 8 #include "gem/i915_gem_lmem.h" 9 #include "gt/gen8_ppgtt.h" 10 11 #include "i915_drv.h" 12 #include "intel_display_rpm.h" 13 #include "intel_display_types.h" 14 #include "intel_dpt.h" 15 #include "intel_fb.h" 16 17 struct i915_dpt { 18 struct i915_address_space vm; 19 20 struct drm_i915_gem_object *obj; 21 struct i915_vma *vma; 22 void __iomem *iomem; 23 }; 24 25 #define i915_is_dpt(vm) ((vm)->is_dpt) 26 27 static inline struct i915_dpt * 28 i915_vm_to_dpt(struct i915_address_space *vm) 29 { 30 BUILD_BUG_ON(offsetof(struct i915_dpt, vm)); 31 drm_WARN_ON(&vm->i915->drm, !i915_is_dpt(vm)); 32 return container_of(vm, struct i915_dpt, vm); 33 } 34 35 #define dpt_total_entries(dpt) ((dpt)->vm.total >> PAGE_SHIFT) 36 37 static void gen8_set_pte(void __iomem *addr, gen8_pte_t pte) 38 { 39 writeq(pte, addr); 40 } 41 42 static void dpt_insert_page(struct i915_address_space *vm, 43 dma_addr_t addr, 44 u64 offset, 45 unsigned int pat_index, 46 u32 flags) 47 { 48 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 49 gen8_pte_t __iomem *base = dpt->iomem; 50 51 gen8_set_pte(base + offset / I915_GTT_PAGE_SIZE, 52 vm->pte_encode(addr, pat_index, flags)); 53 } 54 55 static void dpt_insert_entries(struct i915_address_space *vm, 56 struct i915_vma_resource *vma_res, 57 unsigned int pat_index, 58 u32 flags) 59 { 60 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 61 gen8_pte_t __iomem *base = dpt->iomem; 62 const gen8_pte_t pte_encode = vm->pte_encode(0, pat_index, flags); 63 struct sgt_iter sgt_iter; 64 dma_addr_t addr; 65 int i; 66 67 /* 68 * Note that we ignore PTE_READ_ONLY here. The caller must be careful 69 * not to allow the user to override access to a read only page. 70 */ 71 72 i = vma_res->start / I915_GTT_PAGE_SIZE; 73 for_each_sgt_daddr(addr, sgt_iter, vma_res->bi.pages) 74 gen8_set_pte(&base[i++], pte_encode | addr); 75 } 76 77 static void dpt_clear_range(struct i915_address_space *vm, 78 u64 start, u64 length) 79 { 80 } 81 82 static void dpt_bind_vma(struct i915_address_space *vm, 83 struct i915_vm_pt_stash *stash, 84 struct i915_vma_resource *vma_res, 85 unsigned int pat_index, 86 u32 flags) 87 { 88 u32 pte_flags; 89 90 if (vma_res->bound_flags) 91 return; 92 93 /* Applicable to VLV (gen8+ do not support RO in the GGTT) */ 94 pte_flags = 0; 95 if (vm->has_read_only && vma_res->bi.readonly) 96 pte_flags |= PTE_READ_ONLY; 97 if (vma_res->bi.lmem) 98 pte_flags |= PTE_LM; 99 100 vm->insert_entries(vm, vma_res, pat_index, pte_flags); 101 102 vma_res->page_sizes_gtt = I915_GTT_PAGE_SIZE; 103 104 /* 105 * Without aliasing PPGTT there's no difference between 106 * GLOBAL/LOCAL_BIND, it's all the same ptes. Hence unconditionally 107 * upgrade to both bound if we bind either to avoid double-binding. 108 */ 109 vma_res->bound_flags = I915_VMA_GLOBAL_BIND | I915_VMA_LOCAL_BIND; 110 } 111 112 static void dpt_unbind_vma(struct i915_address_space *vm, 113 struct i915_vma_resource *vma_res) 114 { 115 vm->clear_range(vm, vma_res->start, vma_res->vma_size); 116 } 117 118 static void dpt_cleanup(struct i915_address_space *vm) 119 { 120 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 121 122 i915_gem_object_put(dpt->obj); 123 } 124 125 struct i915_vma *intel_dpt_pin_to_ggtt(struct i915_address_space *vm, 126 unsigned int alignment) 127 { 128 struct drm_i915_private *i915 = vm->i915; 129 struct intel_display *display = &i915->display; 130 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 131 struct ref_tracker *wakeref; 132 struct i915_vma *vma; 133 void __iomem *iomem; 134 struct i915_gem_ww_ctx ww; 135 u64 pin_flags = 0; 136 int err; 137 138 if (i915_gem_object_is_stolen(dpt->obj)) 139 pin_flags |= PIN_MAPPABLE; 140 141 wakeref = intel_display_rpm_get(display); 142 atomic_inc(&display->restore.pending_fb_pin); 143 144 for_i915_gem_ww(&ww, err, true) { 145 err = i915_gem_object_lock(dpt->obj, &ww); 146 if (err) 147 continue; 148 149 vma = i915_gem_object_ggtt_pin_ww(dpt->obj, &ww, NULL, 0, 150 alignment, pin_flags); 151 if (IS_ERR(vma)) { 152 err = PTR_ERR(vma); 153 continue; 154 } 155 156 iomem = i915_vma_pin_iomap(vma); 157 i915_vma_unpin(vma); 158 159 if (IS_ERR(iomem)) { 160 err = PTR_ERR(iomem); 161 continue; 162 } 163 164 dpt->vma = vma; 165 dpt->iomem = iomem; 166 167 i915_vma_get(vma); 168 } 169 170 dpt->obj->mm.dirty = true; 171 172 atomic_dec(&display->restore.pending_fb_pin); 173 intel_display_rpm_put(display, wakeref); 174 175 return err ? ERR_PTR(err) : vma; 176 } 177 178 void intel_dpt_unpin_from_ggtt(struct i915_address_space *vm) 179 { 180 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 181 182 i915_vma_unpin_iomap(dpt->vma); 183 i915_vma_put(dpt->vma); 184 } 185 186 /** 187 * intel_dpt_resume - restore the memory mapping for all DPT FBs during system resume 188 * @display: display device instance 189 * 190 * Restore the memory mapping during system resume for all framebuffers which 191 * are mapped to HW via a GGTT->DPT page table. The content of these page 192 * tables are not stored in the hibernation image during S4 and S3RST->S4 193 * transitions, so here we reprogram the PTE entries in those tables. 194 * 195 * This function must be called after the mappings in GGTT have been restored calling 196 * i915_ggtt_resume(). 197 */ 198 void intel_dpt_resume(struct intel_display *display) 199 { 200 struct drm_framebuffer *drm_fb; 201 202 if (!HAS_DISPLAY(display)) 203 return; 204 205 mutex_lock(&display->drm->mode_config.fb_lock); 206 drm_for_each_fb(drm_fb, display->drm) { 207 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); 208 209 if (fb->dpt_vm) 210 i915_ggtt_resume_vm(fb->dpt_vm, true); 211 } 212 mutex_unlock(&display->drm->mode_config.fb_lock); 213 } 214 215 /** 216 * intel_dpt_suspend - suspend the memory mapping for all DPT FBs during system suspend 217 * @display: display device instance 218 * 219 * Suspend the memory mapping during system suspend for all framebuffers which 220 * are mapped to HW via a GGTT->DPT page table. 221 * 222 * This function must be called before the mappings in GGTT are suspended calling 223 * i915_ggtt_suspend(). 224 */ 225 void intel_dpt_suspend(struct intel_display *display) 226 { 227 struct drm_framebuffer *drm_fb; 228 229 if (!HAS_DISPLAY(display)) 230 return; 231 232 mutex_lock(&display->drm->mode_config.fb_lock); 233 234 drm_for_each_fb(drm_fb, display->drm) { 235 struct intel_framebuffer *fb = to_intel_framebuffer(drm_fb); 236 237 if (fb->dpt_vm) 238 i915_ggtt_suspend_vm(fb->dpt_vm, true); 239 } 240 241 mutex_unlock(&display->drm->mode_config.fb_lock); 242 } 243 244 struct i915_address_space * 245 intel_dpt_create(struct intel_framebuffer *fb) 246 { 247 struct drm_gem_object *obj = intel_fb_bo(&fb->base); 248 struct drm_i915_private *i915 = to_i915(obj->dev); 249 struct drm_i915_gem_object *dpt_obj; 250 struct i915_address_space *vm; 251 struct i915_dpt *dpt; 252 size_t size; 253 int ret; 254 255 if (intel_fb_needs_pot_stride_remap(fb)) 256 size = intel_remapped_info_size(&fb->remapped_view.gtt.remapped); 257 else 258 size = DIV_ROUND_UP_ULL(obj->size, I915_GTT_PAGE_SIZE); 259 260 size = round_up(size * sizeof(gen8_pte_t), I915_GTT_PAGE_SIZE); 261 262 dpt_obj = i915_gem_object_create_lmem(i915, size, I915_BO_ALLOC_CONTIGUOUS); 263 if (IS_ERR(dpt_obj) && i915_ggtt_has_aperture(to_gt(i915)->ggtt)) 264 dpt_obj = i915_gem_object_create_stolen(i915, size); 265 if (IS_ERR(dpt_obj) && !HAS_LMEM(i915)) { 266 drm_dbg_kms(&i915->drm, "Allocating dpt from smem\n"); 267 dpt_obj = i915_gem_object_create_shmem(i915, size); 268 } 269 if (IS_ERR(dpt_obj)) 270 return ERR_CAST(dpt_obj); 271 272 ret = i915_gem_object_lock_interruptible(dpt_obj, NULL); 273 if (!ret) { 274 ret = i915_gem_object_set_cache_level(dpt_obj, I915_CACHE_NONE); 275 i915_gem_object_unlock(dpt_obj); 276 } 277 if (ret) { 278 i915_gem_object_put(dpt_obj); 279 return ERR_PTR(ret); 280 } 281 282 dpt = kzalloc(sizeof(*dpt), GFP_KERNEL); 283 if (!dpt) { 284 i915_gem_object_put(dpt_obj); 285 return ERR_PTR(-ENOMEM); 286 } 287 288 vm = &dpt->vm; 289 290 vm->gt = to_gt(i915); 291 vm->i915 = i915; 292 vm->dma = i915->drm.dev; 293 vm->total = (size / sizeof(gen8_pte_t)) * I915_GTT_PAGE_SIZE; 294 vm->is_dpt = true; 295 296 i915_address_space_init(vm, VM_CLASS_DPT); 297 298 vm->insert_page = dpt_insert_page; 299 vm->clear_range = dpt_clear_range; 300 vm->insert_entries = dpt_insert_entries; 301 vm->cleanup = dpt_cleanup; 302 303 vm->vma_ops.bind_vma = dpt_bind_vma; 304 vm->vma_ops.unbind_vma = dpt_unbind_vma; 305 306 vm->pte_encode = vm->gt->ggtt->vm.pte_encode; 307 308 dpt->obj = dpt_obj; 309 dpt->obj->is_dpt = true; 310 311 return &dpt->vm; 312 } 313 314 void intel_dpt_destroy(struct i915_address_space *vm) 315 { 316 struct i915_dpt *dpt = i915_vm_to_dpt(vm); 317 318 dpt->obj->is_dpt = false; 319 i915_vm_put(&dpt->vm); 320 } 321 322 u64 intel_dpt_offset(struct i915_vma *dpt_vma) 323 { 324 return dpt_vma->node.start; 325 } 326