1 // SPDX-License-Identifier: MIT 2 /* 3 * Copyright © 2022 Intel Corporation 4 */ 5 6 #include "xe_bo_evict.h" 7 8 #include "xe_bo.h" 9 #include "xe_device.h" 10 #include "xe_ggtt.h" 11 #include "xe_tile.h" 12 13 typedef int (*xe_pinned_fn)(struct xe_bo *bo); 14 15 static int xe_bo_apply_to_pinned(struct xe_device *xe, 16 struct list_head *pinned_list, 17 struct list_head *new_list, 18 const xe_pinned_fn pinned_fn) 19 { 20 LIST_HEAD(still_in_list); 21 struct xe_bo *bo; 22 int ret = 0; 23 24 spin_lock(&xe->pinned.lock); 25 while (!ret) { 26 bo = list_first_entry_or_null(pinned_list, typeof(*bo), 27 pinned_link); 28 if (!bo) 29 break; 30 xe_bo_get(bo); 31 list_move_tail(&bo->pinned_link, &still_in_list); 32 spin_unlock(&xe->pinned.lock); 33 34 ret = pinned_fn(bo); 35 if (ret && pinned_list != new_list) { 36 spin_lock(&xe->pinned.lock); 37 /* 38 * We might no longer be pinned, since PM notifier can 39 * call this. If the pinned link is now empty, keep it 40 * that way. 41 */ 42 if (!list_empty(&bo->pinned_link)) 43 list_move(&bo->pinned_link, pinned_list); 44 spin_unlock(&xe->pinned.lock); 45 } 46 xe_bo_put(bo); 47 spin_lock(&xe->pinned.lock); 48 } 49 list_splice_tail(&still_in_list, new_list); 50 spin_unlock(&xe->pinned.lock); 51 52 return ret; 53 } 54 55 /** 56 * xe_bo_notifier_prepare_all_pinned() - Pre-allocate the backing pages for all 57 * pinned VRAM objects which need to be saved. 58 * @xe: xe device 59 * 60 * Should be called from PM notifier when preparing for s3/s4. 61 * 62 * Return: 0 on success, negative error code on error. 63 */ 64 int xe_bo_notifier_prepare_all_pinned(struct xe_device *xe) 65 { 66 int ret; 67 68 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 69 &xe->pinned.early.kernel_bo_present, 70 xe_bo_notifier_prepare_pinned); 71 if (!ret) 72 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 73 &xe->pinned.late.kernel_bo_present, 74 xe_bo_notifier_prepare_pinned); 75 76 return ret; 77 } 78 79 /** 80 * xe_bo_notifier_unprepare_all_pinned() - Remove the backing pages for all 81 * pinned VRAM objects which have been restored. 82 * @xe: xe device 83 * 84 * Should be called from PM notifier after exiting s3/s4 (either on success or 85 * failure). 86 */ 87 void xe_bo_notifier_unprepare_all_pinned(struct xe_device *xe) 88 { 89 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 90 &xe->pinned.early.kernel_bo_present, 91 xe_bo_notifier_unprepare_pinned); 92 93 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 94 &xe->pinned.late.kernel_bo_present, 95 xe_bo_notifier_unprepare_pinned); 96 } 97 98 /** 99 * xe_bo_evict_all_user - evict all non-pinned user BOs from VRAM 100 * @xe: xe device 101 * 102 * Evict non-pinned user BOs (via GPU). 103 * 104 * Evict == move VRAM BOs to temporary (typically system) memory. 105 */ 106 int xe_bo_evict_all_user(struct xe_device *xe) 107 { 108 struct ttm_device *bdev = &xe->ttm; 109 u32 mem_type; 110 int ret; 111 112 /* User memory */ 113 for (mem_type = XE_PL_TT; mem_type <= XE_PL_VRAM1; ++mem_type) { 114 struct ttm_resource_manager *man = 115 ttm_manager_type(bdev, mem_type); 116 117 /* 118 * On igpu platforms with flat CCS we need to ensure we save and restore any CCS 119 * state since this state lives inside graphics stolen memory which doesn't survive 120 * hibernation. 121 * 122 * This can be further improved by only evicting objects that we know have actually 123 * used a compression enabled PAT index. 124 */ 125 if (mem_type == XE_PL_TT && (IS_DGFX(xe) || !xe_device_has_flat_ccs(xe))) 126 continue; 127 128 if (man) { 129 ret = ttm_resource_manager_evict_all(bdev, man); 130 if (ret) 131 return ret; 132 } 133 } 134 135 return 0; 136 } 137 138 /** 139 * xe_bo_evict_all - evict all BOs from VRAM 140 * @xe: xe device 141 * 142 * Evict non-pinned user BOs first (via GPU), evict pinned external BOs next 143 * (via GPU), wait for evictions, and finally evict pinned kernel BOs via CPU. 144 * All eviction magic done via TTM calls. 145 * 146 * Evict == move VRAM BOs to temporary (typically system) memory. 147 * 148 * This function should be called before the device goes into a suspend state 149 * where the VRAM loses power. 150 */ 151 int xe_bo_evict_all(struct xe_device *xe) 152 { 153 struct xe_tile *tile; 154 u8 id; 155 int ret; 156 157 ret = xe_bo_evict_all_user(xe); 158 if (ret) 159 return ret; 160 161 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 162 &xe->pinned.late.evicted, xe_bo_evict_pinned); 163 164 if (!ret) 165 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 166 &xe->pinned.late.evicted, xe_bo_evict_pinned); 167 168 /* 169 * Wait for all user BO to be evicted as those evictions depend on the 170 * memory moved below. 171 */ 172 for_each_tile(tile, xe, id) 173 xe_tile_migrate_wait(tile); 174 175 if (ret) 176 return ret; 177 178 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 179 &xe->pinned.early.evicted, 180 xe_bo_evict_pinned); 181 } 182 183 static int xe_bo_restore_and_map_ggtt(struct xe_bo *bo) 184 { 185 struct xe_device *xe = xe_bo_device(bo); 186 int ret; 187 188 ret = xe_bo_restore_pinned(bo); 189 if (ret) 190 return ret; 191 192 if (bo->flags & XE_BO_FLAG_GGTT) { 193 struct xe_tile *tile; 194 u8 id; 195 196 for_each_tile(tile, xe_bo_device(bo), id) { 197 if (tile != bo->tile && !(bo->flags & XE_BO_FLAG_GGTTx(tile))) 198 continue; 199 200 mutex_lock(&tile->mem.ggtt->lock); 201 xe_ggtt_map_bo(tile->mem.ggtt, bo); 202 mutex_unlock(&tile->mem.ggtt->lock); 203 } 204 } 205 206 /* 207 * We expect validate to trigger a move VRAM and our move code 208 * should setup the iosys map. 209 */ 210 xe_assert(xe, !(bo->flags & XE_BO_FLAG_PINNED_LATE_RESTORE) || 211 !iosys_map_is_null(&bo->vmap)); 212 213 return 0; 214 } 215 216 /** 217 * xe_bo_restore_early - restore early phase kernel BOs to VRAM 218 * 219 * @xe: xe device 220 * 221 * Move kernel BOs from temporary (typically system) memory to VRAM via CPU. All 222 * moves done via TTM calls. 223 * 224 * This function should be called early, before trying to init the GT, on device 225 * resume. 226 */ 227 int xe_bo_restore_early(struct xe_device *xe) 228 { 229 return xe_bo_apply_to_pinned(xe, &xe->pinned.early.evicted, 230 &xe->pinned.early.kernel_bo_present, 231 xe_bo_restore_and_map_ggtt); 232 } 233 234 /** 235 * xe_bo_restore_late - restore pinned late phase BOs 236 * 237 * @xe: xe device 238 * 239 * Move pinned user and kernel BOs which can use blitter from temporary 240 * (typically system) memory to VRAM. All moves done via TTM calls. 241 * 242 * This function should be called late, after GT init, on device resume. 243 */ 244 int xe_bo_restore_late(struct xe_device *xe) 245 { 246 struct xe_tile *tile; 247 int ret, id; 248 249 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.evicted, 250 &xe->pinned.late.kernel_bo_present, 251 xe_bo_restore_and_map_ggtt); 252 253 for_each_tile(tile, xe, id) 254 xe_tile_migrate_wait(tile); 255 256 if (ret) 257 return ret; 258 259 if (!IS_DGFX(xe)) 260 return 0; 261 262 /* Pinned user memory in VRAM should be validated on resume */ 263 ret = xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 264 &xe->pinned.late.external, 265 xe_bo_restore_pinned); 266 267 /* Wait for restore to complete */ 268 for_each_tile(tile, xe, id) 269 xe_tile_migrate_wait(tile); 270 271 return ret; 272 } 273 274 static void xe_bo_pci_dev_remove_pinned(struct xe_device *xe) 275 { 276 struct xe_tile *tile; 277 unsigned int id; 278 279 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.external, 280 &xe->pinned.late.external, 281 xe_bo_dma_unmap_pinned); 282 for_each_tile(tile, xe, id) 283 xe_tile_migrate_wait(tile); 284 } 285 286 /** 287 * xe_bo_pci_dev_remove_all() - Handle bos when the pci_device is about to be removed 288 * @xe: The xe device. 289 * 290 * On pci_device removal we need to drop all dma mappings and move 291 * the data of exported bos out to system. This includes SVM bos and 292 * exported dma-buf bos. This is done by evicting all bos, but 293 * the evict placement in xe_evict_flags() is chosen such that all 294 * bos except those mentioned are purged, and thus their memory 295 * is released. 296 * 297 * For pinned bos, we're unmapping dma. 298 */ 299 void xe_bo_pci_dev_remove_all(struct xe_device *xe) 300 { 301 unsigned int mem_type; 302 303 /* 304 * Move pagemap bos and exported dma-buf to system, and 305 * purge everything else. 306 */ 307 for (mem_type = XE_PL_VRAM1; mem_type >= XE_PL_TT; --mem_type) { 308 struct ttm_resource_manager *man = 309 ttm_manager_type(&xe->ttm, mem_type); 310 311 if (man) { 312 int ret = ttm_resource_manager_evict_all(&xe->ttm, man); 313 314 drm_WARN_ON(&xe->drm, ret); 315 } 316 } 317 318 xe_bo_pci_dev_remove_pinned(xe); 319 } 320 321 static void xe_bo_pinned_fini(void *arg) 322 { 323 struct xe_device *xe = arg; 324 325 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.late.kernel_bo_present, 326 &xe->pinned.late.kernel_bo_present, 327 xe_bo_dma_unmap_pinned); 328 (void)xe_bo_apply_to_pinned(xe, &xe->pinned.early.kernel_bo_present, 329 &xe->pinned.early.kernel_bo_present, 330 xe_bo_dma_unmap_pinned); 331 } 332 333 /** 334 * xe_bo_pinned_init() - Initialize pinned bo tracking 335 * @xe: The xe device. 336 * 337 * Initializes the lists and locks required for pinned bo 338 * tracking and registers a callback to dma-unmap 339 * any remaining pinned bos on pci device removal. 340 * 341 * Return: %0 on success, negative error code on error. 342 */ 343 int xe_bo_pinned_init(struct xe_device *xe) 344 { 345 spin_lock_init(&xe->pinned.lock); 346 INIT_LIST_HEAD(&xe->pinned.early.kernel_bo_present); 347 INIT_LIST_HEAD(&xe->pinned.early.evicted); 348 INIT_LIST_HEAD(&xe->pinned.late.kernel_bo_present); 349 INIT_LIST_HEAD(&xe->pinned.late.evicted); 350 INIT_LIST_HEAD(&xe->pinned.late.external); 351 352 return devm_add_action_or_reset(xe->drm.dev, xe_bo_pinned_fini, xe); 353 } 354