1 /* 2 * Copyright 2020 Advanced Micro Devices, Inc. 3 * 4 * Permission is hereby granted, free of charge, to any person obtaining a 5 * copy of this software and associated documentation files (the "Software"), 6 * to deal in the Software without restriction, including without limitation 7 * the rights to use, copy, modify, merge, publish, distribute, sublicense, 8 * and/or sell copies of the Software, and to permit persons to whom the 9 * Software is furnished to do so, subject to the following conditions: 10 * 11 * The above copyright notice and this permission notice shall be included in 12 * all copies or substantial portions of the Software. 13 * 14 * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR 15 * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, 16 * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL 17 * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR 18 * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, 19 * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR 20 * OTHER DEALINGS IN THE SOFTWARE. 21 * 22 * Authors: Christian König 23 */ 24 25 #ifndef _TTM_RESOURCE_H_ 26 #define _TTM_RESOURCE_H_ 27 28 #include <linux/types.h> 29 #include <linux/list.h> 30 #include <linux/mutex.h> 31 #include <linux/iosys-map.h> 32 #include <linux/dma-fence.h> 33 34 #include <drm/drm_print.h> 35 #include <drm/ttm/ttm_caching.h> 36 #include <drm/ttm/ttm_kmap_iter.h> 37 38 #define TTM_MAX_BO_PRIORITY 4U 39 #define TTM_NUM_MEM_TYPES 8 40 41 struct dmem_cgroup_device; 42 struct ttm_device; 43 struct ttm_resource_manager; 44 struct ttm_resource; 45 struct ttm_place; 46 struct ttm_buffer_object; 47 struct ttm_placement; 48 struct iosys_map; 49 struct io_mapping; 50 struct sg_table; 51 struct scatterlist; 52 53 /** 54 * enum ttm_lru_item_type - enumerate ttm_lru_item subclasses 55 */ 56 enum ttm_lru_item_type { 57 /** @TTM_LRU_RESOURCE: The resource subclass */ 58 TTM_LRU_RESOURCE, 59 /** @TTM_LRU_HITCH: The iterator hitch subclass */ 60 TTM_LRU_HITCH 61 }; 62 63 /** 64 * struct ttm_lru_item - The TTM lru list node base class 65 * @link: The list link 66 * @type: The subclass type 67 */ 68 struct ttm_lru_item { 69 struct list_head link; 70 enum ttm_lru_item_type type; 71 }; 72 73 /** 74 * ttm_lru_item_init() - initialize a struct ttm_lru_item 75 * @item: The item to initialize 76 * @type: The subclass type 77 */ 78 static inline void ttm_lru_item_init(struct ttm_lru_item *item, 79 enum ttm_lru_item_type type) 80 { 81 item->type = type; 82 INIT_LIST_HEAD(&item->link); 83 } 84 85 static inline bool ttm_lru_item_is_res(const struct ttm_lru_item *item) 86 { 87 return item->type == TTM_LRU_RESOURCE; 88 } 89 90 struct ttm_resource_manager_func { 91 /** 92 * struct ttm_resource_manager_func member alloc 93 * 94 * @man: Pointer to a memory type manager. 95 * @bo: Pointer to the buffer object we're allocating space for. 96 * @place: Placement details. 97 * @res: Resulting pointer to the ttm_resource. 98 * 99 * This function should allocate space in the memory type managed 100 * by @man. Placement details if applicable are given by @place. If 101 * successful, a filled in ttm_resource object should be returned in 102 * @res. @res::start should be set to a value identifying the beginning 103 * of the range allocated, and the function should return zero. 104 * If the manager can't fulfill the request -ENOSPC should be returned. 105 * If a system error occurred, preventing the request to be fulfilled, 106 * the function should return a negative error code. 107 * 108 * This function may not be called from within atomic context and needs 109 * to take care of its own locking to protect any data structures 110 * managing the space. 111 */ 112 int (*alloc)(struct ttm_resource_manager *man, 113 struct ttm_buffer_object *bo, 114 const struct ttm_place *place, 115 struct ttm_resource **res); 116 117 /** 118 * struct ttm_resource_manager_func member free 119 * 120 * @man: Pointer to a memory type manager. 121 * @res: Pointer to a struct ttm_resource to be freed. 122 * 123 * This function frees memory type resources previously allocated. 124 * May not be called from within atomic context. 125 */ 126 void (*free)(struct ttm_resource_manager *man, 127 struct ttm_resource *res); 128 129 /** 130 * struct ttm_resource_manager_func member intersects 131 * 132 * @man: Pointer to a memory type manager. 133 * @res: Pointer to a struct ttm_resource to be checked. 134 * @place: Placement to check against. 135 * @size: Size of the check. 136 * 137 * Test if @res intersects with @place + @size. Used to judge if 138 * evictions are valueable or not. 139 */ 140 bool (*intersects)(struct ttm_resource_manager *man, 141 struct ttm_resource *res, 142 const struct ttm_place *place, 143 size_t size); 144 145 /** 146 * struct ttm_resource_manager_func member compatible 147 * 148 * @man: Pointer to a memory type manager. 149 * @res: Pointer to a struct ttm_resource to be checked. 150 * @place: Placement to check against. 151 * @size: Size of the check. 152 * 153 * Test if @res compatible with @place + @size. Used to check of 154 * the need to move the backing store or not. 155 */ 156 bool (*compatible)(struct ttm_resource_manager *man, 157 struct ttm_resource *res, 158 const struct ttm_place *place, 159 size_t size); 160 161 /** 162 * struct ttm_resource_manager_func member debug 163 * 164 * @man: Pointer to a memory type manager. 165 * @printer: Prefix to be used in printout to identify the caller. 166 * 167 * This function is called to print out the state of the memory 168 * type manager to aid debugging of out-of-memory conditions. 169 * It may not be called from within atomic context. 170 */ 171 void (*debug)(struct ttm_resource_manager *man, 172 struct drm_printer *printer); 173 }; 174 175 /** 176 * struct ttm_resource_manager 177 * 178 * @use_type: The memory type is enabled. 179 * @use_tt: If a TT object should be used for the backing store. 180 * @size: Size of the managed region. 181 * @bdev: ttm device this manager belongs to 182 * @func: structure pointer implementing the range manager. See above 183 * @move_lock: lock for move fence 184 * @move: The fence of the last pipelined move operation. 185 * @lru: The lru list for this memory type. 186 * 187 * This structure is used to identify and manage memory types for a device. 188 */ 189 struct ttm_resource_manager { 190 /* 191 * No protection. Constant from start. 192 */ 193 bool use_type; 194 bool use_tt; 195 struct ttm_device *bdev; 196 uint64_t size; 197 const struct ttm_resource_manager_func *func; 198 spinlock_t move_lock; 199 200 /* 201 * Protected by @move_lock. 202 */ 203 struct dma_fence *move; 204 205 /* 206 * Protected by the bdev->lru_lock. 207 */ 208 struct list_head lru[TTM_MAX_BO_PRIORITY]; 209 210 /** 211 * @usage: How much of the resources are used, protected by the 212 * bdev->lru_lock. 213 */ 214 uint64_t usage; 215 216 /** 217 * @cg: &dmem_cgroup_region used for memory accounting, if not NULL. 218 */ 219 struct dmem_cgroup_region *cg; 220 }; 221 222 /** 223 * struct ttm_bus_placement 224 * 225 * @addr: mapped virtual address 226 * @offset: physical addr 227 * @is_iomem: is this io memory ? 228 * @caching: See enum ttm_caching 229 * 230 * Structure indicating the bus placement of an object. 231 */ 232 struct ttm_bus_placement { 233 void *addr; 234 phys_addr_t offset; 235 bool is_iomem; 236 enum ttm_caching caching; 237 }; 238 239 /** 240 * struct ttm_resource 241 * 242 * @start: Start of the allocation. 243 * @size: Actual size of resource in bytes. 244 * @mem_type: Resource type of the allocation. 245 * @placement: Placement flags. 246 * @bus: Placement on io bus accessible to the CPU 247 * @bo: weak reference to the BO, protected by ttm_device::lru_lock 248 * @css: cgroup state this resource is charged to 249 * 250 * Structure indicating the placement and space resources used by a 251 * buffer object. 252 */ 253 struct ttm_resource { 254 unsigned long start; 255 size_t size; 256 uint32_t mem_type; 257 uint32_t placement; 258 struct ttm_bus_placement bus; 259 struct ttm_buffer_object *bo; 260 261 struct dmem_cgroup_pool_state *css; 262 263 /** 264 * @lru: Least recently used list, see &ttm_resource_manager.lru 265 */ 266 struct ttm_lru_item lru; 267 }; 268 269 /** 270 * ttm_lru_item_to_res() - Downcast a struct ttm_lru_item to a struct ttm_resource 271 * @item: The struct ttm_lru_item to downcast 272 * 273 * Return: Pointer to the embedding struct ttm_resource 274 */ 275 static inline struct ttm_resource * 276 ttm_lru_item_to_res(struct ttm_lru_item *item) 277 { 278 return container_of(item, struct ttm_resource, lru); 279 } 280 281 /** 282 * struct ttm_lru_bulk_move_pos 283 * 284 * @first: first res in the bulk move range 285 * @last: last res in the bulk move range 286 * 287 * Range of resources for a lru bulk move. 288 */ 289 struct ttm_lru_bulk_move_pos { 290 struct ttm_resource *first; 291 struct ttm_resource *last; 292 }; 293 294 /** 295 * struct ttm_lru_bulk_move 296 * @pos: first/last lru entry for resources in the each domain/priority 297 * @cursor_list: The list of cursors currently traversing any of 298 * the sublists of @pos. Protected by the ttm device's lru_lock. 299 * 300 * Container for the current bulk move state. Should be used with 301 * ttm_lru_bulk_move_init() and ttm_bo_set_bulk_move(). 302 * All BOs in a bulk_move structure need to share the same reservation object to 303 * ensure that the bulk as a whole is locked for eviction even if only one BO of 304 * the bulk is evicted. 305 */ 306 struct ttm_lru_bulk_move { 307 struct ttm_lru_bulk_move_pos pos[TTM_NUM_MEM_TYPES][TTM_MAX_BO_PRIORITY]; 308 struct list_head cursor_list; 309 }; 310 311 /** 312 * struct ttm_resource_cursor 313 * @man: The resource manager currently being iterated over 314 * @hitch: A hitch list node inserted before the next resource 315 * to iterate over. 316 * @bulk_link: A list link for the list of cursors traversing the 317 * bulk sublist of @bulk. Protected by the ttm device's lru_lock. 318 * @bulk: Pointer to struct ttm_lru_bulk_move whose subrange @hitch is 319 * inserted to. NULL if none. Never dereference this pointer since 320 * the struct ttm_lru_bulk_move object pointed to might have been 321 * freed. The pointer is only for comparison. 322 * @mem_type: The memory type of the LRU list being traversed. 323 * This field is valid iff @bulk != NULL. 324 * @priority: the current priority 325 * 326 * Cursor to iterate over the resources in a manager. 327 */ 328 struct ttm_resource_cursor { 329 struct ttm_resource_manager *man; 330 struct ttm_lru_item hitch; 331 struct list_head bulk_link; 332 struct ttm_lru_bulk_move *bulk; 333 unsigned int mem_type; 334 unsigned int priority; 335 }; 336 337 void ttm_resource_cursor_init(struct ttm_resource_cursor *cursor, 338 struct ttm_resource_manager *man); 339 340 void ttm_resource_cursor_fini(struct ttm_resource_cursor *cursor); 341 342 /** 343 * struct ttm_kmap_iter_iomap - Specialization for a struct io_mapping + 344 * struct sg_table backed struct ttm_resource. 345 * @base: Embedded struct ttm_kmap_iter providing the usage interface. 346 * @iomap: struct io_mapping representing the underlying linear io_memory. 347 * @st: sg_table into @iomap, representing the memory of the struct ttm_resource. 348 * @start: Offset that needs to be subtracted from @st to make 349 * sg_dma_address(st->sgl) - @start == 0 for @iomap start. 350 * @cache: Scatterlist traversal cache for fast lookups. 351 * @cache.sg: Pointer to the currently cached scatterlist segment. 352 * @cache.i: First index of @sg. PAGE_SIZE granularity. 353 * @cache.end: Last index + 1 of @sg. PAGE_SIZE granularity. 354 * @cache.offs: First offset into @iomap of @sg. PAGE_SIZE granularity. 355 */ 356 struct ttm_kmap_iter_iomap { 357 struct ttm_kmap_iter base; 358 struct io_mapping *iomap; 359 struct sg_table *st; 360 resource_size_t start; 361 struct { 362 struct scatterlist *sg; 363 pgoff_t i; 364 pgoff_t end; 365 pgoff_t offs; 366 } cache; 367 }; 368 369 /** 370 * struct ttm_kmap_iter_linear_io - Iterator specialization for linear io 371 * @base: The base iterator 372 * @dmap: Points to the starting address of the region 373 * @needs_unmap: Whether we need to unmap on fini 374 */ 375 struct ttm_kmap_iter_linear_io { 376 struct ttm_kmap_iter base; 377 struct iosys_map dmap; 378 bool needs_unmap; 379 }; 380 381 /** 382 * ttm_resource_manager_set_used 383 * 384 * @man: A memory manager object. 385 * @used: usage state to set. 386 * 387 * Set the manager in use flag. If disabled the manager is no longer 388 * used for object placement. 389 */ 390 static inline void 391 ttm_resource_manager_set_used(struct ttm_resource_manager *man, bool used) 392 { 393 int i; 394 395 for (i = 0; i < TTM_MAX_BO_PRIORITY; i++) 396 WARN_ON(!list_empty(&man->lru[i])); 397 man->use_type = used; 398 } 399 400 /** 401 * ttm_resource_manager_used 402 * 403 * @man: Manager to get used state for 404 * 405 * Get the in use flag for a manager. 406 * Returns: 407 * true is used, false if not. 408 */ 409 static inline bool ttm_resource_manager_used(struct ttm_resource_manager *man) 410 { 411 return man->use_type; 412 } 413 414 /** 415 * ttm_resource_manager_cleanup 416 * 417 * @man: A memory manager object. 418 * 419 * Cleanup the move fences from the memory manager object. 420 */ 421 static inline void 422 ttm_resource_manager_cleanup(struct ttm_resource_manager *man) 423 { 424 dma_fence_put(man->move); 425 man->move = NULL; 426 } 427 428 void ttm_lru_bulk_move_init(struct ttm_lru_bulk_move *bulk); 429 void ttm_lru_bulk_move_tail(struct ttm_lru_bulk_move *bulk); 430 void ttm_lru_bulk_move_fini(struct ttm_device *bdev, 431 struct ttm_lru_bulk_move *bulk); 432 433 void ttm_resource_add_bulk_move(struct ttm_resource *res, 434 struct ttm_buffer_object *bo); 435 void ttm_resource_del_bulk_move(struct ttm_resource *res, 436 struct ttm_buffer_object *bo); 437 void ttm_resource_move_to_lru_tail(struct ttm_resource *res); 438 439 void ttm_resource_init(struct ttm_buffer_object *bo, 440 const struct ttm_place *place, 441 struct ttm_resource *res); 442 void ttm_resource_fini(struct ttm_resource_manager *man, 443 struct ttm_resource *res); 444 445 int ttm_resource_alloc(struct ttm_buffer_object *bo, 446 const struct ttm_place *place, 447 struct ttm_resource **res, 448 struct dmem_cgroup_pool_state **ret_limit_pool); 449 void ttm_resource_free(struct ttm_buffer_object *bo, struct ttm_resource **res); 450 bool ttm_resource_intersects(struct ttm_device *bdev, 451 struct ttm_resource *res, 452 const struct ttm_place *place, 453 size_t size); 454 bool ttm_resource_compatible(struct ttm_resource *res, 455 struct ttm_placement *placement, 456 bool evicting); 457 void ttm_resource_set_bo(struct ttm_resource *res, 458 struct ttm_buffer_object *bo); 459 460 void ttm_resource_manager_init(struct ttm_resource_manager *man, 461 struct ttm_device *bdev, 462 uint64_t size); 463 464 int ttm_resource_manager_evict_all(struct ttm_device *bdev, 465 struct ttm_resource_manager *man); 466 467 uint64_t ttm_resource_manager_usage(struct ttm_resource_manager *man); 468 void ttm_resource_manager_debug(struct ttm_resource_manager *man, 469 struct drm_printer *p); 470 471 struct ttm_resource * 472 ttm_resource_manager_first(struct ttm_resource_cursor *cursor); 473 struct ttm_resource * 474 ttm_resource_manager_next(struct ttm_resource_cursor *cursor); 475 476 struct ttm_resource * 477 ttm_lru_first_res_or_null(struct list_head *head); 478 479 /** 480 * ttm_resource_manager_for_each_res - iterate over all resources 481 * @cursor: struct ttm_resource_cursor for the current position 482 * @res: the current resource 483 * 484 * Iterate over all the evictable resources in a resource manager. 485 */ 486 #define ttm_resource_manager_for_each_res(cursor, res) \ 487 for (res = ttm_resource_manager_first(cursor); res; \ 488 res = ttm_resource_manager_next(cursor)) 489 490 struct ttm_kmap_iter * 491 ttm_kmap_iter_iomap_init(struct ttm_kmap_iter_iomap *iter_io, 492 struct io_mapping *iomap, 493 struct sg_table *st, 494 resource_size_t start); 495 496 struct ttm_kmap_iter_linear_io; 497 498 struct ttm_kmap_iter * 499 ttm_kmap_iter_linear_io_init(struct ttm_kmap_iter_linear_io *iter_io, 500 struct ttm_device *bdev, 501 struct ttm_resource *mem); 502 503 void ttm_kmap_iter_linear_io_fini(struct ttm_kmap_iter_linear_io *iter_io, 504 struct ttm_device *bdev, 505 struct ttm_resource *mem); 506 507 void ttm_resource_manager_create_debugfs(struct ttm_resource_manager *man, 508 struct dentry * parent, 509 const char *name); 510 #endif 511