1 /* SPDX-License-Identifier: MIT */ 2 /* 3 * Copyright © 2022-2023 Intel Corporation 4 */ 5 6 #ifndef _XE_DEVICE_TYPES_H_ 7 #define _XE_DEVICE_TYPES_H_ 8 9 #include <linux/pci.h> 10 11 #include <drm/drm_device.h> 12 #include <drm/drm_file.h> 13 #include <drm/drm_pagemap.h> 14 #include <drm/ttm/ttm_device.h> 15 16 #include "xe_devcoredump_types.h" 17 #include "xe_heci_gsc.h" 18 #include "xe_lmtt_types.h" 19 #include "xe_memirq_types.h" 20 #include "xe_oa_types.h" 21 #include "xe_platform_types.h" 22 #include "xe_pmu_types.h" 23 #include "xe_pt_types.h" 24 #include "xe_sriov_types.h" 25 #include "xe_step_types.h" 26 #include "xe_survivability_mode_types.h" 27 #include "xe_ttm_vram_mgr_types.h" 28 29 #if IS_ENABLED(CONFIG_DRM_XE_DEBUG) 30 #define TEST_VM_OPS_ERROR 31 #endif 32 33 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY) 34 #include "soc/intel_pch.h" 35 #include "intel_display_core.h" 36 #include "intel_display_device.h" 37 #endif 38 39 struct xe_ggtt; 40 struct xe_pat_ops; 41 struct xe_pxp; 42 43 #define XE_BO_INVALID_OFFSET LONG_MAX 44 45 #define GRAPHICS_VER(xe) ((xe)->info.graphics_verx100 / 100) 46 #define MEDIA_VER(xe) ((xe)->info.media_verx100 / 100) 47 #define GRAPHICS_VERx100(xe) ((xe)->info.graphics_verx100) 48 #define MEDIA_VERx100(xe) ((xe)->info.media_verx100) 49 #define IS_DGFX(xe) ((xe)->info.is_dgfx) 50 51 #define XE_VRAM_FLAGS_NEED64K BIT(0) 52 53 #define XE_GT0 0 54 #define XE_GT1 1 55 #define XE_MAX_TILES_PER_DEVICE (XE_GT1 + 1) 56 57 #define XE_MAX_ASID (BIT(20)) 58 59 #define IS_PLATFORM_STEP(_xe, _platform, min_step, max_step) \ 60 ((_xe)->info.platform == (_platform) && \ 61 (_xe)->info.step.graphics >= (min_step) && \ 62 (_xe)->info.step.graphics < (max_step)) 63 #define IS_SUBPLATFORM_STEP(_xe, _platform, sub, min_step, max_step) \ 64 ((_xe)->info.platform == (_platform) && \ 65 (_xe)->info.subplatform == (sub) && \ 66 (_xe)->info.step.graphics >= (min_step) && \ 67 (_xe)->info.step.graphics < (max_step)) 68 69 #define tile_to_xe(tile__) \ 70 _Generic(tile__, \ 71 const struct xe_tile * : (const struct xe_device *)((tile__)->xe), \ 72 struct xe_tile * : (tile__)->xe) 73 74 /** 75 * struct xe_vram_region - memory region structure 76 * This is used to describe a memory region in xe 77 * device, such as HBM memory or CXL extension memory. 78 */ 79 struct xe_vram_region { 80 /** @io_start: IO start address of this VRAM instance */ 81 resource_size_t io_start; 82 /** 83 * @io_size: IO size of this VRAM instance 84 * 85 * This represents how much of this VRAM we can access 86 * via the CPU through the VRAM BAR. This can be smaller 87 * than @usable_size, in which case only part of VRAM is CPU 88 * accessible (typically the first 256M). This 89 * configuration is known as small-bar. 90 */ 91 resource_size_t io_size; 92 /** @dpa_base: This memory regions's DPA (device physical address) base */ 93 resource_size_t dpa_base; 94 /** 95 * @usable_size: usable size of VRAM 96 * 97 * Usable size of VRAM excluding reserved portions 98 * (e.g stolen mem) 99 */ 100 resource_size_t usable_size; 101 /** 102 * @actual_physical_size: Actual VRAM size 103 * 104 * Actual VRAM size including reserved portions 105 * (e.g stolen mem) 106 */ 107 resource_size_t actual_physical_size; 108 /** @mapping: pointer to VRAM mappable space */ 109 void __iomem *mapping; 110 /** @pagemap: Used to remap device memory as ZONE_DEVICE */ 111 struct dev_pagemap pagemap; 112 /** 113 * @dpagemap: The struct drm_pagemap of the ZONE_DEVICE memory 114 * pages of this tile. 115 */ 116 struct drm_pagemap dpagemap; 117 /** 118 * @hpa_base: base host physical address 119 * 120 * This is generated when remap device memory as ZONE_DEVICE 121 */ 122 resource_size_t hpa_base; 123 /** @ttm: VRAM TTM manager */ 124 struct xe_ttm_vram_mgr ttm; 125 }; 126 127 /** 128 * struct xe_mmio - register mmio structure 129 * 130 * Represents an MMIO region that the CPU may use to access registers. A 131 * region may share its IO map with other regions (e.g., all GTs within a 132 * tile share the same map with their parent tile, but represent different 133 * subregions of the overall IO space). 134 */ 135 struct xe_mmio { 136 /** @tile: Backpointer to tile, used for tracing */ 137 struct xe_tile *tile; 138 139 /** @regs: Map used to access registers. */ 140 void __iomem *regs; 141 142 /** 143 * @sriov_vf_gt: Backpointer to GT. 144 * 145 * This pointer is only set for GT MMIO regions and only when running 146 * as an SRIOV VF structure 147 */ 148 struct xe_gt *sriov_vf_gt; 149 150 /** 151 * @regs_size: Length of the register region within the map. 152 * 153 * The size of the iomap set in *regs is generally larger than the 154 * register mmio space since it includes unused regions and/or 155 * non-register regions such as the GGTT PTEs. 156 */ 157 size_t regs_size; 158 159 /** @adj_limit: adjust MMIO address if address is below this value */ 160 u32 adj_limit; 161 162 /** @adj_offset: offset to add to MMIO address when adjusting */ 163 u32 adj_offset; 164 }; 165 166 /** 167 * struct xe_tile - hardware tile structure 168 * 169 * From a driver perspective, a "tile" is effectively a complete GPU, containing 170 * an SGunit, 1-2 GTs, and (for discrete platforms) VRAM. 171 * 172 * Multi-tile platforms effectively bundle multiple GPUs behind a single PCI 173 * device and designate one "root" tile as being responsible for external PCI 174 * communication. PCI BAR0 exposes the GGTT and MMIO register space for each 175 * tile in a stacked layout, and PCI BAR2 exposes the local memory associated 176 * with each tile similarly. Device-wide interrupts can be enabled/disabled 177 * at the root tile, and the MSTR_TILE_INTR register will report which tiles 178 * have interrupts that need servicing. 179 */ 180 struct xe_tile { 181 /** @xe: Backpointer to tile's PCI device */ 182 struct xe_device *xe; 183 184 /** @id: ID of the tile */ 185 u8 id; 186 187 /** 188 * @primary_gt: Primary GT 189 */ 190 struct xe_gt *primary_gt; 191 192 /** 193 * @media_gt: Media GT 194 * 195 * Only present on devices with media version >= 13. 196 */ 197 struct xe_gt *media_gt; 198 199 /** 200 * @mmio: MMIO info for a tile. 201 * 202 * Each tile has its own 16MB space in BAR0, laid out as: 203 * * 0-4MB: registers 204 * * 4MB-8MB: reserved 205 * * 8MB-16MB: global GTT 206 */ 207 struct xe_mmio mmio; 208 209 /** @mem: memory management info for tile */ 210 struct { 211 /** 212 * @mem.vram: VRAM info for tile. 213 * 214 * Although VRAM is associated with a specific tile, it can 215 * still be accessed by all tiles' GTs. 216 */ 217 struct xe_vram_region vram; 218 219 /** @mem.ggtt: Global graphics translation table */ 220 struct xe_ggtt *ggtt; 221 222 /** 223 * @mem.kernel_bb_pool: Pool from which batchbuffers are allocated. 224 * 225 * Media GT shares a pool with its primary GT. 226 */ 227 struct xe_sa_manager *kernel_bb_pool; 228 } mem; 229 230 /** @sriov: tile level virtualization data */ 231 union { 232 struct { 233 /** @sriov.pf.lmtt: Local Memory Translation Table. */ 234 struct xe_lmtt lmtt; 235 } pf; 236 struct { 237 /** @sriov.vf.ggtt_balloon: GGTT regions excluded from use. */ 238 struct xe_ggtt_node *ggtt_balloon[2]; 239 } vf; 240 } sriov; 241 242 /** @memirq: Memory Based Interrupts. */ 243 struct xe_memirq memirq; 244 245 /** @pcode: tile's PCODE */ 246 struct { 247 /** @pcode.lock: protecting tile's PCODE mailbox data */ 248 struct mutex lock; 249 } pcode; 250 251 /** @migrate: Migration helper for vram blits and clearing */ 252 struct xe_migrate *migrate; 253 254 /** @sysfs: sysfs' kobj used by xe_tile_sysfs */ 255 struct kobject *sysfs; 256 }; 257 258 /** 259 * struct xe_device - Top level struct of XE device 260 */ 261 struct xe_device { 262 /** @drm: drm device */ 263 struct drm_device drm; 264 265 /** @devcoredump: device coredump */ 266 struct xe_devcoredump devcoredump; 267 268 /** @info: device info */ 269 struct intel_device_info { 270 /** @info.platform_name: platform name */ 271 const char *platform_name; 272 /** @info.graphics_name: graphics IP name */ 273 const char *graphics_name; 274 /** @info.media_name: media IP name */ 275 const char *media_name; 276 /** @info.graphics_verx100: graphics IP version */ 277 u32 graphics_verx100; 278 /** @info.media_verx100: media IP version */ 279 u32 media_verx100; 280 /** @info.mem_region_mask: mask of valid memory regions */ 281 u32 mem_region_mask; 282 /** @info.platform: XE platform enum */ 283 enum xe_platform platform; 284 /** @info.subplatform: XE subplatform enum */ 285 enum xe_subplatform subplatform; 286 /** @info.devid: device ID */ 287 u16 devid; 288 /** @info.revid: device revision */ 289 u8 revid; 290 /** @info.step: stepping information for each IP */ 291 struct xe_step_info step; 292 /** @info.dma_mask_size: DMA address bits */ 293 u8 dma_mask_size; 294 /** @info.vram_flags: Vram flags */ 295 u8 vram_flags; 296 /** @info.tile_count: Number of tiles */ 297 u8 tile_count; 298 /** @info.gt_count: Total number of GTs for entire device */ 299 u8 gt_count; 300 /** @info.vm_max_level: Max VM level */ 301 u8 vm_max_level; 302 /** @info.va_bits: Maximum bits of a virtual address */ 303 u8 va_bits; 304 305 /* 306 * Keep all flags below alphabetically sorted 307 */ 308 309 /** @info.force_execlist: Forced execlist submission */ 310 u8 force_execlist:1; 311 /** @info.has_asid: Has address space ID */ 312 u8 has_asid:1; 313 /** @info.has_atomic_enable_pte_bit: Device has atomic enable PTE bit */ 314 u8 has_atomic_enable_pte_bit:1; 315 /** @info.has_device_atomics_on_smem: Supports device atomics on SMEM */ 316 u8 has_device_atomics_on_smem:1; 317 /** @info.has_flat_ccs: Whether flat CCS metadata is used */ 318 u8 has_flat_ccs:1; 319 /** @info.has_heci_cscfi: device has heci cscfi */ 320 u8 has_heci_cscfi:1; 321 /** @info.has_heci_gscfi: device has heci gscfi */ 322 u8 has_heci_gscfi:1; 323 /** @info.has_llc: Device has a shared CPU+GPU last level cache */ 324 u8 has_llc:1; 325 /** @info.has_pxp: Device has PXP support */ 326 u8 has_pxp:1; 327 /** @info.has_range_tlb_invalidation: Has range based TLB invalidations */ 328 u8 has_range_tlb_invalidation:1; 329 /** @info.has_sriov: Supports SR-IOV */ 330 u8 has_sriov:1; 331 /** @info.has_usm: Device has unified shared memory support */ 332 u8 has_usm:1; 333 /** @info.has_64bit_timestamp: Device supports 64-bit timestamps */ 334 u8 has_64bit_timestamp:1; 335 /** @info.is_dgfx: is discrete device */ 336 u8 is_dgfx:1; 337 /** 338 * @info.probe_display: Probe display hardware. If set to 339 * false, the driver will behave as if there is no display 340 * hardware present and will not try to read/write to it in any 341 * way. The display hardware, if it exists, will not be 342 * exposed to userspace and will be left untouched in whatever 343 * state the firmware or bootloader left it in. 344 */ 345 u8 probe_display:1; 346 /** @info.skip_guc_pc: Skip GuC based PM feature init */ 347 u8 skip_guc_pc:1; 348 /** @info.skip_mtcfg: skip Multi-Tile configuration from MTCFG register */ 349 u8 skip_mtcfg:1; 350 /** @info.skip_pcode: skip access to PCODE uC */ 351 u8 skip_pcode:1; 352 } info; 353 354 /** @survivability: survivability information for device */ 355 struct xe_survivability survivability; 356 357 /** @irq: device interrupt state */ 358 struct { 359 /** @irq.lock: lock for processing irq's on this device */ 360 spinlock_t lock; 361 362 /** @irq.enabled: interrupts enabled on this device */ 363 atomic_t enabled; 364 365 /** @irq.msix: irq info for platforms that support MSI-X */ 366 struct { 367 /** @irq.msix.nvec: number of MSI-X interrupts */ 368 u16 nvec; 369 /** @irq.msix.indexes: used to allocate MSI-X indexes */ 370 struct xarray indexes; 371 } msix; 372 } irq; 373 374 /** @ttm: ttm device */ 375 struct ttm_device ttm; 376 377 /** @mmio: mmio info for device */ 378 struct { 379 /** @mmio.size: size of MMIO space for device */ 380 size_t size; 381 /** @mmio.regs: pointer to MMIO space for device */ 382 void __iomem *regs; 383 } mmio; 384 385 /** @mem: memory info for device */ 386 struct { 387 /** @mem.vram: VRAM info for device */ 388 struct xe_vram_region vram; 389 /** @mem.sys_mgr: system TTM manager */ 390 struct ttm_resource_manager sys_mgr; 391 /** @mem.sys_mgr: system memory shrinker. */ 392 struct xe_shrinker *shrinker; 393 } mem; 394 395 /** @sriov: device level virtualization data */ 396 struct { 397 /** @sriov.__mode: SR-IOV mode (Don't access directly!) */ 398 enum xe_sriov_mode __mode; 399 400 /** @sriov.pf: PF specific data */ 401 struct xe_device_pf pf; 402 /** @sriov.vf: VF specific data */ 403 struct xe_device_vf vf; 404 405 /** @sriov.wq: workqueue used by the virtualization workers */ 406 struct workqueue_struct *wq; 407 } sriov; 408 409 /** @usm: unified memory state */ 410 struct { 411 /** @usm.asid: convert a ASID to VM */ 412 struct xarray asid_to_vm; 413 /** @usm.next_asid: next ASID, used to cyclical alloc asids */ 414 u32 next_asid; 415 /** @usm.lock: protects UM state */ 416 struct rw_semaphore lock; 417 } usm; 418 419 /** @pinned: pinned BO state */ 420 struct { 421 /** @pinned.lock: protected pinned BO list state */ 422 spinlock_t lock; 423 /** @pinned.kernel_bo_present: pinned kernel BO that are present */ 424 struct list_head kernel_bo_present; 425 /** @pinned.evicted: pinned BO that have been evicted */ 426 struct list_head evicted; 427 /** @pinned.external_vram: pinned external BO in vram*/ 428 struct list_head external_vram; 429 } pinned; 430 431 /** @ufence_wq: user fence wait queue */ 432 wait_queue_head_t ufence_wq; 433 434 /** @preempt_fence_wq: used to serialize preempt fences */ 435 struct workqueue_struct *preempt_fence_wq; 436 437 /** @ordered_wq: used to serialize compute mode resume */ 438 struct workqueue_struct *ordered_wq; 439 440 /** @unordered_wq: used to serialize unordered work, mostly display */ 441 struct workqueue_struct *unordered_wq; 442 443 /** @destroy_wq: used to serialize user destroy work, like queue */ 444 struct workqueue_struct *destroy_wq; 445 446 /** @tiles: device tiles */ 447 struct xe_tile tiles[XE_MAX_TILES_PER_DEVICE]; 448 449 /** 450 * @mem_access: keep track of memory access in the device, possibly 451 * triggering additional actions when they occur. 452 */ 453 struct { 454 /** 455 * @mem_access.vram_userfault: Encapsulate vram_userfault 456 * related stuff 457 */ 458 struct { 459 /** 460 * @mem_access.vram_userfault.lock: Protects access to 461 * @vram_usefault.list Using mutex instead of spinlock 462 * as lock is applied to entire list operation which 463 * may sleep 464 */ 465 struct mutex lock; 466 467 /** 468 * @mem_access.vram_userfault.list: Keep list of userfaulted 469 * vram bo, which require to release their mmap mappings 470 * at runtime suspend path 471 */ 472 struct list_head list; 473 } vram_userfault; 474 } mem_access; 475 476 /** 477 * @pat: Encapsulate PAT related stuff 478 */ 479 struct { 480 /** @pat.ops: Internal operations to abstract platforms */ 481 const struct xe_pat_ops *ops; 482 /** @pat.table: PAT table to program in the HW */ 483 const struct xe_pat_table_entry *table; 484 /** @pat.n_entries: Number of PAT entries */ 485 int n_entries; 486 u32 idx[__XE_CACHE_LEVEL_COUNT]; 487 } pat; 488 489 /** @d3cold: Encapsulate d3cold related stuff */ 490 struct { 491 /** @d3cold.capable: Indicates if root port is d3cold capable */ 492 bool capable; 493 494 /** @d3cold.allowed: Indicates if d3cold is a valid device state */ 495 bool allowed; 496 497 /** 498 * @d3cold.vram_threshold: 499 * 500 * This represents the permissible threshold(in megabytes) 501 * for vram save/restore. d3cold will be disallowed, 502 * when vram_usages is above or equals the threshold value 503 * to avoid the vram save/restore latency. 504 * Default threshold value is 300mb. 505 */ 506 u32 vram_threshold; 507 /** @d3cold.lock: protect vram_threshold */ 508 struct mutex lock; 509 } d3cold; 510 511 /** @pmt: Support the PMT driver callback interface */ 512 struct { 513 /** @pmt.lock: protect access for telemetry data */ 514 struct mutex lock; 515 } pmt; 516 517 /** 518 * @pm_callback_task: Track the active task that is running in either 519 * the runtime_suspend or runtime_resume callbacks. 520 */ 521 struct task_struct *pm_callback_task; 522 523 /** @hwmon: hwmon subsystem integration */ 524 struct xe_hwmon *hwmon; 525 526 /** @heci_gsc: graphics security controller */ 527 struct xe_heci_gsc heci_gsc; 528 529 /** @oa: oa observation subsystem */ 530 struct xe_oa oa; 531 532 /** @pxp: Encapsulate Protected Xe Path support */ 533 struct xe_pxp *pxp; 534 535 /** @needs_flr_on_fini: requests function-reset on fini */ 536 bool needs_flr_on_fini; 537 538 /** @wedged: Struct to control Wedged States and mode */ 539 struct { 540 /** @wedged.flag: Xe device faced a critical error and is now blocked. */ 541 atomic_t flag; 542 /** @wedged.mode: Mode controlled by kernel parameter and debugfs */ 543 int mode; 544 } wedged; 545 546 /** @bo_device: Struct to control async free of BOs */ 547 struct xe_bo_dev { 548 /** @bo_device.async_free: Free worker */ 549 struct work_struct async_free; 550 /** @bo_device.async_list: List of BOs to be freed */ 551 struct llist_head async_list; 552 } bo_device; 553 554 /** @pmu: performance monitoring unit */ 555 struct xe_pmu pmu; 556 557 #ifdef TEST_VM_OPS_ERROR 558 /** 559 * @vm_inject_error_position: inject errors at different places in VM 560 * bind IOCTL based on this value 561 */ 562 u8 vm_inject_error_position; 563 #endif 564 565 /* private: */ 566 567 #if IS_ENABLED(CONFIG_DRM_XE_DISPLAY) 568 /* 569 * Any fields below this point are the ones used by display. 570 * They are temporarily added here so xe_device can be desguised as 571 * drm_i915_private during build. After cleanup these should go away, 572 * migrating to the right sub-structs 573 */ 574 struct intel_display display; 575 enum intel_pch pch_type; 576 577 struct dram_info { 578 bool wm_lv_0_adjust_needed; 579 u8 num_channels; 580 bool symmetric_memory; 581 enum intel_dram_type { 582 INTEL_DRAM_UNKNOWN, 583 INTEL_DRAM_DDR3, 584 INTEL_DRAM_DDR4, 585 INTEL_DRAM_LPDDR3, 586 INTEL_DRAM_LPDDR4, 587 INTEL_DRAM_DDR5, 588 INTEL_DRAM_LPDDR5, 589 INTEL_DRAM_GDDR, 590 INTEL_DRAM_GDDR_ECC, 591 } type; 592 u8 num_qgv_points; 593 u8 num_psf_gv_points; 594 } dram_info; 595 596 /* 597 * edram size in MB. 598 * Cannot be determined by PCIID. You must always read a register. 599 */ 600 u32 edram_size_mb; 601 602 /* To shut up runtime pm macros.. */ 603 struct xe_runtime_pm {} runtime_pm; 604 605 /* only to allow build, not used functionally */ 606 u32 irq_mask; 607 608 struct intel_uncore { 609 spinlock_t lock; 610 } uncore; 611 612 /* only to allow build, not used functionally */ 613 struct { 614 unsigned int hpll_freq; 615 unsigned int czclk_freq; 616 unsigned int fsb_freq, mem_freq, is_ddr3; 617 }; 618 #endif 619 }; 620 621 /** 622 * struct xe_file - file handle for XE driver 623 */ 624 struct xe_file { 625 /** @xe: xe DEVICE **/ 626 struct xe_device *xe; 627 628 /** @drm: base DRM file */ 629 struct drm_file *drm; 630 631 /** @vm: VM state for file */ 632 struct { 633 /** @vm.xe: xarray to store VMs */ 634 struct xarray xa; 635 /** 636 * @vm.lock: Protects VM lookup + reference and removal from 637 * file xarray. Not an intended to be an outer lock which does 638 * thing while being held. 639 */ 640 struct mutex lock; 641 } vm; 642 643 /** @exec_queue: Submission exec queue state for file */ 644 struct { 645 /** @exec_queue.xa: xarray to store exece queues */ 646 struct xarray xa; 647 /** 648 * @exec_queue.lock: Protects exec queue lookup + reference and 649 * removal from file xarray. Not intended to be an outer lock 650 * which does things while being held. 651 */ 652 struct mutex lock; 653 /** 654 * @exec_queue.pending_removal: items pending to be removed to 655 * synchronize GPU state update with ongoing query. 656 */ 657 atomic_t pending_removal; 658 } exec_queue; 659 660 /** @run_ticks: hw engine class run time in ticks for this drm client */ 661 u64 run_ticks[XE_ENGINE_CLASS_MAX]; 662 663 /** @client: drm client */ 664 struct xe_drm_client *client; 665 666 /** 667 * @process_name: process name for file handle, used to safely output 668 * during error situations where xe file can outlive process 669 */ 670 char *process_name; 671 672 /** 673 * @pid: pid for file handle, used to safely output uring error 674 * situations where xe file can outlive process 675 */ 676 pid_t pid; 677 678 /** @refcount: ref count of this xe file */ 679 struct kref refcount; 680 }; 681 682 #endif 683