40 int i915_gem_stolen_insert_node_in_range(struct drm_i915_private *i915,  in i915_gem_stolen_insert_node_in_range()  argument
46 	if (!drm_mm_initialized(&i915->mm.stolen))  in i915_gem_stolen_insert_node_in_range()
50 	if (GRAPHICS_VER(i915) >= 8 && start < 4096)  in i915_gem_stolen_insert_node_in_range()
53 	mutex_lock(&i915->mm.stolen_lock);  in i915_gem_stolen_insert_node_in_range()
54 	ret = drm_mm_insert_node_in_range(&i915->mm.stolen, node,  in i915_gem_stolen_insert_node_in_range()
57 	mutex_unlock(&i915->mm.stolen_lock);  in i915_gem_stolen_insert_node_in_range()
62 int i915_gem_stolen_insert_node(struct drm_i915_private *i915,  in i915_gem_stolen_insert_node()  argument
66 	return i915_gem_stolen_insert_node_in_range(i915, node,  in i915_gem_stolen_insert_node()
72 void i915_gem_stolen_remove_node(struct drm_i915_private *i915,  in i915_gem_stolen_remove_node()  argument
75 	mutex_lock(&i915->mm.stolen_lock);  in i915_gem_stolen_remove_node()
77 	mutex_unlock(&i915->mm.stolen_lock);  in i915_gem_stolen_remove_node()
80 static bool valid_stolen_size(struct drm_i915_private *i915, struct resource *dsm)  in valid_stolen_size()  argument
82 	return (dsm->start != 0 || HAS_LMEMBAR_SMEM_STOLEN(i915)) && dsm->end > dsm->start;  in valid_stolen_size()
85 static int adjust_stolen(struct drm_i915_private *i915,  in adjust_stolen()  argument
88 	struct i915_ggtt *ggtt = to_gt(i915)->ggtt;  in adjust_stolen()
91 	if (!valid_stolen_size(i915, dsm))  in adjust_stolen()
100 	if (GRAPHICS_VER(i915) <= 4 &&  in adjust_stolen()
101 	    !IS_G33(i915) && !IS_PINEVIEW(i915) && !IS_G4X(i915)) {  in adjust_stolen()
107 		if (GRAPHICS_VER(i915) == 4)  in adjust_stolen()
128 			drm_dbg(&i915->drm,  in adjust_stolen()
131 			drm_dbg(&i915->drm, "Stolen memory adjusted to %pR\n",  in adjust_stolen()
136 	if (!valid_stolen_size(i915, dsm))  in adjust_stolen()
142 static int request_smem_stolen(struct drm_i915_private *i915,  in request_smem_stolen()  argument
154 	if (HAS_LMEM(i915) || HAS_LMEMBAR_SMEM_STOLEN(i915))  in request_smem_stolen()
163 	r = devm_request_mem_region(i915->drm.dev, dsm->start,  in request_smem_stolen()
176 		r = devm_request_mem_region(i915->drm.dev, dsm->start + 1,  in request_smem_stolen()
183 		if (!r && GRAPHICS_VER(i915) != 3) {  in request_smem_stolen()
184 			drm_err(&i915->drm,  in request_smem_stolen()
195 static void i915_gem_cleanup_stolen(struct drm_i915_private *i915)  in i915_gem_cleanup_stolen()  argument
197 	if (!drm_mm_initialized(&i915->mm.stolen))  in i915_gem_cleanup_stolen()
200 	drm_mm_takedown(&i915->mm.stolen);  in i915_gem_cleanup_stolen()
203 static void g4x_get_stolen_reserved(struct drm_i915_private *i915,  in g4x_get_stolen_reserved()  argument
209 					IS_GM45(i915) ?  in g4x_get_stolen_reserved()
212 	resource_size_t stolen_top = i915->dsm.stolen.end + 1;  in g4x_get_stolen_reserved()
214 	drm_dbg(&i915->drm, "%s_STOLEN_RESERVED = %08x\n",  in g4x_get_stolen_reserved()
215 		IS_GM45(i915) ? "CTG" : "ELK", reg_val);  in g4x_get_stolen_reserved()
224 	drm_WARN(&i915->drm, GRAPHICS_VER(i915) == 5,  in g4x_get_stolen_reserved()
232 	drm_WARN_ON(&i915->drm,  in g4x_get_stolen_reserved()
238 static void gen6_get_stolen_reserved(struct drm_i915_private *i915,  in gen6_get_stolen_reserved()  argument
245 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in gen6_get_stolen_reserved()
271 static void vlv_get_stolen_reserved(struct drm_i915_private *i915,  in vlv_get_stolen_reserved()  argument
277 	resource_size_t stolen_top = i915->dsm.stolen.end + 1;  in vlv_get_stolen_reserved()
279 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in vlv_get_stolen_reserved()
300 static void gen7_get_stolen_reserved(struct drm_i915_private *i915,  in gen7_get_stolen_reserved()  argument
307 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in gen7_get_stolen_reserved()
327 static void chv_get_stolen_reserved(struct drm_i915_private *i915,  in chv_get_stolen_reserved()  argument
334 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in chv_get_stolen_reserved()
360 static void bdw_get_stolen_reserved(struct drm_i915_private *i915,  in bdw_get_stolen_reserved()  argument
366 	resource_size_t stolen_top = i915->dsm.stolen.end + 1;  in bdw_get_stolen_reserved()
368 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = %08x\n", reg_val);  in bdw_get_stolen_reserved()
380 static void icl_get_stolen_reserved(struct drm_i915_private *i915,  in icl_get_stolen_reserved()  argument
387 	drm_dbg(&i915->drm, "GEN6_STOLEN_RESERVED = 0x%016llx\n", reg_val);  in icl_get_stolen_reserved()
390 	if (MEDIA_VER_FULL(i915) == IP_VER(13, 0)) {  in icl_get_stolen_reserved()
402 		if (gscpsmi_base >= i915->dsm.stolen.start &&  in icl_get_stolen_reserved()
403 		    gscpsmi_base < i915->dsm.stolen.end) {  in icl_get_stolen_reserved()
405 			*size = i915->dsm.stolen.end - gscpsmi_base;  in icl_get_stolen_reserved()
428 	if (HAS_LMEMBAR_SMEM_STOLEN(i915))  in icl_get_stolen_reserved()
436  * Initialize i915->dsm.reserved to contain the reserved space within the Data
445 static int init_reserved_stolen(struct drm_i915_private *i915)  in init_reserved_stolen()  argument
447 	struct intel_uncore *uncore = &i915->uncore;  in init_reserved_stolen()
452 	stolen_top = i915->dsm.stolen.end + 1;  in init_reserved_stolen()
456 	if (GRAPHICS_VER(i915) >= 11) {  in init_reserved_stolen()
457 		icl_get_stolen_reserved(i915, uncore,  in init_reserved_stolen()
459 	} else if (GRAPHICS_VER(i915) >= 8) {  in init_reserved_stolen()
460 		if (IS_LP(i915))  in init_reserved_stolen()
461 			chv_get_stolen_reserved(i915, uncore,  in init_reserved_stolen()
464 			bdw_get_stolen_reserved(i915, uncore,  in init_reserved_stolen()
466 	} else if (GRAPHICS_VER(i915) >= 7) {  in init_reserved_stolen()
467 		if (IS_VALLEYVIEW(i915))  in init_reserved_stolen()
468 			vlv_get_stolen_reserved(i915, uncore,  in init_reserved_stolen()
471 			gen7_get_stolen_reserved(i915, uncore,  in init_reserved_stolen()
473 	} else if (GRAPHICS_VER(i915) >= 6) {  in init_reserved_stolen()
474 		gen6_get_stolen_reserved(i915, uncore,  in init_reserved_stolen()
476 	} else if (GRAPHICS_VER(i915) >= 5 || IS_G4X(i915)) {  in init_reserved_stolen()
477 		g4x_get_stolen_reserved(i915, uncore,  in init_reserved_stolen()
486 		drm_err(&i915->drm,  in init_reserved_stolen()
493 	i915->dsm.reserved = DEFINE_RES_MEM(reserved_base, reserved_size);  in init_reserved_stolen()
495 	if (!resource_contains(&i915->dsm.stolen, &i915->dsm.reserved)) {  in init_reserved_stolen()
496 		drm_err(&i915->drm,  in init_reserved_stolen()
498 			&i915->dsm.reserved, &i915->dsm.stolen);  in init_reserved_stolen()
506 	i915->dsm.reserved = DEFINE_RES_MEM(reserved_base, 0);  in init_reserved_stolen()
513 	struct drm_i915_private *i915 = mem->i915;  in i915_gem_init_stolen()  local
515 	mutex_init(&i915->mm.stolen_lock);  in i915_gem_init_stolen()
517 	if (intel_vgpu_active(i915)) {  in i915_gem_init_stolen()
518 		drm_notice(&i915->drm,  in i915_gem_init_stolen()
524 	if (i915_vtd_active(i915) && GRAPHICS_VER(i915) < 8) {  in i915_gem_init_stolen()
525 		drm_notice(&i915->drm,  in i915_gem_init_stolen()
531 	if (adjust_stolen(i915, &mem->region))  in i915_gem_init_stolen()
534 	if (request_smem_stolen(i915, &mem->region))  in i915_gem_init_stolen()
537 	i915->dsm.stolen = mem->region;  in i915_gem_init_stolen()
539 	if (init_reserved_stolen(i915))  in i915_gem_init_stolen()
543 	mem->region.end = i915->dsm.reserved.start - 1;  in i915_gem_init_stolen()
546 	i915->dsm.usable_size = resource_size(&mem->region);  in i915_gem_init_stolen()
548 	drm_dbg(&i915->drm,  in i915_gem_init_stolen()
550 		(u64)resource_size(&i915->dsm.stolen) >> 10,  in i915_gem_init_stolen()
551 		(u64)i915->dsm.usable_size >> 10);  in i915_gem_init_stolen()
553 	if (i915->dsm.usable_size == 0)  in i915_gem_init_stolen()
557 	drm_mm_init(&i915->mm.stolen, 0, i915->dsm.usable_size);  in i915_gem_init_stolen()
564 	if (IS_METEORLAKE(i915) && INTEL_REVID(i915) == 0x0)  in i915_gem_init_stolen()
565 		i915->dsm.usable_size = 0;  in i915_gem_init_stolen()
589 				     i915_gem_get_pat_index(ggtt->vm.i915,  in dbg_poison()
613 	struct drm_i915_private *i915 = to_i915(dev);  in i915_pages_create_for_stolen()  local
617 	GEM_BUG_ON(range_overflows(offset, size, resource_size(&i915->dsm.stolen)));  in i915_pages_create_for_stolen()
637 	sg_dma_address(sg) = (dma_addr_t)i915->dsm.stolen.start + offset;  in i915_pages_create_for_stolen()
645 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_get_pages_stolen()  local
653 	dbg_poison(to_gt(i915)->ggtt,  in i915_gem_object_get_pages_stolen()
666 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_put_pages_stolen()  local
669 	dbg_poison(to_gt(i915)->ggtt,  in i915_gem_object_put_pages_stolen()
681 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_release_stolen()  local
685 	i915_gem_stolen_remove_node(i915, stolen);  in i915_gem_object_release_stolen()
713 	drm_gem_private_object_init(&mem->i915->drm, &obj->base, stolen->size);  in __i915_gem_object_create_stolen()
718 	cache_level = HAS_LLC(mem->i915) ? I915_CACHE_LLC : I915_CACHE_NONE;  in __i915_gem_object_create_stolen()
741 	struct drm_i915_private *i915 = mem->i915;  in _i915_gem_object_stolen_init()  local
745 	if (!drm_mm_initialized(&i915->mm.stolen))  in _i915_gem_object_stolen_init()
764 		drm_dbg(&i915->drm,  in _i915_gem_object_stolen_init()
770 		mutex_lock(&i915->mm.stolen_lock);  in _i915_gem_object_stolen_init()
771 		ret = drm_mm_reserve_node(&i915->mm.stolen, stolen);  in _i915_gem_object_stolen_init()
772 		mutex_unlock(&i915->mm.stolen_lock);  in _i915_gem_object_stolen_init()
774 		ret = i915_gem_stolen_insert_node(i915, stolen, size,  in _i915_gem_object_stolen_init()
787 	i915_gem_stolen_remove_node(i915, stolen);  in _i915_gem_object_stolen_init()
794 i915_gem_object_create_stolen(struct drm_i915_private *i915,  in i915_gem_object_create_stolen()  argument
797 	return i915_gem_object_create_region(i915->mm.stolen_region, size, 0, 0);  in i915_gem_object_create_stolen()
810 		drm_dbg(&mem->i915->drm, "Skip stolen region: failed to setup\n");  in init_stolen_smem()
817 	i915_gem_cleanup_stolen(mem->i915);  in release_stolen_smem()
829 	struct drm_i915_private *i915 = mem->i915;  in init_stolen_lmem()  local
837 		drm_dbg(&mem->i915->drm, "Skip stolen region: failed to setup\n");  in init_stolen_lmem()
845 	drm_dbg(&i915->drm, "Stolen Local memory IO start: %pa\n",  in init_stolen_lmem()
847 	drm_dbg(&i915->drm, "Stolen Local DSM base: %pa\n", &mem->region.start);  in init_stolen_lmem()
852 	i915_gem_cleanup_stolen(mem->i915);  in init_stolen_lmem()
860 	i915_gem_cleanup_stolen(mem->i915);  in release_stolen_lmem()
894 i915_gem_stolen_lmem_setup(struct drm_i915_private *i915, u16 type,  in i915_gem_stolen_lmem_setup()  argument
897 	struct intel_uncore *uncore = &i915->uncore;  in i915_gem_stolen_lmem_setup()
898 	struct pci_dev *pdev = to_pci_dev(i915->drm.dev);  in i915_gem_stolen_lmem_setup()
911 	if (HAS_LMEMBAR_SMEM_STOLEN(i915) || IS_DG1(i915)) {  in i915_gem_stolen_lmem_setup()
916 		lmem_range = intel_gt_mcr_read_any(to_gt(i915), XEHP_TILE0_ADDR_RANGE) & 0xFFFF;  in i915_gem_stolen_lmem_setup()
921 	if (HAS_LMEMBAR_SMEM_STOLEN(i915)) {  in i915_gem_stolen_lmem_setup()
924 		 * Also MTL uses offset to GSMBASE in ptes, so i915  in i915_gem_stolen_lmem_setup()
930 			drm_err(&i915->drm, "invalid MTL GGC register setting\n");  in i915_gem_stolen_lmem_setup()
955 	min_page_size = HAS_64K_PAGES(i915) ? I915_GTT_PAGE_SIZE_64K :  in i915_gem_stolen_lmem_setup()
958 	mem = intel_memory_region_create(i915, dsm_base, dsm_size,  in i915_gem_stolen_lmem_setup()
974 i915_gem_stolen_smem_setup(struct drm_i915_private *i915, u16 type,  in i915_gem_stolen_smem_setup()  argument
979 	mem = intel_memory_region_create(i915,  in i915_gem_stolen_smem_setup()
999 bool i915_gem_stolen_initialized(const struct drm_i915_private *i915)  in i915_gem_stolen_initialized()  argument
1001 	return drm_mm_initialized(&i915->mm.stolen);  in i915_gem_stolen_initialized()
1004 u64 i915_gem_stolen_area_address(const struct drm_i915_private *i915)  in i915_gem_stolen_area_address()  argument
1006 	return i915->dsm.stolen.start;  in i915_gem_stolen_area_address()
1009 u64 i915_gem_stolen_area_size(const struct drm_i915_private *i915)  in i915_gem_stolen_area_size()  argument
1011 	return resource_size(&i915->dsm.stolen);  in i915_gem_stolen_area_size()
1014 u64 i915_gem_stolen_node_address(const struct drm_i915_private *i915,  in i915_gem_stolen_node_address()  argument
1017 	return i915->dsm.stolen.start + i915_gem_stolen_node_offset(node);  in i915_gem_stolen_node_address()