19 static bool gpu_write_needs_clflush(struct drm_i915_gem_object *obj)  in gpu_write_needs_clflush()  argument
21 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in gpu_write_needs_clflush()
33 	return !(i915_gem_object_has_cache_level(obj, I915_CACHE_NONE) ||  in gpu_write_needs_clflush()
34 		 i915_gem_object_has_cache_level(obj, I915_CACHE_WT));  in gpu_write_needs_clflush()
37 bool i915_gem_cpu_write_needs_clflush(struct drm_i915_gem_object *obj)  in i915_gem_cpu_write_needs_clflush()  argument
39 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_cpu_write_needs_clflush()
41 	if (obj->cache_dirty)  in i915_gem_cpu_write_needs_clflush()
47 	if (!(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE))  in i915_gem_cpu_write_needs_clflush()
51 	return i915_gem_object_is_framebuffer(obj);  in i915_gem_cpu_write_needs_clflush()
55 flush_write_domain(struct drm_i915_gem_object *obj, unsigned int flush_domains)  in flush_write_domain()  argument
59 	assert_object_held(obj);  in flush_write_domain()
61 	if (!(obj->write_domain & flush_domains))  in flush_write_domain()
64 	switch (obj->write_domain) {  in flush_write_domain()
66 		spin_lock(&obj->vma.lock);  in flush_write_domain()
67 		for_each_ggtt_vma(vma, obj)  in flush_write_domain()
69 		spin_unlock(&obj->vma.lock);  in flush_write_domain()
71 		i915_gem_object_flush_frontbuffer(obj, ORIGIN_CPU);  in flush_write_domain()
79 		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);  in flush_write_domain()
83 		if (gpu_write_needs_clflush(obj))  in flush_write_domain()
84 			obj->cache_dirty = true;  in flush_write_domain()
88 	obj->write_domain = 0;  in flush_write_domain()
91 static void __i915_gem_object_flush_for_display(struct drm_i915_gem_object *obj)  in __i915_gem_object_flush_for_display()  argument
97 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in __i915_gem_object_flush_for_display()
98 	if (obj->cache_dirty)  in __i915_gem_object_flush_for_display()
99 		i915_gem_clflush_object(obj, I915_CLFLUSH_FORCE);  in __i915_gem_object_flush_for_display()
100 	obj->write_domain = 0;  in __i915_gem_object_flush_for_display()
103 void i915_gem_object_flush_if_display(struct drm_i915_gem_object *obj)  in i915_gem_object_flush_if_display()  argument
105 	if (!i915_gem_object_is_framebuffer(obj))  in i915_gem_object_flush_if_display()
108 	i915_gem_object_lock(obj, NULL);  in i915_gem_object_flush_if_display()
109 	__i915_gem_object_flush_for_display(obj);  in i915_gem_object_flush_if_display()
110 	i915_gem_object_unlock(obj);  in i915_gem_object_flush_if_display()
113 void i915_gem_object_flush_if_display_locked(struct drm_i915_gem_object *obj)  in i915_gem_object_flush_if_display_locked()  argument
115 	if (i915_gem_object_is_framebuffer(obj))  in i915_gem_object_flush_if_display_locked()
116 		__i915_gem_object_flush_for_display(obj);  in i915_gem_object_flush_if_display_locked()
129 i915_gem_object_set_to_wc_domain(struct drm_i915_gem_object *obj, bool write)  in i915_gem_object_set_to_wc_domain()  argument
133 	assert_object_held(obj);  in i915_gem_object_set_to_wc_domain()
135 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_to_wc_domain()
142 	if (obj->write_domain == I915_GEM_DOMAIN_WC)  in i915_gem_object_set_to_wc_domain()
153 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_set_to_wc_domain()
157 	flush_write_domain(obj, ~I915_GEM_DOMAIN_WC);  in i915_gem_object_set_to_wc_domain()
163 	if ((obj->read_domains & I915_GEM_DOMAIN_WC) == 0)  in i915_gem_object_set_to_wc_domain()
169 	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_WC) != 0);  in i915_gem_object_set_to_wc_domain()
170 	obj->read_domains |= I915_GEM_DOMAIN_WC;  in i915_gem_object_set_to_wc_domain()
172 		obj->read_domains = I915_GEM_DOMAIN_WC;  in i915_gem_object_set_to_wc_domain()
173 		obj->write_domain = I915_GEM_DOMAIN_WC;  in i915_gem_object_set_to_wc_domain()
174 		obj->mm.dirty = true;  in i915_gem_object_set_to_wc_domain()
177 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_set_to_wc_domain()
191 i915_gem_object_set_to_gtt_domain(struct drm_i915_gem_object *obj, bool write)  in i915_gem_object_set_to_gtt_domain()  argument
195 	assert_object_held(obj);  in i915_gem_object_set_to_gtt_domain()
197 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_to_gtt_domain()
204 	if (obj->write_domain == I915_GEM_DOMAIN_GTT)  in i915_gem_object_set_to_gtt_domain()
215 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_set_to_gtt_domain()
219 	flush_write_domain(obj, ~I915_GEM_DOMAIN_GTT);  in i915_gem_object_set_to_gtt_domain()
225 	if ((obj->read_domains & I915_GEM_DOMAIN_GTT) == 0)  in i915_gem_object_set_to_gtt_domain()
231 	GEM_BUG_ON((obj->write_domain & ~I915_GEM_DOMAIN_GTT) != 0);  in i915_gem_object_set_to_gtt_domain()
232 	obj->read_domains |= I915_GEM_DOMAIN_GTT;  in i915_gem_object_set_to_gtt_domain()
236 		obj->read_domains = I915_GEM_DOMAIN_GTT;  in i915_gem_object_set_to_gtt_domain()
237 		obj->write_domain = I915_GEM_DOMAIN_GTT;  in i915_gem_object_set_to_gtt_domain()
238 		obj->mm.dirty = true;  in i915_gem_object_set_to_gtt_domain()
240 		spin_lock(&obj->vma.lock);  in i915_gem_object_set_to_gtt_domain()
241 		for_each_ggtt_vma(vma, obj)  in i915_gem_object_set_to_gtt_domain()
244 		spin_unlock(&obj->vma.lock);  in i915_gem_object_set_to_gtt_domain()
247 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_set_to_gtt_domain()
266 int i915_gem_object_set_cache_level(struct drm_i915_gem_object *obj,  in i915_gem_object_set_cache_level()  argument
277 	if (i915_gem_object_has_cache_level(obj, cache_level))  in i915_gem_object_set_cache_level()
280 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_cache_level()
288 	i915_gem_object_set_cache_coherency(obj, cache_level);  in i915_gem_object_set_cache_level()
289 	obj->cache_dirty = true;  in i915_gem_object_set_cache_level()
292 	return i915_gem_object_unbind(obj,  in i915_gem_object_set_cache_level()
301 	struct drm_i915_gem_object *obj;  in i915_gem_get_caching_ioctl()  local
308 	obj = i915_gem_object_lookup_rcu(file, args->handle);  in i915_gem_get_caching_ioctl()
309 	if (!obj) {  in i915_gem_get_caching_ioctl()
318 	if (obj->pat_set_by_user) {  in i915_gem_get_caching_ioctl()
323 	if (i915_gem_object_has_cache_level(obj, I915_CACHE_LLC) ||  in i915_gem_get_caching_ioctl()
324 	    i915_gem_object_has_cache_level(obj, I915_CACHE_L3_LLC))  in i915_gem_get_caching_ioctl()
326 	else if (i915_gem_object_has_cache_level(obj, I915_CACHE_WT))  in i915_gem_get_caching_ioctl()
340 	struct drm_i915_gem_object *obj;  in i915_gem_set_caching_ioctl()  local
373 	obj = i915_gem_object_lookup(file, args->handle);  in i915_gem_set_caching_ioctl()
374 	if (!obj)  in i915_gem_set_caching_ioctl()
381 	if (obj->pat_set_by_user) {  in i915_gem_set_caching_ioctl()
390 	if (i915_gem_object_is_proxy(obj)) {  in i915_gem_set_caching_ioctl()
395 		if (!i915_gem_object_is_userptr(obj) ||  in i915_gem_set_caching_ioctl()
402 	ret = i915_gem_object_lock_interruptible(obj, NULL);  in i915_gem_set_caching_ioctl()
406 	ret = i915_gem_object_set_cache_level(obj, level);  in i915_gem_set_caching_ioctl()
407 	i915_gem_object_unlock(obj);  in i915_gem_set_caching_ioctl()
410 	i915_gem_object_put(obj);  in i915_gem_set_caching_ioctl()
421 i915_gem_object_pin_to_display_plane(struct drm_i915_gem_object *obj,  in i915_gem_object_pin_to_display_plane()  argument
427 	struct drm_i915_private *i915 = to_i915(obj->base.dev);  in i915_gem_object_pin_to_display_plane()
432 	if (HAS_LMEM(i915) && !i915_gem_object_is_lmem(obj))  in i915_gem_object_pin_to_display_plane()
445 	ret = i915_gem_object_set_cache_level(obj,  in i915_gem_object_pin_to_display_plane()
466 		vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0, alignment,  in i915_gem_object_pin_to_display_plane()
470 		vma = i915_gem_object_ggtt_pin_ww(obj, ww, view, 0,  in i915_gem_object_pin_to_display_plane()
478 	i915_gem_object_flush_if_display_locked(obj);  in i915_gem_object_pin_to_display_plane()
493 i915_gem_object_set_to_cpu_domain(struct drm_i915_gem_object *obj, bool write)  in i915_gem_object_set_to_cpu_domain()  argument
497 	assert_object_held(obj);  in i915_gem_object_set_to_cpu_domain()
499 	ret = i915_gem_object_wait(obj,  in i915_gem_object_set_to_cpu_domain()
506 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_set_to_cpu_domain()
509 	if ((obj->read_domains & I915_GEM_DOMAIN_CPU) == 0) {  in i915_gem_object_set_to_cpu_domain()
510 		i915_gem_clflush_object(obj, I915_CLFLUSH_SYNC);  in i915_gem_object_set_to_cpu_domain()
511 		obj->read_domains |= I915_GEM_DOMAIN_CPU;  in i915_gem_object_set_to_cpu_domain()
517 	GEM_BUG_ON(obj->write_domain & ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_set_to_cpu_domain()
523 		__start_cpu_write(obj);  in i915_gem_object_set_to_cpu_domain()
541 	struct drm_i915_gem_object *obj;  in i915_gem_set_domain_ioctl()  local
563 	obj = i915_gem_object_lookup(file, args->handle);  in i915_gem_set_domain_ioctl()
564 	if (!obj)  in i915_gem_set_domain_ioctl()
572 	err = i915_gem_object_wait(obj,  in i915_gem_set_domain_ioctl()
580 	if (i915_gem_object_is_userptr(obj)) {  in i915_gem_set_domain_ioctl()
585 		err = i915_gem_object_userptr_validate(obj);  in i915_gem_set_domain_ioctl()
587 			err = i915_gem_object_wait(obj,  in i915_gem_set_domain_ioctl()
601 	if (i915_gem_object_is_proxy(obj)) {  in i915_gem_set_domain_ioctl()
606 	err = i915_gem_object_lock_interruptible(obj, NULL);  in i915_gem_set_domain_ioctl()
619 	err = i915_gem_object_pin_pages(obj);  in i915_gem_set_domain_ioctl()
633 	if (READ_ONCE(obj->write_domain) == read_domains)  in i915_gem_set_domain_ioctl()
637 		err = i915_gem_object_set_to_wc_domain(obj, write_domain);  in i915_gem_set_domain_ioctl()
639 		err = i915_gem_object_set_to_gtt_domain(obj, write_domain);  in i915_gem_set_domain_ioctl()
641 		err = i915_gem_object_set_to_cpu_domain(obj, write_domain);  in i915_gem_set_domain_ioctl()
644 	i915_gem_object_unpin_pages(obj);  in i915_gem_set_domain_ioctl()
647 	i915_gem_object_unlock(obj);  in i915_gem_set_domain_ioctl()
650 		i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);  in i915_gem_set_domain_ioctl()
653 	i915_gem_object_put(obj);  in i915_gem_set_domain_ioctl()
662 int i915_gem_object_prepare_read(struct drm_i915_gem_object *obj,  in i915_gem_object_prepare_read()  argument
668 	if (!i915_gem_object_has_struct_page(obj))  in i915_gem_object_prepare_read()
671 	assert_object_held(obj);  in i915_gem_object_prepare_read()
673 	ret = i915_gem_object_wait(obj,  in i915_gem_object_prepare_read()
679 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_prepare_read()
683 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ ||  in i915_gem_object_prepare_read()
685 		ret = i915_gem_object_set_to_cpu_domain(obj, false);  in i915_gem_object_prepare_read()
692 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_prepare_read()
699 	if (!obj->cache_dirty &&  in i915_gem_object_prepare_read()
700 	    !(obj->read_domains & I915_GEM_DOMAIN_CPU))  in i915_gem_object_prepare_read()
708 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_prepare_read()
712 int i915_gem_object_prepare_write(struct drm_i915_gem_object *obj,  in i915_gem_object_prepare_write()  argument
718 	if (!i915_gem_object_has_struct_page(obj))  in i915_gem_object_prepare_write()
721 	assert_object_held(obj);  in i915_gem_object_prepare_write()
723 	ret = i915_gem_object_wait(obj,  in i915_gem_object_prepare_write()
730 	ret = i915_gem_object_pin_pages(obj);  in i915_gem_object_prepare_write()
734 	if (obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_WRITE ||  in i915_gem_object_prepare_write()
736 		ret = i915_gem_object_set_to_cpu_domain(obj, true);  in i915_gem_object_prepare_write()
743 	flush_write_domain(obj, ~I915_GEM_DOMAIN_CPU);  in i915_gem_object_prepare_write()
750 	if (!obj->cache_dirty) {  in i915_gem_object_prepare_write()
757 		if (!(obj->read_domains & I915_GEM_DOMAIN_CPU))  in i915_gem_object_prepare_write()
762 	i915_gem_object_invalidate_frontbuffer(obj, ORIGIN_CPU);  in i915_gem_object_prepare_write()
763 	obj->mm.dirty = true;  in i915_gem_object_prepare_write()
768 	i915_gem_object_unpin_pages(obj);  in i915_gem_object_prepare_write()