Lines Matching +full:ext +full:- +full:gen
2 * SPDX-License-Identifier: MIT
7 #include <linux/intel-iommu.h>
8 #include <linux/dma-resv.h>
125 * 1. Validation - Ensure all the pointers, handles and flags are valid.
126 * 2. Reservation - Assign GPU address space for every object
127 * 3. Relocation - Update any addresses to point to the final locations
128 * 4. Serialisation - Order the request with respect to its dependencies
129 * 5. Construction - Construct a request to execute the batchbuffer
157 * object already bound in its current location - so as long as meets the
185 * we do leave the objects pinned in their final locations - which is a
205 * time (completing any reads beforehand) - using semaphores where available
207 * write (either via mmaps using set-domain, or via pwrite) must flush all GPU
208 * reads before starting, and any read (either using set-domain or pread) must
239 struct drm_file *file; /** per-file lookup tables and limits */
272 unsigned int gen; /** Cached value of INTEL_GEN */ member
284 struct intel_gt_buffer_pool_node *reloc_pool; /** relocation pool for -EDEADLK handling */
314 return intel_engine_requires_cmd_parser(eb->engine) || in eb_use_cmdparser()
315 (intel_engine_using_cmd_parser(eb->engine) && in eb_use_cmdparser()
316 eb->args->batch_len); in eb_use_cmdparser()
321 if (!(eb->args->flags & I915_EXEC_HANDLE_LUT)) { in eb_create()
322 unsigned int size = 1 + ilog2(eb->buffer_count); in eb_create()
348 eb->buckets = kzalloc(sizeof(struct hlist_head) << size, in eb_create()
350 if (eb->buckets) in eb_create()
352 } while (--size); in eb_create()
355 return -ENOMEM; in eb_create()
357 eb->lut_size = size; in eb_create()
359 eb->lut_size = -eb->buffer_count; in eb_create()
370 if (vma->node.size < entry->pad_to_size) in eb_vma_misplaced()
373 if (entry->alignment && !IS_ALIGNED(vma->node.start, entry->alignment)) in eb_vma_misplaced()
377 vma->node.start != entry->offset) in eb_vma_misplaced()
381 vma->node.start < BATCH_OFFSET_BIAS) in eb_vma_misplaced()
385 (vma->node.start + vma->node.size - 1) >> 32) in eb_vma_misplaced()
414 pin_flags |= entry->offset | PIN_OFFSET_FIXED; in eb_pin_flags()
426 struct i915_vma *vma = ev->vma; in eb_pin_vma()
429 if (vma->node.size) in eb_pin_vma()
430 pin_flags = vma->node.start; in eb_pin_vma()
432 pin_flags = entry->offset & PIN_OFFSET_MASK; in eb_pin_vma()
435 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_GTT)) in eb_pin_vma()
439 /* TODO: Add -EDEADLK handling here */ in eb_pin_vma()
440 if (unlikely(i915_vma_pin_ww(vma, &eb->ww, 0, 0, pin_flags))) { in eb_pin_vma()
441 if (entry->flags & EXEC_OBJECT_PINNED) in eb_pin_vma()
445 if (unlikely(i915_vma_pin_ww(vma, &eb->ww, in eb_pin_vma()
446 entry->pad_to_size, in eb_pin_vma()
447 entry->alignment, in eb_pin_vma()
448 eb_pin_flags(entry, ev->flags) | in eb_pin_vma()
453 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { in eb_pin_vma()
459 if (vma->fence) in eb_pin_vma()
460 ev->flags |= __EXEC_OBJECT_HAS_FENCE; in eb_pin_vma()
463 ev->flags |= __EXEC_OBJECT_HAS_PIN; in eb_pin_vma()
464 return !eb_vma_misplaced(entry, vma, ev->flags); in eb_pin_vma()
470 if (!(ev->flags & __EXEC_OBJECT_HAS_PIN)) in eb_unreserve_vma()
473 if (unlikely(ev->flags & __EXEC_OBJECT_HAS_FENCE)) in eb_unreserve_vma()
474 __i915_vma_unpin_fence(ev->vma); in eb_unreserve_vma()
476 __i915_vma_unpin(ev->vma); in eb_unreserve_vma()
477 ev->flags &= ~__EXEC_OBJECT_RESERVED; in eb_unreserve_vma()
485 if (unlikely(entry->flags & eb->invalid_flags)) in eb_validate_vma()
486 return -EINVAL; in eb_validate_vma()
488 if (unlikely(entry->alignment && in eb_validate_vma()
489 !is_power_of_2_u64(entry->alignment))) in eb_validate_vma()
490 return -EINVAL; in eb_validate_vma()
494 * any non-page-aligned or non-canonical addresses. in eb_validate_vma()
496 if (unlikely(entry->flags & EXEC_OBJECT_PINNED && in eb_validate_vma()
497 entry->offset != gen8_canonical_addr(entry->offset & I915_GTT_PAGE_MASK))) in eb_validate_vma()
498 return -EINVAL; in eb_validate_vma()
501 if (entry->flags & EXEC_OBJECT_PAD_TO_SIZE) { in eb_validate_vma()
502 if (unlikely(offset_in_page(entry->pad_to_size))) in eb_validate_vma()
503 return -EINVAL; in eb_validate_vma()
505 entry->pad_to_size = 0; in eb_validate_vma()
509 * so from this point we're always using non-canonical in eb_validate_vma()
512 entry->offset = gen8_noncanonical_addr(entry->offset); in eb_validate_vma()
514 if (!eb->reloc_cache.has_fence) { in eb_validate_vma()
515 entry->flags &= ~EXEC_OBJECT_NEEDS_FENCE; in eb_validate_vma()
517 if ((entry->flags & EXEC_OBJECT_NEEDS_FENCE || in eb_validate_vma()
518 eb->reloc_cache.needs_unfenced) && in eb_validate_vma()
519 i915_gem_object_is_tiled(vma->obj)) in eb_validate_vma()
520 entry->flags |= EXEC_OBJECT_NEEDS_GTT | __EXEC_OBJECT_NEEDS_MAP; in eb_validate_vma()
523 if (!(entry->flags & EXEC_OBJECT_PINNED)) in eb_validate_vma()
524 entry->flags |= eb->context_flags; in eb_validate_vma()
534 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; in eb_add_vma()
535 struct eb_vma *ev = &eb->vma[i]; in eb_add_vma()
539 ev->vma = vma; in eb_add_vma()
540 ev->exec = entry; in eb_add_vma()
541 ev->flags = entry->flags; in eb_add_vma()
543 if (eb->lut_size > 0) { in eb_add_vma()
544 ev->handle = entry->handle; in eb_add_vma()
545 hlist_add_head(&ev->node, in eb_add_vma()
546 &eb->buckets[hash_32(entry->handle, in eb_add_vma()
547 eb->lut_size)]); in eb_add_vma()
550 if (entry->relocation_count) in eb_add_vma()
551 list_add_tail(&ev->reloc_link, &eb->relocs); in eb_add_vma()
563 if (entry->relocation_count && in eb_add_vma()
564 !(ev->flags & EXEC_OBJECT_PINNED)) in eb_add_vma()
565 ev->flags |= __EXEC_OBJECT_NEEDS_BIAS; in eb_add_vma()
566 if (eb->reloc_cache.has_fence) in eb_add_vma()
567 ev->flags |= EXEC_OBJECT_NEEDS_FENCE; in eb_add_vma()
569 eb->batch = ev; in eb_add_vma()
585 return (cache->has_llc || in use_cpu_reloc()
586 obj->cache_dirty || in use_cpu_reloc()
587 obj->cache_level != I915_CACHE_NONE); in use_cpu_reloc()
594 struct drm_i915_gem_exec_object2 *entry = ev->exec; in eb_reserve_vma()
595 struct i915_vma *vma = ev->vma; in eb_reserve_vma()
598 if (drm_mm_node_allocated(&vma->node) && in eb_reserve_vma()
599 eb_vma_misplaced(entry, vma, ev->flags)) { in eb_reserve_vma()
605 err = i915_vma_pin_ww(vma, &eb->ww, in eb_reserve_vma()
606 entry->pad_to_size, entry->alignment, in eb_reserve_vma()
607 eb_pin_flags(entry, ev->flags) | pin_flags); in eb_reserve_vma()
611 if (entry->offset != vma->node.start) { in eb_reserve_vma()
612 entry->offset = vma->node.start | UPDATE; in eb_reserve_vma()
613 eb->args->flags |= __EXEC_HAS_RELOC; in eb_reserve_vma()
616 if (unlikely(ev->flags & EXEC_OBJECT_NEEDS_FENCE)) { in eb_reserve_vma()
623 if (vma->fence) in eb_reserve_vma()
624 ev->flags |= __EXEC_OBJECT_HAS_FENCE; in eb_reserve_vma()
627 ev->flags |= __EXEC_OBJECT_HAS_PIN; in eb_reserve_vma()
628 GEM_BUG_ON(eb_vma_misplaced(entry, vma, ev->flags)); in eb_reserve_vma()
635 const unsigned int count = eb->buffer_count; in eb_reserve()
657 list_for_each_entry(ev, &eb->unbound, bind_link) { in eb_reserve()
662 if (err != -ENOSPC) in eb_reserve()
666 INIT_LIST_HEAD(&eb->unbound); in eb_reserve()
671 ev = &eb->vma[i]; in eb_reserve()
672 flags = ev->flags; in eb_reserve()
681 list_add(&ev->bind_link, &eb->unbound); in eb_reserve()
684 list_add_tail(&ev->bind_link, &eb->unbound); in eb_reserve()
687 list_add(&ev->bind_link, &last); in eb_reserve()
689 list_add_tail(&ev->bind_link, &last); in eb_reserve()
691 list_splice_tail(&last, &eb->unbound); in eb_reserve()
699 mutex_lock(&eb->context->vm->mutex); in eb_reserve()
700 err = i915_gem_evict_vm(eb->context->vm); in eb_reserve()
701 mutex_unlock(&eb->context->vm->mutex); in eb_reserve()
707 return -ENOSPC; in eb_reserve()
716 if (eb->args->flags & I915_EXEC_BATCH_FIRST) in eb_batch_index()
719 return eb->buffer_count - 1; in eb_batch_index()
726 ctx = i915_gem_context_lookup(eb->file->driver_priv, eb->args->rsvd1); in eb_select_context()
728 return -ENOENT; in eb_select_context()
730 eb->gem_context = ctx; in eb_select_context()
731 if (rcu_access_pointer(ctx->vm)) in eb_select_context()
732 eb->invalid_flags |= EXEC_OBJECT_NEEDS_GTT; in eb_select_context()
734 eb->context_flags = 0; in eb_select_context()
735 if (test_bit(UCONTEXT_NO_ZEROMAP, &ctx->user_flags)) in eb_select_context()
736 eb->context_flags |= __EXEC_OBJECT_NEEDS_BIAS; in eb_select_context()
744 struct i915_gem_context *ctx = eb->gem_context; in __eb_add_lut()
750 return -ENOMEM; in __eb_add_lut()
753 if (!atomic_fetch_inc(&vma->open_count)) in __eb_add_lut()
755 lut->handle = handle; in __eb_add_lut()
756 lut->ctx = ctx; in __eb_add_lut()
759 err = -EINTR; in __eb_add_lut()
760 if (!mutex_lock_interruptible(&ctx->lut_mutex)) { in __eb_add_lut()
761 struct i915_address_space *vm = rcu_access_pointer(ctx->vm); in __eb_add_lut()
763 if (unlikely(vm && vma->vm != vm)) in __eb_add_lut()
764 err = -EAGAIN; /* user racing with ctx set-vm */ in __eb_add_lut()
766 err = radix_tree_insert(&ctx->handles_vma, handle, vma); in __eb_add_lut()
768 err = -ENOENT; in __eb_add_lut()
770 struct drm_i915_gem_object *obj = vma->obj; in __eb_add_lut()
772 spin_lock(&obj->lut_lock); in __eb_add_lut()
773 if (idr_find(&eb->file->object_idr, handle) == obj) { in __eb_add_lut()
774 list_add(&lut->obj_link, &obj->lut_list); in __eb_add_lut()
776 radix_tree_delete(&ctx->handles_vma, handle); in __eb_add_lut()
777 err = -ENOENT; in __eb_add_lut()
779 spin_unlock(&obj->lut_lock); in __eb_add_lut()
781 mutex_unlock(&ctx->lut_mutex); in __eb_add_lut()
797 struct i915_address_space *vm = eb->context->vm; in eb_lookup_vma()
805 vma = radix_tree_lookup(&eb->gem_context->handles_vma, handle); in eb_lookup_vma()
806 if (likely(vma && vma->vm == vm)) in eb_lookup_vma()
812 obj = i915_gem_object_lookup(eb->file, handle); in eb_lookup_vma()
814 return ERR_PTR(-ENOENT); in eb_lookup_vma()
827 if (err != -EEXIST) in eb_lookup_vma()
834 struct drm_i915_private *i915 = eb->i915; in eb_lookup_vmas()
839 INIT_LIST_HEAD(&eb->relocs); in eb_lookup_vmas()
841 for (i = 0; i < eb->buffer_count; i++) { in eb_lookup_vmas()
844 vma = eb_lookup_vma(eb, eb->exec[i].handle); in eb_lookup_vmas()
850 err = eb_validate_vma(eb, &eb->exec[i], vma); in eb_lookup_vmas()
859 if (unlikely(eb->batch->flags & EXEC_OBJECT_WRITE)) { in eb_lookup_vmas()
860 drm_dbg(&i915->drm, in eb_lookup_vmas()
861 "Attempting to use self-modifying batch buffer\n"); in eb_lookup_vmas()
862 return -EINVAL; in eb_lookup_vmas()
866 eb->batch_start_offset, eb->batch_len, in eb_lookup_vmas()
867 eb->batch->vma->size)) { in eb_lookup_vmas()
868 drm_dbg(&i915->drm, "Attempting to use out-of-bounds batch\n"); in eb_lookup_vmas()
869 return -EINVAL; in eb_lookup_vmas()
872 if (eb->batch_len == 0) in eb_lookup_vmas()
873 eb->batch_len = eb->batch->vma->size - eb->batch_start_offset; in eb_lookup_vmas()
874 if (unlikely(eb->batch_len == 0)) { /* impossible! */ in eb_lookup_vmas()
875 drm_dbg(&i915->drm, "Invalid batch length\n"); in eb_lookup_vmas()
876 return -EINVAL; in eb_lookup_vmas()
882 eb->vma[i].vma = NULL; in eb_lookup_vmas()
891 INIT_LIST_HEAD(&eb->unbound); in eb_validate_vmas()
893 for (i = 0; i < eb->buffer_count; i++) { in eb_validate_vmas()
894 struct drm_i915_gem_exec_object2 *entry = &eb->exec[i]; in eb_validate_vmas()
895 struct eb_vma *ev = &eb->vma[i]; in eb_validate_vmas()
896 struct i915_vma *vma = ev->vma; in eb_validate_vmas()
898 err = i915_gem_object_lock(vma->obj, &eb->ww); in eb_validate_vmas()
903 if (entry->offset != vma->node.start) { in eb_validate_vmas()
904 entry->offset = vma->node.start | UPDATE; in eb_validate_vmas()
905 eb->args->flags |= __EXEC_HAS_RELOC; in eb_validate_vmas()
910 list_add_tail(&ev->bind_link, &eb->unbound); in eb_validate_vmas()
911 if (drm_mm_node_allocated(&vma->node)) { in eb_validate_vmas()
918 GEM_BUG_ON(drm_mm_node_allocated(&vma->node) && in eb_validate_vmas()
919 eb_vma_misplaced(&eb->exec[i], vma, ev->flags)); in eb_validate_vmas()
922 if (!list_empty(&eb->unbound)) in eb_validate_vmas()
931 if (eb->lut_size < 0) { in eb_get_vma()
932 if (handle >= -eb->lut_size) in eb_get_vma()
934 return &eb->vma[handle]; in eb_get_vma()
939 head = &eb->buckets[hash_32(handle, eb->lut_size)]; in eb_get_vma()
941 if (ev->handle == handle) in eb_get_vma()
950 const unsigned int count = eb->buffer_count; in eb_release_vmas()
954 struct eb_vma *ev = &eb->vma[i]; in eb_release_vmas()
955 struct i915_vma *vma = ev->vma; in eb_release_vmas()
971 GEM_BUG_ON(eb->reloc_cache.rq); in eb_destroy()
973 if (eb->lut_size > 0) in eb_destroy()
974 kfree(eb->buckets); in eb_destroy()
981 return gen8_canonical_addr((int)reloc->delta + target->node.start); in relocation_target()
986 cache->rq = NULL; in reloc_cache_clear()
987 cache->rq_cmd = NULL; in reloc_cache_clear()
988 cache->pool = NULL; in reloc_cache_clear()
989 cache->rq_size = 0; in reloc_cache_clear()
995 cache->page = -1; in reloc_cache_init()
996 cache->vaddr = 0; in reloc_cache_init()
998 cache->gen = INTEL_GEN(i915); in reloc_cache_init()
999 cache->has_llc = HAS_LLC(i915); in reloc_cache_init()
1000 cache->use_64bit_reloc = HAS_64BIT_RELOC(i915); in reloc_cache_init()
1001 cache->has_fence = cache->gen < 4; in reloc_cache_init()
1002 cache->needs_unfenced = INTEL_INFO(i915)->unfenced_needs_alignment; in reloc_cache_init()
1003 cache->node.flags = 0; in reloc_cache_init()
1022 container_of(cache, struct i915_execbuffer, reloc_cache)->i915; in cache_to_ggtt()
1023 return &i915->ggtt; in cache_to_ggtt()
1028 if (!cache->pool) in reloc_cache_put_pool()
1037 i915_gem_ww_unlock_single(cache->pool->obj); in reloc_cache_put_pool()
1038 intel_gt_buffer_pool_put(cache->pool); in reloc_cache_put_pool()
1039 cache->pool = NULL; in reloc_cache_put_pool()
1044 struct drm_i915_gem_object *obj = cache->rq->batch->obj; in reloc_gpu_flush()
1046 GEM_BUG_ON(cache->rq_size >= obj->base.size / sizeof(u32)); in reloc_gpu_flush()
1047 cache->rq_cmd[cache->rq_size] = MI_BATCH_BUFFER_END; in reloc_gpu_flush()
1049 __i915_gem_object_flush_map(obj, 0, sizeof(u32) * (cache->rq_size + 1)); in reloc_gpu_flush()
1052 intel_gt_chipset_flush(cache->rq->engine->gt); in reloc_gpu_flush()
1054 i915_request_add(cache->rq); in reloc_gpu_flush()
1058 eb->reloc_pool = NULL; in reloc_gpu_flush()
1065 if (cache->rq) in reloc_cache_reset()
1068 if (!cache->vaddr) in reloc_cache_reset()
1071 vaddr = unmask_page(cache->vaddr); in reloc_cache_reset()
1072 if (cache->vaddr & KMAP) { in reloc_cache_reset()
1074 (struct drm_i915_gem_object *)cache->node.mm; in reloc_cache_reset()
1075 if (cache->vaddr & CLFLUSH_AFTER) in reloc_cache_reset()
1083 intel_gt_flush_ggtt_writes(ggtt->vm.gt); in reloc_cache_reset()
1086 if (drm_mm_node_allocated(&cache->node)) { in reloc_cache_reset()
1087 ggtt->vm.clear_range(&ggtt->vm, in reloc_cache_reset()
1088 cache->node.start, in reloc_cache_reset()
1089 cache->node.size); in reloc_cache_reset()
1090 mutex_lock(&ggtt->vm.mutex); in reloc_cache_reset()
1091 drm_mm_remove_node(&cache->node); in reloc_cache_reset()
1092 mutex_unlock(&ggtt->vm.mutex); in reloc_cache_reset()
1094 i915_vma_unpin((struct i915_vma *)cache->node.mm); in reloc_cache_reset()
1098 cache->vaddr = 0; in reloc_cache_reset()
1099 cache->page = -1; in reloc_cache_reset()
1109 if (cache->vaddr) { in reloc_kmap()
1110 kunmap_atomic(unmask_page(cache->vaddr)); in reloc_kmap()
1122 cache->vaddr = flushes | KMAP; in reloc_kmap()
1123 cache->node.mm = (void *)obj; in reloc_kmap()
1129 if (!obj->mm.dirty) in reloc_kmap()
1133 cache->vaddr = unmask_flags(cache->vaddr) | (unsigned long)vaddr; in reloc_kmap()
1134 cache->page = pageno; in reloc_kmap()
1143 struct reloc_cache *cache = &eb->reloc_cache; in reloc_iomap()
1148 if (cache->vaddr) { in reloc_iomap()
1149 intel_gt_flush_ggtt_writes(ggtt->vm.gt); in reloc_iomap()
1150 io_mapping_unmap_atomic((void __force __iomem *) unmask_page(cache->vaddr)); in reloc_iomap()
1156 return ERR_PTR(-EINVAL); in reloc_iomap()
1165 vma = i915_gem_object_ggtt_pin_ww(obj, &eb->ww, NULL, 0, 0, in reloc_iomap()
1169 if (vma == ERR_PTR(-EDEADLK)) in reloc_iomap()
1173 memset(&cache->node, 0, sizeof(cache->node)); in reloc_iomap()
1174 mutex_lock(&ggtt->vm.mutex); in reloc_iomap()
1176 (&ggtt->vm.mm, &cache->node, in reloc_iomap()
1178 0, ggtt->mappable_end, in reloc_iomap()
1180 mutex_unlock(&ggtt->vm.mutex); in reloc_iomap()
1184 cache->node.start = vma->node.start; in reloc_iomap()
1185 cache->node.mm = (void *)vma; in reloc_iomap()
1189 offset = cache->node.start; in reloc_iomap()
1190 if (drm_mm_node_allocated(&cache->node)) { in reloc_iomap()
1191 ggtt->vm.insert_page(&ggtt->vm, in reloc_iomap()
1198 vaddr = (void __force *)io_mapping_map_atomic_wc(&ggtt->iomap, in reloc_iomap()
1200 cache->page = page; in reloc_iomap()
1201 cache->vaddr = (unsigned long)vaddr; in reloc_iomap()
1210 struct reloc_cache *cache = &eb->reloc_cache; in reloc_vaddr()
1213 if (cache->page == page) { in reloc_vaddr()
1214 vaddr = unmask_page(cache->vaddr); in reloc_vaddr()
1217 if ((cache->vaddr & KMAP) == 0) in reloc_vaddr()
1251 struct drm_i915_gem_object *obj = vma->obj; in reloc_move_to_gpu()
1256 if (obj->cache_dirty & ~obj->cache_coherent) in reloc_move_to_gpu()
1258 obj->write_domain = 0; in reloc_move_to_gpu()
1260 err = i915_request_await_object(rq, vma->obj, true); in reloc_move_to_gpu()
1272 struct reloc_cache *cache = &eb->reloc_cache; in __reloc_gpu_alloc()
1273 struct intel_gt_buffer_pool_node *pool = eb->reloc_pool; in __reloc_gpu_alloc()
1280 pool = intel_gt_get_buffer_pool(engine->gt, PAGE_SIZE); in __reloc_gpu_alloc()
1284 eb->reloc_pool = NULL; in __reloc_gpu_alloc()
1286 err = i915_gem_object_lock(pool->obj, &eb->ww); in __reloc_gpu_alloc()
1290 cmd = i915_gem_object_pin_map(pool->obj, in __reloc_gpu_alloc()
1291 cache->has_llc ? in __reloc_gpu_alloc()
1299 batch = i915_vma_instance(pool->obj, vma->vm, NULL); in __reloc_gpu_alloc()
1305 err = i915_vma_pin_ww(batch, &eb->ww, 0, 0, PIN_USER | PIN_NONBLOCK); in __reloc_gpu_alloc()
1309 if (engine == eb->context->engine) { in __reloc_gpu_alloc()
1310 rq = i915_request_create(eb->context); in __reloc_gpu_alloc()
1312 struct intel_context *ce = eb->reloc_context; in __reloc_gpu_alloc()
1321 i915_vm_put(ce->vm); in __reloc_gpu_alloc()
1322 ce->vm = i915_vm_get(eb->context->vm); in __reloc_gpu_alloc()
1323 eb->reloc_context = ce; in __reloc_gpu_alloc()
1326 err = intel_context_pin_ww(ce, &eb->ww); in __reloc_gpu_alloc()
1346 err = eb->engine->emit_bb_start(rq, in __reloc_gpu_alloc()
1347 batch->node.start, PAGE_SIZE, in __reloc_gpu_alloc()
1348 cache->gen > 5 ? 0 : I915_DISPATCH_SECURE); in __reloc_gpu_alloc()
1353 err = i915_request_await_object(rq, batch->obj, false); in __reloc_gpu_alloc()
1359 rq->batch = batch; in __reloc_gpu_alloc()
1362 cache->rq = rq; in __reloc_gpu_alloc()
1363 cache->rq_cmd = cmd; in __reloc_gpu_alloc()
1364 cache->rq_size = 0; in __reloc_gpu_alloc()
1365 cache->pool = pool; in __reloc_gpu_alloc()
1377 i915_gem_object_unpin_map(pool->obj); in __reloc_gpu_alloc()
1379 eb->reloc_pool = pool; in __reloc_gpu_alloc()
1385 return engine->class != VIDEO_DECODE_CLASS || !IS_GEN(engine->i915, 6); in reloc_can_use_engine()
1392 struct reloc_cache *cache = &eb->reloc_cache; in reloc_gpu()
1395 if (cache->rq_size > PAGE_SIZE/sizeof(u32) - (len + 1)) in reloc_gpu()
1398 if (unlikely(!cache->rq)) { in reloc_gpu()
1400 struct intel_engine_cs *engine = eb->engine; in reloc_gpu()
1403 engine = engine->gt->engine_class[COPY_ENGINE_CLASS][0]; in reloc_gpu()
1405 return ERR_PTR(-ENODEV); in reloc_gpu()
1413 cmd = cache->rq_cmd + cache->rq_size; in reloc_gpu()
1414 cache->rq_size += len; in reloc_gpu()
1427 return !dma_resv_test_signaled_rcu(vma->resv, true); in use_reloc_gpu()
1435 GEM_BUG_ON(vma->pages != vma->obj->mm.pages); in vma_phys_addr()
1437 page = i915_gem_object_get_page(vma->obj, offset >> PAGE_SHIFT); in vma_phys_addr()
1449 const unsigned int gen = eb->reloc_cache.gen; in __reloc_entry_gpu() local
1454 if (gen >= 8) in __reloc_entry_gpu()
1456 else if (gen >= 4) in __reloc_entry_gpu()
1462 if (batch == ERR_PTR(-EDEADLK)) in __reloc_entry_gpu()
1463 return -EDEADLK; in __reloc_entry_gpu()
1467 addr = gen8_canonical_addr(vma->node.start + offset); in __reloc_entry_gpu()
1468 if (gen >= 8) { in __reloc_entry_gpu()
1488 } else if (gen >= 6) { in __reloc_entry_gpu()
1493 } else if (IS_I965G(eb->i915)) { in __reloc_entry_gpu()
1498 } else if (gen >= 4) { in __reloc_entry_gpu()
1503 } else if (gen >= 3 && in __reloc_entry_gpu()
1504 !(IS_I915G(eb->i915) || IS_I915GM(eb->i915))) { in __reloc_entry_gpu()
1522 if (eb->reloc_cache.vaddr) in reloc_entry_gpu()
1538 u64 offset = reloc->offset; in relocate_entry()
1545 bool wide = eb->reloc_cache.use_64bit_reloc; in relocate_entry()
1549 vaddr = reloc_vaddr(vma->obj, eb, in relocate_entry()
1557 eb->reloc_cache.vaddr); in relocate_entry()
1567 return target->node.start | UPDATE; in relocate_entry()
1575 struct drm_i915_private *i915 = eb->i915; in eb_relocate_entry()
1580 target = eb_get_vma(eb, reloc->target_handle); in eb_relocate_entry()
1582 return -ENOENT; in eb_relocate_entry()
1585 if (unlikely(reloc->write_domain & (reloc->write_domain - 1))) { in eb_relocate_entry()
1586 drm_dbg(&i915->drm, "reloc with multiple write domains: " in eb_relocate_entry()
1589 reloc->target_handle, in eb_relocate_entry()
1590 (int) reloc->offset, in eb_relocate_entry()
1591 reloc->read_domains, in eb_relocate_entry()
1592 reloc->write_domain); in eb_relocate_entry()
1593 return -EINVAL; in eb_relocate_entry()
1595 if (unlikely((reloc->write_domain | reloc->read_domains) in eb_relocate_entry()
1597 drm_dbg(&i915->drm, "reloc with read/write non-GPU domains: " in eb_relocate_entry()
1600 reloc->target_handle, in eb_relocate_entry()
1601 (int) reloc->offset, in eb_relocate_entry()
1602 reloc->read_domains, in eb_relocate_entry()
1603 reloc->write_domain); in eb_relocate_entry()
1604 return -EINVAL; in eb_relocate_entry()
1607 if (reloc->write_domain) { in eb_relocate_entry()
1608 target->flags |= EXEC_OBJECT_WRITE; in eb_relocate_entry()
1616 if (reloc->write_domain == I915_GEM_DOMAIN_INSTRUCTION && in eb_relocate_entry()
1617 IS_GEN(eb->i915, 6)) { in eb_relocate_entry()
1618 err = i915_vma_bind(target->vma, in eb_relocate_entry()
1619 target->vma->obj->cache_level, in eb_relocate_entry()
1631 gen8_canonical_addr(target->vma->node.start) == reloc->presumed_offset) in eb_relocate_entry()
1635 if (unlikely(reloc->offset > in eb_relocate_entry()
1636 ev->vma->size - (eb->reloc_cache.use_64bit_reloc ? 8 : 4))) { in eb_relocate_entry()
1637 drm_dbg(&i915->drm, "Relocation beyond object bounds: " in eb_relocate_entry()
1639 reloc->target_handle, in eb_relocate_entry()
1640 (int)reloc->offset, in eb_relocate_entry()
1641 (int)ev->vma->size); in eb_relocate_entry()
1642 return -EINVAL; in eb_relocate_entry()
1644 if (unlikely(reloc->offset & 3)) { in eb_relocate_entry()
1645 drm_dbg(&i915->drm, "Relocation not 4-byte aligned: " in eb_relocate_entry()
1647 reloc->target_handle, in eb_relocate_entry()
1648 (int)reloc->offset); in eb_relocate_entry()
1649 return -EINVAL; in eb_relocate_entry()
1660 ev->flags &= ~EXEC_OBJECT_ASYNC; in eb_relocate_entry()
1663 return relocate_entry(ev->vma, reloc, eb, target->vma); in eb_relocate_entry()
1670 const struct drm_i915_gem_exec_object2 *entry = ev->exec; in eb_relocate_vma()
1672 u64_to_user_ptr(entry->relocs_ptr); in eb_relocate_vma()
1673 unsigned long remain = entry->relocation_count; in eb_relocate_vma()
1676 return -EINVAL; in eb_relocate_vma()
1684 return -EFAULT; in eb_relocate_vma()
1704 remain = -EFAULT; in eb_relocate_vma()
1708 remain -= count; in eb_relocate_vma()
1740 &urelocs[r - stack].presumed_offset); in eb_relocate_vma()
1742 } while (r++, --count); in eb_relocate_vma()
1746 reloc_cache_reset(&eb->reloc_cache, eb); in eb_relocate_vma()
1753 const struct drm_i915_gem_exec_object2 *entry = ev->exec; in eb_relocate_vma_slow()
1755 u64_to_ptr(typeof(*relocs), entry->relocs_ptr); in eb_relocate_vma_slow()
1759 for (i = 0; i < entry->relocation_count; i++) { in eb_relocate_vma_slow()
1769 reloc_cache_reset(&eb->reloc_cache, eb); in eb_relocate_vma_slow()
1779 size = entry->relocation_count; in check_relocations()
1784 return -EINVAL; in check_relocations()
1786 addr = u64_to_user_ptr(entry->relocs_ptr); in check_relocations()
1789 return -EFAULT; in check_relocations()
1797 return __get_user(c, end - 1); in check_relocations()
1803 const unsigned int count = eb->buffer_count; in eb_copy_relocations()
1808 const unsigned int nreloc = eb->exec[i].relocation_count; in eb_copy_relocations()
1816 err = check_relocations(&eb->exec[i]); in eb_copy_relocations()
1820 urelocs = u64_to_user_ptr(eb->exec[i].relocs_ptr); in eb_copy_relocations()
1825 err = -ENOMEM; in eb_copy_relocations()
1833 min_t(u64, BIT_ULL(31), size - copied); in eb_copy_relocations()
1849 * presumed_offset before the next execbuffer - if that in eb_copy_relocations()
1857 unsafe_put_user(-1, in eb_copy_relocations()
1862 eb->exec[i].relocs_ptr = (uintptr_t)relocs; in eb_copy_relocations()
1871 err = -EFAULT; in eb_copy_relocations()
1873 while (i--) { in eb_copy_relocations()
1874 relocs = u64_to_ptr(typeof(*relocs), eb->exec[i].relocs_ptr); in eb_copy_relocations()
1875 if (eb->exec[i].relocation_count) in eb_copy_relocations()
1883 const unsigned int count = eb->buffer_count; in eb_prefault_relocations()
1889 err = check_relocations(&eb->exec[i]); in eb_prefault_relocations()
1906 err = -ERESTARTSYS; in eb_relocate_parse_slow()
1912 i915_gem_ww_ctx_fini(&eb->ww); in eb_relocate_parse_slow()
1921 err = -EINTR; in eb_relocate_parse_slow()
1932 * 1 - we try to just prefault all the user relocation entries and in eb_relocate_parse_slow()
1935 * 2 - we copy the user entries to a local buffer here outside of the in eb_relocate_parse_slow()
1939 * 3 - we already have a local copy of the relocation entries, but in eb_relocate_parse_slow()
1953 flush_workqueue(eb->i915->mm.userptr_wq); in eb_relocate_parse_slow()
1956 i915_gem_ww_ctx_init(&eb->ww, true); in eb_relocate_parse_slow()
1976 GEM_BUG_ON(!eb->batch); in eb_relocate_parse_slow()
1978 list_for_each_entry(ev, &eb->relocs, reloc_link) { in eb_relocate_parse_slow()
1992 if (err == -EDEADLK) in eb_relocate_parse_slow()
2014 if (err == -EDEADLK) { in eb_relocate_parse_slow()
2016 err = i915_gem_ww_ctx_backoff(&eb->ww); in eb_relocate_parse_slow()
2021 if (err == -EAGAIN) in eb_relocate_parse_slow()
2026 const unsigned int count = eb->buffer_count; in eb_relocate_parse_slow()
2031 &eb->exec[i]; in eb_relocate_parse_slow()
2034 if (!entry->relocation_count) in eb_relocate_parse_slow()
2037 relocs = u64_to_ptr(typeof(*relocs), entry->relocs_ptr); in eb_relocate_parse_slow()
2059 if (err != -EDEADLK) in eb_relocate_parse()
2066 bool nonblock = eb->file->filp->f_flags & O_NONBLOCK; in eb_relocate_parse()
2070 if (err == -ETIME) { in eb_relocate_parse()
2072 err = -EWOULDBLOCK; in eb_relocate_parse()
2086 if (err == -EAGAIN) in eb_relocate_parse()
2092 if (eb->args->flags & __EXEC_HAS_RELOC) { in eb_relocate_parse()
2095 list_for_each_entry(ev, &eb->relocs, reloc_link) { in eb_relocate_parse()
2101 if (err == -EDEADLK) in eb_relocate_parse()
2111 if (err == -EDEADLK) { in eb_relocate_parse()
2113 err = i915_gem_ww_ctx_backoff(&eb->ww); in eb_relocate_parse()
2130 eb->args->flags &= ~__EXEC_HAS_RELOC; in eb_relocate_parse()
2137 const unsigned int count = eb->buffer_count; in eb_move_to_gpu()
2141 while (i--) { in eb_move_to_gpu()
2142 struct eb_vma *ev = &eb->vma[i]; in eb_move_to_gpu()
2143 struct i915_vma *vma = ev->vma; in eb_move_to_gpu()
2144 unsigned int flags = ev->flags; in eb_move_to_gpu()
2145 struct drm_i915_gem_object *obj = vma->obj; in eb_move_to_gpu()
2154 capture->next = eb->request->capture_list; in eb_move_to_gpu()
2155 capture->vma = vma; in eb_move_to_gpu()
2156 eb->request->capture_list = capture; in eb_move_to_gpu()
2167 * obj->cache_dirty && in eb_move_to_gpu()
2168 * !(obj->cache_coherent & I915_BO_CACHE_COHERENT_FOR_READ) in eb_move_to_gpu()
2172 if (unlikely(obj->cache_dirty & ~obj->cache_coherent)) { in eb_move_to_gpu()
2179 (eb->request, obj, flags & EXEC_OBJECT_WRITE); in eb_move_to_gpu()
2183 err = i915_vma_move_to_active(vma, eb->request, flags); in eb_move_to_gpu()
2190 intel_gt_chipset_flush(eb->engine->gt); in eb_move_to_gpu()
2194 i915_request_set_error_once(eb->request, err); in eb_move_to_gpu()
2200 if (exec->flags & __I915_EXEC_ILLEGAL_FLAGS) in i915_gem_check_execbuffer()
2201 return -EINVAL; in i915_gem_check_execbuffer()
2204 if (!(exec->flags & (I915_EXEC_FENCE_ARRAY | in i915_gem_check_execbuffer()
2206 if (exec->num_cliprects || exec->cliprects_ptr) in i915_gem_check_execbuffer()
2207 return -EINVAL; in i915_gem_check_execbuffer()
2210 if (exec->DR4 == 0xffffffff) { in i915_gem_check_execbuffer()
2212 exec->DR4 = 0; in i915_gem_check_execbuffer()
2214 if (exec->DR1 || exec->DR4) in i915_gem_check_execbuffer()
2215 return -EINVAL; in i915_gem_check_execbuffer()
2217 if ((exec->batch_start_offset | exec->batch_len) & 0x7) in i915_gem_check_execbuffer()
2218 return -EINVAL; in i915_gem_check_execbuffer()
2228 if (!IS_GEN(rq->engine->i915, 7) || rq->engine->id != RCS0) { in i915_reset_gen7_sol_offsets()
2229 drm_dbg(&rq->engine->i915->drm, "sol reset is gen7/rcs only\n"); in i915_reset_gen7_sol_offsets()
2230 return -EINVAL; in i915_reset_gen7_sol_offsets()
2261 err = i915_vma_pin_ww(vma, &eb->ww, 0, 0, flags); in shadow_batch_pin()
2282 return intel_engine_cmd_parser(pw->engine, in __eb_parse()
2283 pw->batch, in __eb_parse()
2284 pw->batch_offset, in __eb_parse()
2285 pw->batch_length, in __eb_parse()
2286 pw->shadow, in __eb_parse()
2287 pw->trampoline); in __eb_parse()
2294 if (pw->trampoline) in __eb_parse_release()
2295 i915_active_release(&pw->trampoline->active); in __eb_parse_release()
2296 i915_active_release(&pw->shadow->active); in __eb_parse_release()
2297 i915_active_release(&pw->batch->active); in __eb_parse_release()
2311 struct intel_gt_buffer_pool_node *node = vma->private; in __parser_mark_active()
2313 return i915_active_ref(&node->active, tl->fence_context, fence); in __parser_mark_active()
2321 mutex_lock(&tl->mutex); in parser_mark_active()
2323 err = __parser_mark_active(pw->shadow, tl, &pw->base.dma); in parser_mark_active()
2327 if (pw->trampoline) { in parser_mark_active()
2328 err = __parser_mark_active(pw->trampoline, tl, &pw->base.dma); in parser_mark_active()
2334 mutex_unlock(&tl->mutex); in parser_mark_active()
2345 GEM_BUG_ON(overflows_type(eb->batch_start_offset, pw->batch_offset)); in eb_parse_pipeline()
2346 GEM_BUG_ON(overflows_type(eb->batch_len, pw->batch_length)); in eb_parse_pipeline()
2350 return -ENOMEM; in eb_parse_pipeline()
2352 err = i915_active_acquire(&eb->batch->vma->active); in eb_parse_pipeline()
2356 err = i915_active_acquire(&shadow->active); in eb_parse_pipeline()
2361 err = i915_active_acquire(&trampoline->active); in eb_parse_pipeline()
2366 dma_fence_work_init(&pw->base, &eb_parse_ops); in eb_parse_pipeline()
2368 pw->engine = eb->engine; in eb_parse_pipeline()
2369 pw->batch = eb->batch->vma; in eb_parse_pipeline()
2370 pw->batch_offset = eb->batch_start_offset; in eb_parse_pipeline()
2371 pw->batch_length = eb->batch_len; in eb_parse_pipeline()
2372 pw->shadow = shadow; in eb_parse_pipeline()
2373 pw->trampoline = trampoline; in eb_parse_pipeline()
2376 err = parser_mark_active(pw, eb->context->timeline); in eb_parse_pipeline()
2380 err = dma_resv_reserve_shared(pw->batch->resv, 1); in eb_parse_pipeline()
2385 err = i915_sw_fence_await_reservation(&pw->base.chain, in eb_parse_pipeline()
2386 pw->batch->resv, NULL, false, in eb_parse_pipeline()
2392 dma_resv_add_shared_fence(pw->batch->resv, &pw->base.dma); in eb_parse_pipeline()
2395 dma_resv_add_excl_fence(shadow->resv, &pw->base.dma); in eb_parse_pipeline()
2397 dma_fence_work_commit_imm(&pw->base); in eb_parse_pipeline()
2401 i915_sw_fence_set_error_once(&pw->base.chain, err); in eb_parse_pipeline()
2402 dma_fence_work_commit_imm(&pw->base); in eb_parse_pipeline()
2406 i915_active_release(&shadow->active); in eb_parse_pipeline()
2408 i915_active_release(&eb->batch->vma->active); in eb_parse_pipeline()
2417 * snb/ivb/vlv conflate the "batch in ppgtt" bit with the "non-secure in eb_dispatch_secure()
2420 if (eb->batch_flags & I915_DISPATCH_SECURE) in eb_dispatch_secure()
2421 return i915_gem_object_ggtt_pin_ww(vma->obj, &eb->ww, NULL, 0, 0, 0); in eb_dispatch_secure()
2428 struct drm_i915_private *i915 = eb->i915; in eb_parse()
2429 struct intel_gt_buffer_pool_node *pool = eb->batch_pool; in eb_parse()
2435 batch = eb_dispatch_secure(eb, eb->batch->vma); in eb_parse()
2442 len = eb->batch_len; in eb_parse()
2443 if (!CMDPARSER_USES_GGTT(eb->i915)) { in eb_parse()
2446 * post-scan tampering in eb_parse()
2448 if (!eb->context->vm->has_read_only) { in eb_parse()
2449 drm_dbg(&i915->drm, in eb_parse()
2450 "Cannot prevent post-scan tampering without RO capable vm\n"); in eb_parse()
2451 return -EINVAL; in eb_parse()
2456 if (unlikely(len < eb->batch_len)) /* last paranoid check of overflow */ in eb_parse()
2457 return -EINVAL; in eb_parse()
2460 pool = intel_gt_get_buffer_pool(eb->engine->gt, len); in eb_parse()
2463 eb->batch_pool = pool; in eb_parse()
2466 err = i915_gem_object_lock(pool->obj, &eb->ww); in eb_parse()
2470 shadow = shadow_batch_pin(eb, pool->obj, eb->context->vm, PIN_USER); in eb_parse()
2475 i915_gem_object_set_readonly(shadow->obj); in eb_parse()
2476 shadow->private = pool; in eb_parse()
2479 if (CMDPARSER_USES_GGTT(eb->i915)) { in eb_parse()
2482 shadow = shadow_batch_pin(eb, pool->obj, in eb_parse()
2483 &eb->engine->gt->ggtt->vm, in eb_parse()
2490 shadow->private = pool; in eb_parse()
2492 eb->batch_flags |= I915_DISPATCH_SECURE; in eb_parse()
2505 eb->batch = &eb->vma[eb->buffer_count++]; in eb_parse()
2506 eb->batch->vma = i915_vma_get(shadow); in eb_parse()
2507 eb->batch->flags = __EXEC_OBJECT_HAS_PIN; in eb_parse()
2509 eb->trampoline = trampoline; in eb_parse()
2510 eb->batch_start_offset = 0; in eb_parse()
2514 eb->batch = &eb->vma[eb->buffer_count++]; in eb_parse()
2515 eb->batch->flags = __EXEC_OBJECT_HAS_PIN; in eb_parse()
2516 eb->batch->vma = i915_vma_get(batch); in eb_parse()
2540 if (eb->args->flags & I915_EXEC_GEN7_SOL_RESET) { in eb_submit()
2541 err = i915_reset_gen7_sol_offsets(eb->request); in eb_submit()
2552 if (eb->engine->emit_init_breadcrumb) { in eb_submit()
2553 err = eb->engine->emit_init_breadcrumb(eb->request); in eb_submit()
2558 err = eb->engine->emit_bb_start(eb->request, in eb_submit()
2559 batch->node.start + in eb_submit()
2560 eb->batch_start_offset, in eb_submit()
2561 eb->batch_len, in eb_submit()
2562 eb->batch_flags); in eb_submit()
2566 if (eb->trampoline) { in eb_submit()
2567 GEM_BUG_ON(eb->batch_start_offset); in eb_submit()
2568 err = eb->engine->emit_bb_start(eb->request, in eb_submit()
2569 eb->trampoline->node.start + in eb_submit()
2570 eb->batch_len, in eb_submit()
2576 if (intel_context_nopreempt(eb->context)) in eb_submit()
2577 __set_bit(I915_FENCE_FLAG_NOPREEMPT, &eb->request->fence.flags); in eb_submit()
2584 return hweight64(VDBOX_MASK(&i915->gt)); in num_vcs_engines()
2595 struct drm_i915_file_private *file_priv = file->driver_priv; in gen8_dispatch_bsd_engine()
2598 if ((int)file_priv->bsd_engine < 0) in gen8_dispatch_bsd_engine()
2599 file_priv->bsd_engine = in gen8_dispatch_bsd_engine()
2602 return file_priv->bsd_engine; in gen8_dispatch_bsd_engine()
2615 struct intel_ring *ring = ce->ring; in eb_throttle()
2616 struct intel_timeline *tl = ce->timeline; in eb_throttle()
2620 * Completely unscientific finger-in-the-air estimates for suitable in eb_throttle()
2633 list_for_each_entry(rq, &tl->requests, link) { in eb_throttle()
2634 if (rq->ring != ring) in eb_throttle()
2637 if (__intel_ring_space(rq->postfix, in eb_throttle()
2638 ring->emit, ring->size) > ring->size / 2) in eb_throttle()
2641 if (&rq->link == &tl->requests) in eb_throttle()
2649 struct intel_context *ce = eb->context; in eb_pin_engine()
2654 GEM_BUG_ON(eb->args->flags & __EXEC_ENGINE_PINNED); in eb_pin_engine()
2657 return ERR_PTR(-EIO); in eb_pin_engine()
2664 err = intel_context_pin_ww(ce, &eb->ww); in eb_pin_engine()
2687 eb->args->flags |= __EXEC_ENGINE_PINNED; in eb_pin_engine()
2693 struct intel_context *ce = eb->context; in eb_unpin_engine()
2694 struct intel_timeline *tl = ce->timeline; in eb_unpin_engine()
2696 if (!(eb->args->flags & __EXEC_ENGINE_PINNED)) in eb_unpin_engine()
2699 eb->args->flags &= ~__EXEC_ENGINE_PINNED; in eb_unpin_engine()
2701 mutex_lock(&tl->mutex); in eb_unpin_engine()
2703 mutex_unlock(&tl->mutex); in eb_unpin_engine()
2711 struct drm_i915_private *i915 = eb->i915; in eb_select_legacy_ring()
2712 struct drm_i915_gem_execbuffer2 *args = eb->args; in eb_select_legacy_ring()
2713 unsigned int user_ring_id = args->flags & I915_EXEC_RING_MASK; in eb_select_legacy_ring()
2716 (args->flags & I915_EXEC_BSD_MASK)) { in eb_select_legacy_ring()
2717 drm_dbg(&i915->drm, in eb_select_legacy_ring()
2719 "bsd dispatch flags: %d\n", (int)(args->flags)); in eb_select_legacy_ring()
2720 return -1; in eb_select_legacy_ring()
2724 unsigned int bsd_idx = args->flags & I915_EXEC_BSD_MASK; in eb_select_legacy_ring()
2727 bsd_idx = gen8_dispatch_bsd_engine(i915, eb->file); in eb_select_legacy_ring()
2731 bsd_idx--; in eb_select_legacy_ring()
2733 drm_dbg(&i915->drm, in eb_select_legacy_ring()
2736 return -1; in eb_select_legacy_ring()
2743 drm_dbg(&i915->drm, "execbuf with unknown ring: %u\n", in eb_select_legacy_ring()
2745 return -1; in eb_select_legacy_ring()
2758 if (i915_gem_context_user_engines(eb->gem_context)) in eb_select_engine()
2759 idx = eb->args->flags & I915_EXEC_RING_MASK; in eb_select_engine()
2763 ce = i915_gem_context_get_engine(eb->gem_context, idx); in eb_select_engine()
2767 intel_gt_pm_get(ce->engine->gt); in eb_select_engine()
2769 if (!test_bit(CONTEXT_ALLOC_BIT, &ce->flags)) { in eb_select_engine()
2779 err = intel_gt_terminally_wedged(ce->engine->gt); in eb_select_engine()
2783 eb->context = ce; in eb_select_engine()
2784 eb->engine = ce->engine; in eb_select_engine()
2789 * is dropped, which breaks our -EDEADLK handling. in eb_select_engine()
2794 intel_gt_pm_put(ce->engine->gt); in eb_select_engine()
2802 intel_gt_pm_put(eb->engine->gt); in eb_put_engine()
2803 intel_context_put(eb->context); in eb_put_engine()
2809 while (n--) { in __free_fence_array()
2827 nfences = timeline_fences->fence_count; in add_timeline_fence_array()
2835 SIZE_MAX / sizeof(*f)) - eb->num_fences) in add_timeline_fence_array()
2836 return -EINVAL; in add_timeline_fence_array()
2838 user_fences = u64_to_user_ptr(timeline_fences->handles_ptr); in add_timeline_fence_array()
2840 return -EFAULT; in add_timeline_fence_array()
2842 user_values = u64_to_user_ptr(timeline_fences->values_ptr); in add_timeline_fence_array()
2844 return -EFAULT; in add_timeline_fence_array()
2846 f = krealloc(eb->fences, in add_timeline_fence_array()
2847 (eb->num_fences + nfences) * sizeof(*f), in add_timeline_fence_array()
2850 return -ENOMEM; in add_timeline_fence_array()
2852 eb->fences = f; in add_timeline_fence_array()
2853 f += eb->num_fences; in add_timeline_fence_array()
2855 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & in add_timeline_fence_array()
2858 while (nfences--) { in add_timeline_fence_array()
2867 return -EFAULT; in add_timeline_fence_array()
2870 return -EINVAL; in add_timeline_fence_array()
2873 return -EFAULT; in add_timeline_fence_array()
2875 syncobj = drm_syncobj_find(eb->file, user_fence.handle); in add_timeline_fence_array()
2878 return -ENOENT; in add_timeline_fence_array()
2887 return -EINVAL; in add_timeline_fence_array()
2923 return -EINVAL; in add_timeline_fence_array()
2926 f->chain_fence = in add_timeline_fence_array()
2927 kmalloc(sizeof(*f->chain_fence), in add_timeline_fence_array()
2929 if (!f->chain_fence) { in add_timeline_fence_array()
2932 return -ENOMEM; in add_timeline_fence_array()
2935 f->chain_fence = NULL; in add_timeline_fence_array()
2938 f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2); in add_timeline_fence_array()
2939 f->dma_fence = fence; in add_timeline_fence_array()
2940 f->value = point; in add_timeline_fence_array()
2942 eb->num_fences++; in add_timeline_fence_array()
2950 struct drm_i915_gem_execbuffer2 *args = eb->args; in add_fence_array()
2952 unsigned long num_fences = args->num_cliprects; in add_fence_array()
2955 if (!(args->flags & I915_EXEC_FENCE_ARRAY)) in add_fence_array()
2965 SIZE_MAX / sizeof(*f) - eb->num_fences)) in add_fence_array()
2966 return -EINVAL; in add_fence_array()
2968 user = u64_to_user_ptr(args->cliprects_ptr); in add_fence_array()
2970 return -EFAULT; in add_fence_array()
2972 f = krealloc(eb->fences, in add_fence_array()
2973 (eb->num_fences + num_fences) * sizeof(*f), in add_fence_array()
2976 return -ENOMEM; in add_fence_array()
2978 eb->fences = f; in add_fence_array()
2979 f += eb->num_fences; in add_fence_array()
2980 while (num_fences--) { in add_fence_array()
2986 return -EFAULT; in add_fence_array()
2989 return -EINVAL; in add_fence_array()
2991 syncobj = drm_syncobj_find(eb->file, user_fence.handle); in add_fence_array()
2994 return -ENOENT; in add_fence_array()
3002 return -EINVAL; in add_fence_array()
3006 BUILD_BUG_ON(~(ARCH_KMALLOC_MINALIGN - 1) & in add_fence_array()
3009 f->syncobj = ptr_pack_bits(syncobj, user_fence.flags, 2); in add_fence_array()
3010 f->dma_fence = fence; in add_fence_array()
3011 f->value = 0; in add_fence_array()
3012 f->chain_fence = NULL; in add_fence_array()
3014 eb->num_fences++; in add_fence_array()
3032 for (n = 0; n < eb->num_fences; n++) { in await_fence_array()
3036 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); in await_fence_array()
3038 if (!eb->fences[n].dma_fence) in await_fence_array()
3041 err = i915_request_await_dma_fence(eb->request, in await_fence_array()
3042 eb->fences[n].dma_fence); in await_fence_array()
3052 struct dma_fence * const fence = &eb->request->fence; in signal_fence_array()
3055 for (n = 0; n < eb->num_fences; n++) { in signal_fence_array()
3059 syncobj = ptr_unpack_bits(eb->fences[n].syncobj, &flags, 2); in signal_fence_array()
3063 if (eb->fences[n].chain_fence) { in signal_fence_array()
3065 eb->fences[n].chain_fence, in signal_fence_array()
3067 eb->fences[n].value); in signal_fence_array()
3072 eb->fences[n].chain_fence = NULL; in signal_fence_array()
3080 parse_timeline_fences(struct i915_user_extension __user *ext, void *data) in parse_timeline_fences() argument
3085 if (copy_from_user(&timeline_fences, ext, sizeof(timeline_fences))) in parse_timeline_fences()
3086 return -EFAULT; in parse_timeline_fences()
3095 list_for_each_entry_safe(rq, rn, &tl->requests, link) in retire_requests()
3102 struct i915_request *rq = eb->request; in eb_request_add()
3107 lockdep_assert_held(&tl->mutex); in eb_request_add()
3108 lockdep_unpin_lock(&tl->mutex, rq->cookie); in eb_request_add()
3115 if (likely(!intel_context_is_closed(eb->context))) { in eb_request_add()
3116 attr = eb->gem_context->sched; in eb_request_add()
3119 i915_request_set_error_once(rq, -ENOENT); in eb_request_add()
3121 err = -ENOENT; /* override any transient errors */ in eb_request_add()
3130 mutex_unlock(&tl->mutex); in eb_request_add()
3143 if (!(args->flags & I915_EXEC_USE_EXTENSIONS)) in parse_execbuf2_extensions()
3149 if (eb->args->flags & I915_EXEC_FENCE_ARRAY) in parse_execbuf2_extensions()
3150 return -EINVAL; in parse_execbuf2_extensions()
3152 if (args->num_cliprects != 0) in parse_execbuf2_extensions()
3153 return -EINVAL; in parse_execbuf2_extensions()
3155 return i915_user_extensions(u64_to_user_ptr(args->cliprects_ptr), in parse_execbuf2_extensions()
3172 int out_fence_fd = -1; in i915_gem_do_execbuffer()
3182 if (DBG_FORCE_RELOC || !(args->flags & I915_EXEC_NO_RELOC)) in i915_gem_do_execbuffer()
3183 args->flags |= __EXEC_HAS_RELOC; in i915_gem_do_execbuffer()
3186 eb.vma = (struct eb_vma *)(exec + args->buffer_count + 1); in i915_gem_do_execbuffer()
3194 eb.buffer_count = args->buffer_count; in i915_gem_do_execbuffer()
3195 eb.batch_start_offset = args->batch_start_offset; in i915_gem_do_execbuffer()
3196 eb.batch_len = args->batch_len; in i915_gem_do_execbuffer()
3203 if (args->flags & I915_EXEC_SECURE) { in i915_gem_do_execbuffer()
3205 return -ENODEV; in i915_gem_do_execbuffer()
3207 /* Return -EPERM to trigger fallback code on old binaries. */ in i915_gem_do_execbuffer()
3209 return -EPERM; in i915_gem_do_execbuffer()
3212 return -EPERM; in i915_gem_do_execbuffer()
3216 if (args->flags & I915_EXEC_IS_PINNED) in i915_gem_do_execbuffer()
3228 if (args->flags & IN_FENCES) { in i915_gem_do_execbuffer()
3229 if ((args->flags & IN_FENCES) == IN_FENCES) in i915_gem_do_execbuffer()
3230 return -EINVAL; in i915_gem_do_execbuffer()
3232 in_fence = sync_file_get_fence(lower_32_bits(args->rsvd2)); in i915_gem_do_execbuffer()
3234 err = -EINVAL; in i915_gem_do_execbuffer()
3240 if (args->flags & I915_EXEC_FENCE_OUT) { in i915_gem_do_execbuffer()
3279 args->flags &= ~__EXEC_HAS_RELOC; in i915_gem_do_execbuffer()
3285 batch = eb.batch->vma; in i915_gem_do_execbuffer()
3298 if (args->flags & I915_EXEC_FENCE_SUBMIT) in i915_gem_do_execbuffer()
3301 eb.engine->bond_execute); in i915_gem_do_execbuffer()
3315 if (out_fence_fd != -1) { in i915_gem_do_execbuffer()
3316 out_fence = sync_file_create(&eb.request->fence); in i915_gem_do_execbuffer()
3318 err = -ENOMEM; in i915_gem_do_execbuffer()
3330 eb.request->batch = batch; in i915_gem_do_execbuffer()
3345 fd_install(out_fence_fd, out_fence->file); in i915_gem_do_execbuffer()
3346 args->rsvd2 &= GENMASK_ULL(31, 0); /* keep in-fence */ in i915_gem_do_execbuffer()
3347 args->rsvd2 |= (u64)out_fence_fd << 32; in i915_gem_do_execbuffer()
3348 out_fence_fd = -1; in i915_gem_do_execbuffer()
3350 fput(out_fence->file); in i915_gem_do_execbuffer()
3359 WARN_ON(err == -EDEADLK); in i915_gem_do_execbuffer()
3375 if (out_fence_fd != -1) in i915_gem_do_execbuffer()
3399 return !(count < 1 || count > INT_MAX || count > SIZE_MAX / sz - 1); in check_buffer_count()
3415 const size_t count = args->buffer_count; in i915_gem_execbuffer_ioctl()
3420 drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count); in i915_gem_execbuffer_ioctl()
3421 return -EINVAL; in i915_gem_execbuffer_ioctl()
3424 exec2.buffers_ptr = args->buffers_ptr; in i915_gem_execbuffer_ioctl()
3425 exec2.buffer_count = args->buffer_count; in i915_gem_execbuffer_ioctl()
3426 exec2.batch_start_offset = args->batch_start_offset; in i915_gem_execbuffer_ioctl()
3427 exec2.batch_len = args->batch_len; in i915_gem_execbuffer_ioctl()
3428 exec2.DR1 = args->DR1; in i915_gem_execbuffer_ioctl()
3429 exec2.DR4 = args->DR4; in i915_gem_execbuffer_ioctl()
3430 exec2.num_cliprects = args->num_cliprects; in i915_gem_execbuffer_ioctl()
3431 exec2.cliprects_ptr = args->cliprects_ptr; in i915_gem_execbuffer_ioctl()
3447 drm_dbg(&i915->drm, in i915_gem_execbuffer_ioctl()
3449 args->buffer_count); in i915_gem_execbuffer_ioctl()
3452 return -ENOMEM; in i915_gem_execbuffer_ioctl()
3455 u64_to_user_ptr(args->buffers_ptr), in i915_gem_execbuffer_ioctl()
3458 drm_dbg(&i915->drm, "copy %d exec entries failed %d\n", in i915_gem_execbuffer_ioctl()
3459 args->buffer_count, err); in i915_gem_execbuffer_ioctl()
3462 return -EFAULT; in i915_gem_execbuffer_ioctl()
3465 for (i = 0; i < args->buffer_count; i++) { in i915_gem_execbuffer_ioctl()
3480 u64_to_user_ptr(args->buffers_ptr); in i915_gem_execbuffer_ioctl()
3483 for (i = 0; i < args->buffer_count; i++) { in i915_gem_execbuffer_ioctl()
3509 const size_t count = args->buffer_count; in i915_gem_execbuffer2_ioctl()
3513 drm_dbg(&i915->drm, "execbuf2 with %zd buffers\n", count); in i915_gem_execbuffer2_ioctl()
3514 return -EINVAL; in i915_gem_execbuffer2_ioctl()
3525 drm_dbg(&i915->drm, "Failed to allocate exec list for %zd buffers\n", in i915_gem_execbuffer2_ioctl()
3527 return -ENOMEM; in i915_gem_execbuffer2_ioctl()
3530 u64_to_user_ptr(args->buffers_ptr), in i915_gem_execbuffer2_ioctl()
3532 drm_dbg(&i915->drm, "copy %zd exec entries failed\n", count); in i915_gem_execbuffer2_ioctl()
3534 return -EFAULT; in i915_gem_execbuffer2_ioctl()
3545 if (args->flags & __EXEC_HAS_RELOC) { in i915_gem_execbuffer2_ioctl()
3547 u64_to_user_ptr(args->buffers_ptr); in i915_gem_execbuffer2_ioctl()
3562 for (i = 0; i < args->buffer_count; i++) { in i915_gem_execbuffer2_ioctl()
3577 args->flags &= ~__I915_EXEC_UNKNOWN_FLAGS; in i915_gem_execbuffer2_ioctl()