1 /*
2  * Copyright © 2016 Intel Corporation
3  *
4  * Permission is hereby granted, free of charge, to any person obtaining a
5  * copy of this software and associated documentation files (the "Software"),
6  * to deal in the Software without restriction, including without limitation
7  * the rights to use, copy, modify, merge, publish, distribute, sublicense,
8  * and/or sell copies of the Software, and to permit persons to whom the
9  * Software is furnished to do so, subject to the following conditions:
10  *
11  * The above copyright notice and this permission notice (including the next
12  * paragraph) shall be included in all copies or substantial portions of the
13  * Software.
14  *
15  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16  * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17  * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.  IN NO EVENT SHALL
18  * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19  * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
20  * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
21  * IN THE SOFTWARE.
22  *
23  */
24 
25 #include <linux/prime_numbers.h>
26 
27 #include "gem/i915_gem_context.h"
28 #include "gem/i915_gem_internal.h"
29 #include "gem/selftests/mock_context.h"
30 
31 #include "i915_scatterlist.h"
32 #include "i915_selftest.h"
33 
34 #include "mock_gem_device.h"
35 #include "mock_gtt.h"
36 
assert_vma(struct i915_vma * vma,struct drm_i915_gem_object * obj,struct i915_gem_context * ctx)37 static bool assert_vma(struct i915_vma *vma,
38 		       struct drm_i915_gem_object *obj,
39 		       struct i915_gem_context *ctx)
40 {
41 	bool ok = true;
42 
43 	if (vma->vm != ctx->vm) {
44 		pr_err("VMA created with wrong VM\n");
45 		ok = false;
46 	}
47 
48 	if (vma->size != obj->base.size) {
49 		pr_err("VMA created with wrong size, found %llu, expected %zu\n",
50 		       vma->size, obj->base.size);
51 		ok = false;
52 	}
53 
54 	if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) {
55 		pr_err("VMA created with wrong type [%d]\n",
56 		       vma->gtt_view.type);
57 		ok = false;
58 	}
59 
60 	return ok;
61 }
62 
63 static struct i915_vma *
checked_vma_instance(struct drm_i915_gem_object * obj,struct i915_address_space * vm,const struct i915_gtt_view * view)64 checked_vma_instance(struct drm_i915_gem_object *obj,
65 		     struct i915_address_space *vm,
66 		     const struct i915_gtt_view *view)
67 {
68 	struct i915_vma *vma;
69 	bool ok = true;
70 
71 	vma = i915_vma_instance(obj, vm, view);
72 	if (IS_ERR(vma))
73 		return vma;
74 
75 	/* Manual checks, will be reinforced by i915_vma_compare! */
76 	if (vma->vm != vm) {
77 		pr_err("VMA's vm [%p] does not match request [%p]\n",
78 		       vma->vm, vm);
79 		ok = false;
80 	}
81 
82 	if (i915_is_ggtt(vm) != i915_vma_is_ggtt(vma)) {
83 		pr_err("VMA ggtt status [%d] does not match parent [%d]\n",
84 		       i915_vma_is_ggtt(vma), i915_is_ggtt(vm));
85 		ok = false;
86 	}
87 
88 	if (i915_vma_compare(vma, vm, view)) {
89 		pr_err("i915_vma_compare failed with create parameters!\n");
90 		return ERR_PTR(-EINVAL);
91 	}
92 
93 	if (i915_vma_compare(vma, vma->vm,
94 			     i915_vma_is_ggtt(vma) ? &vma->gtt_view : NULL)) {
95 		pr_err("i915_vma_compare failed with itself\n");
96 		return ERR_PTR(-EINVAL);
97 	}
98 
99 	if (!ok) {
100 		pr_err("i915_vma_compare failed to detect the difference!\n");
101 		return ERR_PTR(-EINVAL);
102 	}
103 
104 	return vma;
105 }
106 
create_vmas(struct drm_i915_private * i915,struct list_head * objects,struct list_head * contexts)107 static int create_vmas(struct drm_i915_private *i915,
108 		       struct list_head *objects,
109 		       struct list_head *contexts)
110 {
111 	struct drm_i915_gem_object *obj;
112 	struct i915_gem_context *ctx;
113 	int pinned;
114 
115 	list_for_each_entry(obj, objects, st_link) {
116 		for (pinned = 0; pinned <= 1; pinned++) {
117 			list_for_each_entry(ctx, contexts, link) {
118 				struct i915_address_space *vm;
119 				struct i915_vma *vma;
120 				int err;
121 
122 				vm = i915_gem_context_get_eb_vm(ctx);
123 				vma = checked_vma_instance(obj, vm, NULL);
124 				i915_vm_put(vm);
125 				if (IS_ERR(vma))
126 					return PTR_ERR(vma);
127 
128 				if (!assert_vma(vma, obj, ctx)) {
129 					pr_err("VMA lookup/create failed\n");
130 					return -EINVAL;
131 				}
132 
133 				if (!pinned) {
134 					err = i915_vma_pin(vma, 0, 0, PIN_USER);
135 					if (err) {
136 						pr_err("Failed to pin VMA\n");
137 						return err;
138 					}
139 				} else {
140 					i915_vma_unpin(vma);
141 				}
142 			}
143 		}
144 	}
145 
146 	return 0;
147 }
148 
igt_vma_create(void * arg)149 static int igt_vma_create(void *arg)
150 {
151 	struct i915_ggtt *ggtt = arg;
152 	struct drm_i915_private *i915 = ggtt->vm.i915;
153 	struct drm_i915_gem_object *obj, *on;
154 	struct i915_gem_context *ctx, *cn;
155 	unsigned long num_obj, num_ctx;
156 	unsigned long no, nc;
157 	IGT_TIMEOUT(end_time);
158 	LIST_HEAD(contexts);
159 	LIST_HEAD(objects);
160 	int err = -ENOMEM;
161 
162 	/*
163 	 * Exercise creating many vma amongst many objections, checking the
164 	 * vma creation and lookup routines.
165 	 */
166 
167 	no = 0;
168 	for_each_prime_number(num_obj, ULONG_MAX - 1) {
169 		for (; no < num_obj; no++) {
170 			obj = i915_gem_object_create_internal(i915, PAGE_SIZE);
171 			if (IS_ERR(obj))
172 				goto out;
173 
174 			list_add(&obj->st_link, &objects);
175 		}
176 
177 		nc = 0;
178 		for_each_prime_number(num_ctx, 2 * BITS_PER_LONG) {
179 			for (; nc < num_ctx; nc++) {
180 				ctx = mock_context(i915, "mock");
181 				if (!ctx)
182 					goto out;
183 
184 				list_move(&ctx->link, &contexts);
185 			}
186 
187 			err = create_vmas(i915, &objects, &contexts);
188 			if (err)
189 				goto out;
190 
191 			if (igt_timeout(end_time,
192 					"%s timed out: after %lu objects in %lu contexts\n",
193 					__func__, no, nc))
194 				goto end;
195 		}
196 
197 		list_for_each_entry_safe(ctx, cn, &contexts, link) {
198 			list_del_init(&ctx->link);
199 			mock_context_close(ctx);
200 		}
201 
202 		cond_resched();
203 	}
204 
205 end:
206 	/* Final pass to lookup all created contexts */
207 	err = create_vmas(i915, &objects, &contexts);
208 out:
209 	list_for_each_entry_safe(ctx, cn, &contexts, link) {
210 		list_del_init(&ctx->link);
211 		mock_context_close(ctx);
212 	}
213 
214 	list_for_each_entry_safe(obj, on, &objects, st_link)
215 		i915_gem_object_put(obj);
216 	return err;
217 }
218 
219 struct pin_mode {
220 	u64 size;
221 	u64 flags;
222 	bool (*assert)(const struct i915_vma *,
223 		       const struct pin_mode *mode,
224 		       int result);
225 	const char *string;
226 };
227 
assert_pin_valid(const struct i915_vma * vma,const struct pin_mode * mode,int result)228 static bool assert_pin_valid(const struct i915_vma *vma,
229 			     const struct pin_mode *mode,
230 			     int result)
231 {
232 	if (result)
233 		return false;
234 
235 	if (i915_vma_misplaced(vma, mode->size, 0, mode->flags))
236 		return false;
237 
238 	return true;
239 }
240 
241 __maybe_unused
assert_pin_enospc(const struct i915_vma * vma,const struct pin_mode * mode,int result)242 static bool assert_pin_enospc(const struct i915_vma *vma,
243 			      const struct pin_mode *mode,
244 			      int result)
245 {
246 	return result == -ENOSPC;
247 }
248 
249 __maybe_unused
assert_pin_einval(const struct i915_vma * vma,const struct pin_mode * mode,int result)250 static bool assert_pin_einval(const struct i915_vma *vma,
251 			      const struct pin_mode *mode,
252 			      int result)
253 {
254 	return result == -EINVAL;
255 }
256 
igt_vma_pin1(void * arg)257 static int igt_vma_pin1(void *arg)
258 {
259 	struct i915_ggtt *ggtt = arg;
260 	const struct pin_mode modes[] = {
261 #define VALID(sz, fl) { .size = (sz), .flags = (fl), .assert = assert_pin_valid, .string = #sz ", " #fl ", (valid) " }
262 #define __INVALID(sz, fl, check, eval) { .size = (sz), .flags = (fl), .assert = (check), .string = #sz ", " #fl ", (invalid " #eval ")" }
263 #define INVALID(sz, fl) __INVALID(sz, fl, assert_pin_einval, EINVAL)
264 #define NOSPACE(sz, fl) __INVALID(sz, fl, assert_pin_enospc, ENOSPC)
265 		VALID(0, PIN_GLOBAL),
266 		VALID(0, PIN_GLOBAL | PIN_MAPPABLE),
267 
268 		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 4096),
269 		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | 8192),
270 		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
271 		VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
272 		VALID(0, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)),
273 
274 		VALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)),
275 		INVALID(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | ggtt->mappable_end),
276 		VALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)),
277 		INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | ggtt->vm.total),
278 		INVALID(0, PIN_GLOBAL | PIN_OFFSET_FIXED | round_down(U64_MAX, PAGE_SIZE)),
279 
280 		VALID(4096, PIN_GLOBAL),
281 		VALID(8192, PIN_GLOBAL),
282 		VALID(ggtt->mappable_end - 4096, PIN_GLOBAL | PIN_MAPPABLE),
283 		VALID(ggtt->mappable_end, PIN_GLOBAL | PIN_MAPPABLE),
284 		NOSPACE(ggtt->mappable_end + 4096, PIN_GLOBAL | PIN_MAPPABLE),
285 		VALID(ggtt->vm.total - 4096, PIN_GLOBAL),
286 		VALID(ggtt->vm.total, PIN_GLOBAL),
287 		NOSPACE(ggtt->vm.total + 4096, PIN_GLOBAL),
288 		NOSPACE(round_down(U64_MAX, PAGE_SIZE), PIN_GLOBAL),
289 		INVALID(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_FIXED | (ggtt->mappable_end - 4096)),
290 		INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (ggtt->vm.total - 4096)),
291 		INVALID(8192, PIN_GLOBAL | PIN_OFFSET_FIXED | (round_down(U64_MAX, PAGE_SIZE) - 4096)),
292 
293 		VALID(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
294 
295 #if !IS_ENABLED(CONFIG_DRM_I915_DEBUG_GEM)
296 		/*
297 		 * Misusing BIAS is a programming error (it is not controllable
298 		 * from userspace) so when debugging is enabled, it explodes.
299 		 * However, the tests are still quite interesting for checking
300 		 * variable start, end and size.
301 		 */
302 		NOSPACE(0, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | ggtt->mappable_end),
303 		NOSPACE(0, PIN_GLOBAL | PIN_OFFSET_BIAS | ggtt->vm.total),
304 		NOSPACE(8192, PIN_GLOBAL | PIN_MAPPABLE | PIN_OFFSET_BIAS | (ggtt->mappable_end - 4096)),
305 		NOSPACE(8192, PIN_GLOBAL | PIN_OFFSET_BIAS | (ggtt->vm.total - 4096)),
306 #endif
307 		{ },
308 #undef NOSPACE
309 #undef INVALID
310 #undef __INVALID
311 #undef VALID
312 	}, *m;
313 	struct drm_i915_gem_object *obj;
314 	struct i915_vma *vma;
315 	int err = -EINVAL;
316 
317 	/*
318 	 * Exercise all the weird and wonderful i915_vma_pin requests,
319 	 * focusing on error handling of boundary conditions.
320 	 */
321 
322 	GEM_BUG_ON(!drm_mm_clean(&ggtt->vm.mm));
323 
324 	obj = i915_gem_object_create_internal(ggtt->vm.i915, PAGE_SIZE);
325 	if (IS_ERR(obj))
326 		return PTR_ERR(obj);
327 
328 	vma = checked_vma_instance(obj, &ggtt->vm, NULL);
329 	if (IS_ERR(vma))
330 		goto out;
331 
332 	for (m = modes; m->assert; m++) {
333 		err = i915_vma_pin(vma, m->size, 0, m->flags);
334 		if (!m->assert(vma, m, err)) {
335 			pr_err("%s to pin single page into GGTT with mode[%d:%s]: size=%llx flags=%llx, err=%d\n",
336 			       m->assert == assert_pin_valid ? "Failed" : "Unexpectedly succeeded",
337 			       (int)(m - modes), m->string, m->size, m->flags,
338 			       err);
339 			if (!err)
340 				i915_vma_unpin(vma);
341 			err = -EINVAL;
342 			goto out;
343 		}
344 
345 		if (!err) {
346 			i915_vma_unpin(vma);
347 			err = i915_vma_unbind_unlocked(vma);
348 			if (err) {
349 				pr_err("Failed to unbind single page from GGTT, err=%d\n", err);
350 				goto out;
351 			}
352 		}
353 
354 		cond_resched();
355 	}
356 
357 	err = 0;
358 out:
359 	i915_gem_object_put(obj);
360 	return err;
361 }
362 
rotated_index(const struct intel_rotation_info * r,unsigned int n,unsigned int x,unsigned int y)363 static unsigned long rotated_index(const struct intel_rotation_info *r,
364 				   unsigned int n,
365 				   unsigned int x,
366 				   unsigned int y)
367 {
368 	return (r->plane[n].src_stride * (r->plane[n].height - y - 1) +
369 		r->plane[n].offset + x);
370 }
371 
372 static struct scatterlist *
assert_rotated(struct drm_i915_gem_object * obj,const struct intel_rotation_info * r,unsigned int n,struct scatterlist * sg)373 assert_rotated(struct drm_i915_gem_object *obj,
374 	       const struct intel_rotation_info *r, unsigned int n,
375 	       struct scatterlist *sg)
376 {
377 	unsigned int x, y;
378 
379 	for (x = 0; x < r->plane[n].width; x++) {
380 		unsigned int left;
381 
382 		for (y = 0; y < r->plane[n].height; y++) {
383 			unsigned long src_idx;
384 			dma_addr_t src;
385 
386 			if (!sg) {
387 				pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
388 				       n, x, y);
389 				return ERR_PTR(-EINVAL);
390 			}
391 
392 			src_idx = rotated_index(r, n, x, y);
393 			src = i915_gem_object_get_dma_address(obj, src_idx);
394 
395 			if (sg_dma_len(sg) != PAGE_SIZE) {
396 				pr_err("Invalid sg.length, found %d, expected %lu for rotated page (%d, %d) [src index %lu]\n",
397 				       sg_dma_len(sg), PAGE_SIZE,
398 				       x, y, src_idx);
399 				return ERR_PTR(-EINVAL);
400 			}
401 
402 			if (sg_dma_address(sg) != src) {
403 				pr_err("Invalid address for rotated page (%d, %d) [src index %lu]\n",
404 				       x, y, src_idx);
405 				return ERR_PTR(-EINVAL);
406 			}
407 
408 			sg = sg_next(sg);
409 		}
410 
411 		left = (r->plane[n].dst_stride - y) * PAGE_SIZE;
412 
413 		if (!left)
414 			continue;
415 
416 		if (!sg) {
417 			pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
418 			       n, x, y);
419 			return ERR_PTR(-EINVAL);
420 		}
421 
422 		if (sg_dma_len(sg) != left) {
423 			pr_err("Invalid sg.length, found %d, expected %u for rotated page (%d, %d)\n",
424 			       sg_dma_len(sg), left, x, y);
425 			return ERR_PTR(-EINVAL);
426 		}
427 
428 		if (sg_dma_address(sg) != 0) {
429 			pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n",
430 			       &sg_dma_address(sg), x, y);
431 			return ERR_PTR(-EINVAL);
432 		}
433 
434 		sg = sg_next(sg);
435 	}
436 
437 	return sg;
438 }
439 
remapped_index(const struct intel_remapped_info * r,unsigned int n,unsigned int x,unsigned int y)440 static unsigned long remapped_index(const struct intel_remapped_info *r,
441 				    unsigned int n,
442 				    unsigned int x,
443 				    unsigned int y)
444 {
445 	return (r->plane[n].src_stride * y +
446 		r->plane[n].offset + x);
447 }
448 
449 static struct scatterlist *
assert_remapped(struct drm_i915_gem_object * obj,const struct intel_remapped_info * r,unsigned int n,struct scatterlist * sg)450 assert_remapped(struct drm_i915_gem_object *obj,
451 		const struct intel_remapped_info *r, unsigned int n,
452 		struct scatterlist *sg)
453 {
454 	unsigned int x, y;
455 	unsigned int left = 0;
456 	unsigned int offset;
457 
458 	for (y = 0; y < r->plane[n].height; y++) {
459 		for (x = 0; x < r->plane[n].width; x++) {
460 			unsigned long src_idx;
461 			dma_addr_t src;
462 
463 			if (!sg) {
464 				pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
465 				       n, x, y);
466 				return ERR_PTR(-EINVAL);
467 			}
468 			if (!left) {
469 				offset = 0;
470 				left = sg_dma_len(sg);
471 			}
472 
473 			src_idx = remapped_index(r, n, x, y);
474 			src = i915_gem_object_get_dma_address(obj, src_idx);
475 
476 			if (left < PAGE_SIZE || left & (PAGE_SIZE-1)) {
477 				pr_err("Invalid sg.length, found %d, expected %lu for remapped page (%d, %d) [src index %lu]\n",
478 				       sg_dma_len(sg), PAGE_SIZE,
479 				       x, y, src_idx);
480 				return ERR_PTR(-EINVAL);
481 			}
482 
483 			if (sg_dma_address(sg) + offset != src) {
484 				pr_err("Invalid address for remapped page (%d, %d) [src index %lu]\n",
485 				       x, y, src_idx);
486 				return ERR_PTR(-EINVAL);
487 			}
488 
489 			left -= PAGE_SIZE;
490 			offset += PAGE_SIZE;
491 
492 
493 			if (!left)
494 				sg = sg_next(sg);
495 		}
496 
497 		if (left) {
498 			pr_err("Unexpected sg tail with %d size for remapped page (%d, %d)\n",
499 			       left,
500 			       x, y);
501 			return ERR_PTR(-EINVAL);
502 		}
503 
504 		left = (r->plane[n].dst_stride - r->plane[n].width) * PAGE_SIZE;
505 
506 		if (!left)
507 			continue;
508 
509 		if (!sg) {
510 			pr_err("Invalid sg table: too short at plane %d, (%d, %d)!\n",
511 			       n, x, y);
512 			return ERR_PTR(-EINVAL);
513 		}
514 
515 		if (sg_dma_len(sg) != left) {
516 			pr_err("Invalid sg.length, found %u, expected %u for remapped page (%d, %d)\n",
517 			       sg_dma_len(sg), left,
518 			       x, y);
519 			return ERR_PTR(-EINVAL);
520 		}
521 
522 		if (sg_dma_address(sg) != 0) {
523 			pr_err("Invalid address, found %pad, expected 0 for remapped page (%d, %d)\n",
524 			       &sg_dma_address(sg),
525 			       x, y);
526 			return ERR_PTR(-EINVAL);
527 		}
528 
529 		sg = sg_next(sg);
530 		left = 0;
531 	}
532 
533 	return sg;
534 }
535 
remapped_size(enum i915_gtt_view_type view_type,const struct intel_remapped_plane_info * a,const struct intel_remapped_plane_info * b)536 static unsigned int remapped_size(enum i915_gtt_view_type view_type,
537 				  const struct intel_remapped_plane_info *a,
538 				  const struct intel_remapped_plane_info *b)
539 {
540 
541 	if (view_type == I915_GTT_VIEW_ROTATED)
542 		return a->dst_stride * a->width + b->dst_stride * b->width;
543 	else
544 		return a->dst_stride * a->height + b->dst_stride * b->height;
545 }
546 
igt_vma_rotate_remap(void * arg)547 static int igt_vma_rotate_remap(void *arg)
548 {
549 	struct i915_ggtt *ggtt = arg;
550 	struct i915_address_space *vm = &ggtt->vm;
551 	struct drm_i915_gem_object *obj;
552 	const struct intel_remapped_plane_info planes[] = {
553 		{ .width = 1, .height = 1, .src_stride = 1 },
554 		{ .width = 2, .height = 2, .src_stride = 2 },
555 		{ .width = 4, .height = 4, .src_stride = 4 },
556 		{ .width = 8, .height = 8, .src_stride = 8 },
557 
558 		{ .width = 3, .height = 5, .src_stride = 3 },
559 		{ .width = 3, .height = 5, .src_stride = 4 },
560 		{ .width = 3, .height = 5, .src_stride = 5 },
561 
562 		{ .width = 5, .height = 3, .src_stride = 5 },
563 		{ .width = 5, .height = 3, .src_stride = 7 },
564 		{ .width = 5, .height = 3, .src_stride = 9 },
565 
566 		{ .width = 4, .height = 6, .src_stride = 6 },
567 		{ .width = 6, .height = 4, .src_stride = 6 },
568 
569 		{ .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 },
570 		{ .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 },
571 		{ .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 },
572 
573 		{ }
574 	}, *a, *b;
575 	enum i915_gtt_view_type types[] = {
576 		I915_GTT_VIEW_ROTATED,
577 		I915_GTT_VIEW_REMAPPED,
578 		0,
579 	}, *t;
580 	const unsigned int max_pages = 64;
581 	int err = -ENOMEM;
582 
583 	/*
584 	 * Create VMA for many different combinations of planes and check
585 	 * that the page layout within the rotated VMA match our expectations.
586 	 */
587 
588 	obj = i915_gem_object_create_internal(vm->i915, max_pages * PAGE_SIZE);
589 	if (IS_ERR(obj))
590 		goto out;
591 
592 	for (t = types; *t; t++) {
593 	for (a = planes; a->width; a++) {
594 		for (b = planes + ARRAY_SIZE(planes); b-- != planes; ) {
595 			struct i915_gtt_view view = {
596 				.type = *t,
597 				.remapped.plane[0] = *a,
598 				.remapped.plane[1] = *b,
599 			};
600 			struct intel_remapped_plane_info *plane_info = view.remapped.plane;
601 			unsigned int n, max_offset;
602 
603 			max_offset = max(plane_info[0].src_stride * plane_info[0].height,
604 					 plane_info[1].src_stride * plane_info[1].height);
605 			GEM_BUG_ON(max_offset > max_pages);
606 			max_offset = max_pages - max_offset;
607 
608 			if (!plane_info[0].dst_stride)
609 				plane_info[0].dst_stride = view.type == I915_GTT_VIEW_ROTATED ?
610 									plane_info[0].height :
611 									plane_info[0].width;
612 			if (!plane_info[1].dst_stride)
613 				plane_info[1].dst_stride = view.type == I915_GTT_VIEW_ROTATED ?
614 									plane_info[1].height :
615 									plane_info[1].width;
616 
617 			for_each_prime_number_from(plane_info[0].offset, 0, max_offset) {
618 				for_each_prime_number_from(plane_info[1].offset, 0, max_offset) {
619 					struct scatterlist *sg;
620 					struct i915_vma *vma;
621 					unsigned int expected_pages;
622 
623 					vma = checked_vma_instance(obj, vm, &view);
624 					if (IS_ERR(vma)) {
625 						err = PTR_ERR(vma);
626 						goto out_object;
627 					}
628 
629 					err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
630 					if (err) {
631 						pr_err("Failed to pin VMA, err=%d\n", err);
632 						goto out_object;
633 					}
634 
635 					expected_pages = remapped_size(view.type, &plane_info[0], &plane_info[1]);
636 
637 					if (view.type == I915_GTT_VIEW_ROTATED &&
638 					    vma->size != expected_pages * PAGE_SIZE) {
639 						pr_err("VMA is wrong size, expected %lu, found %llu\n",
640 						       PAGE_SIZE * expected_pages, vma->size);
641 						err = -EINVAL;
642 						goto out_object;
643 					}
644 
645 					if (view.type == I915_GTT_VIEW_REMAPPED &&
646 					    vma->size > expected_pages * PAGE_SIZE) {
647 						pr_err("VMA is wrong size, expected %lu, found %llu\n",
648 						       PAGE_SIZE * expected_pages, vma->size);
649 						err = -EINVAL;
650 						goto out_object;
651 					}
652 
653 					if (vma->pages->nents > expected_pages) {
654 						pr_err("sg table is wrong sizeo, expected %u, found %u nents\n",
655 						       expected_pages, vma->pages->nents);
656 						err = -EINVAL;
657 						goto out_object;
658 					}
659 
660 					if (vma->node.size < vma->size) {
661 						pr_err("VMA binding too small, expected %llu, found %llu\n",
662 						       vma->size, vma->node.size);
663 						err = -EINVAL;
664 						goto out_object;
665 					}
666 
667 					if (vma->pages == obj->mm.pages) {
668 						pr_err("VMA using unrotated object pages!\n");
669 						err = -EINVAL;
670 						goto out_object;
671 					}
672 
673 					sg = vma->pages->sgl;
674 					for (n = 0; n < ARRAY_SIZE(view.rotated.plane); n++) {
675 						if (view.type == I915_GTT_VIEW_ROTATED)
676 							sg = assert_rotated(obj, &view.rotated, n, sg);
677 						else
678 							sg = assert_remapped(obj, &view.remapped, n, sg);
679 						if (IS_ERR(sg)) {
680 							pr_err("Inconsistent %s VMA pages for plane %d: [(%d, %d, %d, %d, %d), (%d, %d, %d, %d, %d)]\n",
681 							       view.type == I915_GTT_VIEW_ROTATED ?
682 							       "rotated" : "remapped", n,
683 							       plane_info[0].width,
684 							       plane_info[0].height,
685 							       plane_info[0].src_stride,
686 							       plane_info[0].dst_stride,
687 							       plane_info[0].offset,
688 							       plane_info[1].width,
689 							       plane_info[1].height,
690 							       plane_info[1].src_stride,
691 							       plane_info[1].dst_stride,
692 							       plane_info[1].offset);
693 							err = -EINVAL;
694 							goto out_object;
695 						}
696 					}
697 
698 					i915_vma_unpin(vma);
699 					err = i915_vma_unbind_unlocked(vma);
700 					if (err) {
701 						pr_err("Unbinding returned %i\n", err);
702 						goto out_object;
703 					}
704 					cond_resched();
705 				}
706 			}
707 		}
708 	}
709 	}
710 
711 out_object:
712 	i915_gem_object_put(obj);
713 out:
714 	return err;
715 }
716 
assert_partial(struct drm_i915_gem_object * obj,struct i915_vma * vma,unsigned long offset,unsigned long size)717 static bool assert_partial(struct drm_i915_gem_object *obj,
718 			   struct i915_vma *vma,
719 			   unsigned long offset,
720 			   unsigned long size)
721 {
722 	struct sgt_iter sgt;
723 	dma_addr_t dma;
724 
725 	for_each_sgt_daddr(dma, sgt, vma->pages) {
726 		dma_addr_t src;
727 
728 		if (!size) {
729 			pr_err("Partial scattergather list too long\n");
730 			return false;
731 		}
732 
733 		src = i915_gem_object_get_dma_address(obj, offset);
734 		if (src != dma) {
735 			pr_err("DMA mismatch for partial page offset %lu\n",
736 			       offset);
737 			return false;
738 		}
739 
740 		offset++;
741 		size--;
742 	}
743 
744 	return true;
745 }
746 
assert_pin(struct i915_vma * vma,struct i915_gtt_view * view,u64 size,const char * name)747 static bool assert_pin(struct i915_vma *vma,
748 		       struct i915_gtt_view *view,
749 		       u64 size,
750 		       const char *name)
751 {
752 	bool ok = true;
753 
754 	if (vma->size != size) {
755 		pr_err("(%s) VMA is wrong size, expected %llu, found %llu\n",
756 		       name, size, vma->size);
757 		ok = false;
758 	}
759 
760 	if (vma->node.size < vma->size) {
761 		pr_err("(%s) VMA binding too small, expected %llu, found %llu\n",
762 		       name, vma->size, vma->node.size);
763 		ok = false;
764 	}
765 
766 	if (view && view->type != I915_GTT_VIEW_NORMAL) {
767 		if (memcmp(&vma->gtt_view, view, sizeof(*view))) {
768 			pr_err("(%s) VMA mismatch upon creation!\n",
769 			       name);
770 			ok = false;
771 		}
772 
773 		if (vma->pages == vma->obj->mm.pages) {
774 			pr_err("(%s) VMA using original object pages!\n",
775 			       name);
776 			ok = false;
777 		}
778 	} else {
779 		if (vma->gtt_view.type != I915_GTT_VIEW_NORMAL) {
780 			pr_err("Not the normal ggtt view! Found %d\n",
781 			       vma->gtt_view.type);
782 			ok = false;
783 		}
784 
785 		if (vma->pages != vma->obj->mm.pages) {
786 			pr_err("VMA not using object pages!\n");
787 			ok = false;
788 		}
789 	}
790 
791 	return ok;
792 }
793 
igt_vma_partial(void * arg)794 static int igt_vma_partial(void *arg)
795 {
796 	struct i915_ggtt *ggtt = arg;
797 	struct i915_address_space *vm = &ggtt->vm;
798 	const unsigned int npages = 1021; /* prime! */
799 	struct drm_i915_gem_object *obj;
800 	const struct phase {
801 		const char *name;
802 	} phases[] = {
803 		{ "create" },
804 		{ "lookup" },
805 		{ },
806 	}, *p;
807 	unsigned int sz, offset;
808 	struct i915_vma *vma;
809 	int err = -ENOMEM;
810 
811 	/*
812 	 * Create lots of different VMA for the object and check that
813 	 * we are returned the same VMA when we later request the same range.
814 	 */
815 
816 	obj = i915_gem_object_create_internal(vm->i915, npages * PAGE_SIZE);
817 	if (IS_ERR(obj))
818 		goto out;
819 
820 	for (p = phases; p->name; p++) { /* exercise both create/lookup */
821 		unsigned int count, nvma;
822 
823 		nvma = 0;
824 		for_each_prime_number_from(sz, 1, npages) {
825 			for_each_prime_number_from(offset, 0, npages - sz) {
826 				struct i915_gtt_view view;
827 
828 				view.type = I915_GTT_VIEW_PARTIAL;
829 				view.partial.offset = offset;
830 				view.partial.size = sz;
831 
832 				if (sz == npages)
833 					view.type = I915_GTT_VIEW_NORMAL;
834 
835 				vma = checked_vma_instance(obj, vm, &view);
836 				if (IS_ERR(vma)) {
837 					err = PTR_ERR(vma);
838 					goto out_object;
839 				}
840 
841 				err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
842 				if (err)
843 					goto out_object;
844 
845 				if (!assert_pin(vma, &view, sz*PAGE_SIZE, p->name)) {
846 					pr_err("(%s) Inconsistent partial pinning for (offset=%d, size=%d)\n",
847 					       p->name, offset, sz);
848 					err = -EINVAL;
849 					goto out_object;
850 				}
851 
852 				if (!assert_partial(obj, vma, offset, sz)) {
853 					pr_err("(%s) Inconsistent partial pages for (offset=%d, size=%d)\n",
854 					       p->name, offset, sz);
855 					err = -EINVAL;
856 					goto out_object;
857 				}
858 
859 				i915_vma_unpin(vma);
860 				nvma++;
861 				err = i915_vma_unbind_unlocked(vma);
862 				if (err) {
863 					pr_err("Unbinding returned %i\n", err);
864 					goto out_object;
865 				}
866 
867 				cond_resched();
868 			}
869 		}
870 
871 		count = 0;
872 		list_for_each_entry(vma, &obj->vma.list, obj_link)
873 			count++;
874 		if (count != nvma) {
875 			pr_err("(%s) All partial vma were not recorded on the obj->vma_list: found %u, expected %u\n",
876 			       p->name, count, nvma);
877 			err = -EINVAL;
878 			goto out_object;
879 		}
880 
881 		/* Check that we did create the whole object mapping */
882 		vma = checked_vma_instance(obj, vm, NULL);
883 		if (IS_ERR(vma)) {
884 			err = PTR_ERR(vma);
885 			goto out_object;
886 		}
887 
888 		err = i915_vma_pin(vma, 0, 0, PIN_GLOBAL);
889 		if (err)
890 			goto out_object;
891 
892 		if (!assert_pin(vma, NULL, obj->base.size, p->name)) {
893 			pr_err("(%s) inconsistent full pin\n", p->name);
894 			err = -EINVAL;
895 			goto out_object;
896 		}
897 
898 		i915_vma_unpin(vma);
899 
900 		err = i915_vma_unbind_unlocked(vma);
901 		if (err) {
902 			pr_err("Unbinding returned %i\n", err);
903 			goto out_object;
904 		}
905 
906 		count = 0;
907 		list_for_each_entry(vma, &obj->vma.list, obj_link)
908 			count++;
909 		if (count != nvma) {
910 			pr_err("(%s) allocated an extra full vma!\n", p->name);
911 			err = -EINVAL;
912 			goto out_object;
913 		}
914 	}
915 
916 out_object:
917 	i915_gem_object_put(obj);
918 out:
919 	return err;
920 }
921 
i915_vma_mock_selftests(void)922 int i915_vma_mock_selftests(void)
923 {
924 	static const struct i915_subtest tests[] = {
925 		SUBTEST(igt_vma_create),
926 		SUBTEST(igt_vma_pin1),
927 		SUBTEST(igt_vma_rotate_remap),
928 		SUBTEST(igt_vma_partial),
929 	};
930 	struct drm_i915_private *i915;
931 	struct intel_gt *gt;
932 	int err;
933 
934 	i915 = mock_gem_device();
935 	if (!i915)
936 		return -ENOMEM;
937 
938 	/* allocate the ggtt */
939 	err = intel_gt_assign_ggtt(to_gt(i915));
940 	if (err)
941 		goto out_put;
942 
943 	gt = to_gt(i915);
944 
945 	mock_init_ggtt(gt);
946 
947 	err = i915_subtests(tests, gt->ggtt);
948 
949 	mock_device_flush(i915);
950 	i915_gem_drain_freed_objects(i915);
951 	mock_fini_ggtt(gt->ggtt);
952 
953 out_put:
954 	mock_destroy_device(i915);
955 	return err;
956 }
957 
igt_vma_remapped_gtt(void * arg)958 static int igt_vma_remapped_gtt(void *arg)
959 {
960 	struct drm_i915_private *i915 = arg;
961 	const struct intel_remapped_plane_info planes[] = {
962 		{ .width = 1, .height = 1, .src_stride = 1 },
963 		{ .width = 2, .height = 2, .src_stride = 2 },
964 		{ .width = 4, .height = 4, .src_stride = 4 },
965 		{ .width = 8, .height = 8, .src_stride = 8 },
966 
967 		{ .width = 3, .height = 5, .src_stride = 3 },
968 		{ .width = 3, .height = 5, .src_stride = 4 },
969 		{ .width = 3, .height = 5, .src_stride = 5 },
970 
971 		{ .width = 5, .height = 3, .src_stride = 5 },
972 		{ .width = 5, .height = 3, .src_stride = 7 },
973 		{ .width = 5, .height = 3, .src_stride = 9 },
974 
975 		{ .width = 4, .height = 6, .src_stride = 6 },
976 		{ .width = 6, .height = 4, .src_stride = 6 },
977 
978 		{ .width = 2, .height = 2, .src_stride = 2, .dst_stride = 2 },
979 		{ .width = 3, .height = 3, .src_stride = 3, .dst_stride = 4 },
980 		{ .width = 5, .height = 6, .src_stride = 7, .dst_stride = 8 },
981 
982 		{ }
983 	}, *p;
984 	enum i915_gtt_view_type types[] = {
985 		I915_GTT_VIEW_ROTATED,
986 		I915_GTT_VIEW_REMAPPED,
987 		0,
988 	}, *t;
989 	struct drm_i915_gem_object *obj;
990 	intel_wakeref_t wakeref;
991 	int err = 0;
992 
993 	if (!i915_ggtt_has_aperture(to_gt(i915)->ggtt))
994 		return 0;
995 
996 	obj = i915_gem_object_create_internal(i915, 10 * 10 * PAGE_SIZE);
997 	if (IS_ERR(obj))
998 		return PTR_ERR(obj);
999 
1000 	wakeref = intel_runtime_pm_get(&i915->runtime_pm);
1001 
1002 	for (t = types; *t; t++) {
1003 		for (p = planes; p->width; p++) {
1004 			struct i915_gtt_view view = {
1005 				.type = *t,
1006 				.rotated.plane[0] = *p,
1007 			};
1008 			struct intel_remapped_plane_info *plane_info = view.rotated.plane;
1009 			struct i915_vma *vma;
1010 			u32 __iomem *map;
1011 			unsigned int x, y;
1012 
1013 			i915_gem_object_lock(obj, NULL);
1014 			err = i915_gem_object_set_to_gtt_domain(obj, true);
1015 			i915_gem_object_unlock(obj);
1016 			if (err)
1017 				goto out;
1018 
1019 			if (!plane_info[0].dst_stride)
1020 				plane_info[0].dst_stride = *t == I915_GTT_VIEW_ROTATED ?
1021 								 p->height : p->width;
1022 
1023 			vma = i915_gem_object_ggtt_pin(obj, &view, 0, 0, PIN_MAPPABLE);
1024 			if (IS_ERR(vma)) {
1025 				err = PTR_ERR(vma);
1026 				goto out;
1027 			}
1028 
1029 			GEM_BUG_ON(vma->gtt_view.type != *t);
1030 
1031 			map = i915_vma_pin_iomap(vma);
1032 			i915_vma_unpin(vma);
1033 			if (IS_ERR(map)) {
1034 				err = PTR_ERR(map);
1035 				goto out;
1036 			}
1037 
1038 			for (y = 0 ; y < plane_info[0].height; y++) {
1039 				for (x = 0 ; x < plane_info[0].width; x++) {
1040 					unsigned int offset;
1041 					u32 val = y << 16 | x;
1042 
1043 					if (*t == I915_GTT_VIEW_ROTATED)
1044 						offset = (x * plane_info[0].dst_stride + y) * PAGE_SIZE;
1045 					else
1046 						offset = (y * plane_info[0].dst_stride + x) * PAGE_SIZE;
1047 
1048 					iowrite32(val, &map[offset / sizeof(*map)]);
1049 				}
1050 			}
1051 
1052 			i915_vma_unpin_iomap(vma);
1053 
1054 			vma = i915_gem_object_ggtt_pin(obj, NULL, 0, 0, PIN_MAPPABLE);
1055 			if (IS_ERR(vma)) {
1056 				err = PTR_ERR(vma);
1057 				goto out;
1058 			}
1059 
1060 			GEM_BUG_ON(vma->gtt_view.type != I915_GTT_VIEW_NORMAL);
1061 
1062 			map = i915_vma_pin_iomap(vma);
1063 			i915_vma_unpin(vma);
1064 			if (IS_ERR(map)) {
1065 				err = PTR_ERR(map);
1066 				goto out;
1067 			}
1068 
1069 			for (y = 0 ; y < plane_info[0].height; y++) {
1070 				for (x = 0 ; x < plane_info[0].width; x++) {
1071 					unsigned int offset, src_idx;
1072 					u32 exp = y << 16 | x;
1073 					u32 val;
1074 
1075 					if (*t == I915_GTT_VIEW_ROTATED)
1076 						src_idx = rotated_index(&view.rotated, 0, x, y);
1077 					else
1078 						src_idx = remapped_index(&view.remapped, 0, x, y);
1079 					offset = src_idx * PAGE_SIZE;
1080 
1081 					val = ioread32(&map[offset / sizeof(*map)]);
1082 					if (val != exp) {
1083 						pr_err("%s VMA write test failed, expected 0x%x, found 0x%x\n",
1084 						       *t == I915_GTT_VIEW_ROTATED ? "Rotated" : "Remapped",
1085 						       exp, val);
1086 						i915_vma_unpin_iomap(vma);
1087 						err = -EINVAL;
1088 						goto out;
1089 					}
1090 				}
1091 			}
1092 			i915_vma_unpin_iomap(vma);
1093 
1094 			cond_resched();
1095 		}
1096 	}
1097 
1098 out:
1099 	intel_runtime_pm_put(&i915->runtime_pm, wakeref);
1100 	i915_gem_object_put(obj);
1101 
1102 	return err;
1103 }
1104 
i915_vma_live_selftests(struct drm_i915_private * i915)1105 int i915_vma_live_selftests(struct drm_i915_private *i915)
1106 {
1107 	static const struct i915_subtest tests[] = {
1108 		SUBTEST(igt_vma_remapped_gtt),
1109 	};
1110 
1111 	return i915_live_subtests(tests, i915);
1112 }
1113