1 // SPDX-License-Identifier: MIT
2
3 /*
4 * Copyright © 2019 Intel Corporation
5 */
6
7 #include <linux/delay.h>
8 #include <linux/dma-fence.h>
9 #include <linux/dma-fence-chain.h>
10 #include <linux/kernel.h>
11 #include <linux/kthread.h>
12 #include <linux/mm.h>
13 #include <linux/sched/signal.h>
14 #include <linux/slab.h>
15 #include <linux/spinlock.h>
16 #include <linux/random.h>
17
18 #include "selftest.h"
19
20 #define CHAIN_SZ (4 << 10)
21
22 static struct kmem_cache *slab_fences;
23
24 static inline struct mock_fence {
25 struct dma_fence base;
26 spinlock_t lock;
to_mock_fence(struct dma_fence * f)27 } *to_mock_fence(struct dma_fence *f) {
28 return container_of(f, struct mock_fence, base);
29 }
30
mock_name(struct dma_fence * f)31 static const char *mock_name(struct dma_fence *f)
32 {
33 return "mock";
34 }
35
mock_fence_release(struct dma_fence * f)36 static void mock_fence_release(struct dma_fence *f)
37 {
38 kmem_cache_free(slab_fences, to_mock_fence(f));
39 }
40
41 static const struct dma_fence_ops mock_ops = {
42 .get_driver_name = mock_name,
43 .get_timeline_name = mock_name,
44 .release = mock_fence_release,
45 };
46
mock_fence(void)47 static struct dma_fence *mock_fence(void)
48 {
49 struct mock_fence *f;
50
51 f = kmem_cache_alloc(slab_fences, GFP_KERNEL);
52 if (!f)
53 return NULL;
54
55 spin_lock_init(&f->lock);
56 dma_fence_init(&f->base, &mock_ops, &f->lock, 0, 0);
57
58 return &f->base;
59 }
60
mock_chain(struct dma_fence * prev,struct dma_fence * fence,u64 seqno)61 static struct dma_fence *mock_chain(struct dma_fence *prev,
62 struct dma_fence *fence,
63 u64 seqno)
64 {
65 struct dma_fence_chain *f;
66
67 f = dma_fence_chain_alloc();
68 if (!f)
69 return NULL;
70
71 dma_fence_chain_init(f, dma_fence_get(prev), dma_fence_get(fence),
72 seqno);
73
74 return &f->base;
75 }
76
sanitycheck(void * arg)77 static int sanitycheck(void *arg)
78 {
79 struct dma_fence *f, *chain;
80 int err = 0;
81
82 f = mock_fence();
83 if (!f)
84 return -ENOMEM;
85
86 chain = mock_chain(NULL, f, 1);
87 if (chain)
88 dma_fence_enable_sw_signaling(chain);
89 else
90 err = -ENOMEM;
91
92 dma_fence_signal(f);
93 dma_fence_put(f);
94
95 dma_fence_put(chain);
96
97 return err;
98 }
99
100 struct fence_chains {
101 unsigned int chain_length;
102 struct dma_fence **fences;
103 struct dma_fence **chains;
104
105 struct dma_fence *tail;
106 };
107
seqno_inc(unsigned int i)108 static uint64_t seqno_inc(unsigned int i)
109 {
110 return i + 1;
111 }
112
fence_chains_init(struct fence_chains * fc,unsigned int count,uint64_t (* seqno_fn)(unsigned int))113 static int fence_chains_init(struct fence_chains *fc, unsigned int count,
114 uint64_t (*seqno_fn)(unsigned int))
115 {
116 unsigned int i;
117 int err = 0;
118
119 fc->chains = kvmalloc_objs(*fc->chains, count, GFP_KERNEL | __GFP_ZERO);
120 if (!fc->chains)
121 return -ENOMEM;
122
123 fc->fences = kvmalloc_objs(*fc->fences, count, GFP_KERNEL | __GFP_ZERO);
124 if (!fc->fences) {
125 err = -ENOMEM;
126 goto err_chains;
127 }
128
129 fc->tail = NULL;
130 for (i = 0; i < count; i++) {
131 fc->fences[i] = mock_fence();
132 if (!fc->fences[i]) {
133 err = -ENOMEM;
134 goto unwind;
135 }
136
137 fc->chains[i] = mock_chain(fc->tail,
138 fc->fences[i],
139 seqno_fn(i));
140 if (!fc->chains[i]) {
141 err = -ENOMEM;
142 goto unwind;
143 }
144
145 fc->tail = fc->chains[i];
146
147 dma_fence_enable_sw_signaling(fc->chains[i]);
148 }
149
150 fc->chain_length = i;
151 return 0;
152
153 unwind:
154 for (i = 0; i < count; i++) {
155 dma_fence_put(fc->fences[i]);
156 dma_fence_put(fc->chains[i]);
157 }
158 kvfree(fc->fences);
159 err_chains:
160 kvfree(fc->chains);
161 return err;
162 }
163
fence_chains_fini(struct fence_chains * fc)164 static void fence_chains_fini(struct fence_chains *fc)
165 {
166 unsigned int i;
167
168 for (i = 0; i < fc->chain_length; i++) {
169 dma_fence_signal(fc->fences[i]);
170 dma_fence_put(fc->fences[i]);
171 }
172 kvfree(fc->fences);
173
174 for (i = 0; i < fc->chain_length; i++)
175 dma_fence_put(fc->chains[i]);
176 kvfree(fc->chains);
177 }
178
find_seqno(void * arg)179 static int find_seqno(void *arg)
180 {
181 struct fence_chains fc;
182 struct dma_fence *fence;
183 int err;
184 int i;
185
186 err = fence_chains_init(&fc, 64, seqno_inc);
187 if (err)
188 return err;
189
190 fence = dma_fence_get(fc.tail);
191 err = dma_fence_chain_find_seqno(&fence, 0);
192 dma_fence_put(fence);
193 if (err) {
194 pr_err("Reported %d for find_seqno(0)!\n", err);
195 goto err;
196 }
197
198 for (i = 0; i < fc.chain_length; i++) {
199 fence = dma_fence_get(fc.tail);
200 err = dma_fence_chain_find_seqno(&fence, i + 1);
201 dma_fence_put(fence);
202 if (err) {
203 pr_err("Reported %d for find_seqno(%d:%d)!\n",
204 err, fc.chain_length + 1, i + 1);
205 goto err;
206 }
207 if (fence != fc.chains[i]) {
208 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
209 fc.chain_length + 1, i + 1);
210 err = -EINVAL;
211 goto err;
212 }
213
214 dma_fence_get(fence);
215 err = dma_fence_chain_find_seqno(&fence, i + 1);
216 dma_fence_put(fence);
217 if (err) {
218 pr_err("Error reported for finding self\n");
219 goto err;
220 }
221 if (fence != fc.chains[i]) {
222 pr_err("Incorrect fence reported by find self\n");
223 err = -EINVAL;
224 goto err;
225 }
226
227 dma_fence_get(fence);
228 err = dma_fence_chain_find_seqno(&fence, i + 2);
229 dma_fence_put(fence);
230 if (!err) {
231 pr_err("Error not reported for future fence: find_seqno(%d:%d)!\n",
232 i + 1, i + 2);
233 err = -EINVAL;
234 goto err;
235 }
236
237 dma_fence_get(fence);
238 err = dma_fence_chain_find_seqno(&fence, i);
239 dma_fence_put(fence);
240 if (err) {
241 pr_err("Error reported for previous fence!\n");
242 goto err;
243 }
244 if (i > 0 && fence != fc.chains[i - 1]) {
245 pr_err("Incorrect fence reported by find_seqno(%d:%d)\n",
246 i + 1, i);
247 err = -EINVAL;
248 goto err;
249 }
250 }
251
252 err:
253 fence_chains_fini(&fc);
254 return err;
255 }
256
find_signaled(void * arg)257 static int find_signaled(void *arg)
258 {
259 struct fence_chains fc;
260 struct dma_fence *fence;
261 int err;
262
263 err = fence_chains_init(&fc, 2, seqno_inc);
264 if (err)
265 return err;
266
267 dma_fence_signal(fc.fences[0]);
268
269 fence = dma_fence_get(fc.tail);
270 err = dma_fence_chain_find_seqno(&fence, 1);
271 dma_fence_put(fence);
272 if (err) {
273 pr_err("Reported %d for find_seqno()!\n", err);
274 goto err;
275 }
276
277 if (fence && fence != fc.chains[0]) {
278 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:1\n",
279 fence->seqno);
280
281 dma_fence_get(fence);
282 err = dma_fence_chain_find_seqno(&fence, 1);
283 dma_fence_put(fence);
284 if (err)
285 pr_err("Reported %d for finding self!\n", err);
286
287 err = -EINVAL;
288 }
289
290 err:
291 fence_chains_fini(&fc);
292 return err;
293 }
294
find_out_of_order(void * arg)295 static int find_out_of_order(void *arg)
296 {
297 struct fence_chains fc;
298 struct dma_fence *fence;
299 int err;
300
301 err = fence_chains_init(&fc, 3, seqno_inc);
302 if (err)
303 return err;
304
305 dma_fence_signal(fc.fences[1]);
306
307 fence = dma_fence_get(fc.tail);
308 err = dma_fence_chain_find_seqno(&fence, 2);
309 dma_fence_put(fence);
310 if (err) {
311 pr_err("Reported %d for find_seqno()!\n", err);
312 goto err;
313 }
314
315 /*
316 * We signaled the middle fence (2) of the 1-2-3 chain. The behavior
317 * of the dma-fence-chain is to make us wait for all the fences up to
318 * the point we want. Since fence 1 is still not signaled, this what
319 * we should get as fence to wait upon (fence 2 being garbage
320 * collected during the traversal of the chain).
321 */
322 if (fence != fc.chains[0]) {
323 pr_err("Incorrect chain-fence.seqno:%lld reported for completed seqno:2\n",
324 fence ? fence->seqno : 0);
325
326 err = -EINVAL;
327 }
328
329 err:
330 fence_chains_fini(&fc);
331 return err;
332 }
333
seqno_inc2(unsigned int i)334 static uint64_t seqno_inc2(unsigned int i)
335 {
336 return 2 * i + 2;
337 }
338
find_gap(void * arg)339 static int find_gap(void *arg)
340 {
341 struct fence_chains fc;
342 struct dma_fence *fence;
343 int err;
344 int i;
345
346 err = fence_chains_init(&fc, 64, seqno_inc2);
347 if (err)
348 return err;
349
350 for (i = 0; i < fc.chain_length; i++) {
351 fence = dma_fence_get(fc.tail);
352 err = dma_fence_chain_find_seqno(&fence, 2 * i + 1);
353 dma_fence_put(fence);
354 if (err) {
355 pr_err("Reported %d for find_seqno(%d:%d)!\n",
356 err, fc.chain_length + 1, 2 * i + 1);
357 goto err;
358 }
359 if (fence != fc.chains[i]) {
360 pr_err("Incorrect fence.seqno:%lld reported by find_seqno(%d:%d)\n",
361 fence->seqno,
362 fc.chain_length + 1,
363 2 * i + 1);
364 err = -EINVAL;
365 goto err;
366 }
367
368 dma_fence_get(fence);
369 err = dma_fence_chain_find_seqno(&fence, 2 * i + 2);
370 dma_fence_put(fence);
371 if (err) {
372 pr_err("Error reported for finding self\n");
373 goto err;
374 }
375 if (fence != fc.chains[i]) {
376 pr_err("Incorrect fence reported by find self\n");
377 err = -EINVAL;
378 goto err;
379 }
380 }
381
382 err:
383 fence_chains_fini(&fc);
384 return err;
385 }
386
387 struct find_race {
388 struct fence_chains fc;
389 atomic_t children;
390 };
391
__find_race(void * arg)392 static int __find_race(void *arg)
393 {
394 struct find_race *data = arg;
395 int err = 0;
396
397 while (!kthread_should_stop()) {
398 struct dma_fence *fence = dma_fence_get(data->fc.tail);
399 int seqno;
400
401 seqno = get_random_u32_inclusive(1, data->fc.chain_length);
402
403 err = dma_fence_chain_find_seqno(&fence, seqno);
404 if (err) {
405 pr_err("Failed to find fence seqno:%d\n",
406 seqno);
407 dma_fence_put(fence);
408 break;
409 }
410 if (!fence)
411 goto signal;
412
413 /*
414 * We can only find ourselves if we are on fence we were
415 * looking for.
416 */
417 if (fence->seqno == seqno) {
418 err = dma_fence_chain_find_seqno(&fence, seqno);
419 if (err) {
420 pr_err("Reported an invalid fence for find-self:%d\n",
421 seqno);
422 dma_fence_put(fence);
423 break;
424 }
425 }
426
427 dma_fence_put(fence);
428
429 signal:
430 seqno = get_random_u32_below(data->fc.chain_length - 1);
431 dma_fence_signal(data->fc.fences[seqno]);
432 cond_resched();
433 }
434
435 if (atomic_dec_and_test(&data->children))
436 wake_up_var(&data->children);
437 return err;
438 }
439
find_race(void * arg)440 static int find_race(void *arg)
441 {
442 struct find_race data;
443 int ncpus = num_online_cpus();
444 struct task_struct **threads;
445 unsigned long count;
446 int err;
447 int i;
448
449 err = fence_chains_init(&data.fc, CHAIN_SZ, seqno_inc);
450 if (err)
451 return err;
452
453 threads = kmalloc_objs(*threads, ncpus);
454 if (!threads) {
455 err = -ENOMEM;
456 goto err;
457 }
458
459 atomic_set(&data.children, 0);
460 for (i = 0; i < ncpus; i++) {
461 threads[i] = kthread_run(__find_race, &data, "dmabuf/%d", i);
462 if (IS_ERR(threads[i])) {
463 ncpus = i;
464 break;
465 }
466 atomic_inc(&data.children);
467 get_task_struct(threads[i]);
468 }
469
470 wait_var_event_timeout(&data.children,
471 !atomic_read(&data.children),
472 5 * HZ);
473
474 for (i = 0; i < ncpus; i++) {
475 int ret;
476
477 ret = kthread_stop_put(threads[i]);
478 if (ret && !err)
479 err = ret;
480 }
481 kfree(threads);
482
483 count = 0;
484 for (i = 0; i < data.fc.chain_length; i++)
485 if (dma_fence_is_signaled(data.fc.fences[i]))
486 count++;
487 pr_info("Completed %lu cycles\n", count);
488
489 err:
490 fence_chains_fini(&data.fc);
491 return err;
492 }
493
signal_forward(void * arg)494 static int signal_forward(void *arg)
495 {
496 struct fence_chains fc;
497 int err;
498 int i;
499
500 err = fence_chains_init(&fc, 64, seqno_inc);
501 if (err)
502 return err;
503
504 for (i = 0; i < fc.chain_length; i++) {
505 dma_fence_signal(fc.fences[i]);
506
507 if (!dma_fence_is_signaled(fc.chains[i])) {
508 pr_err("chain[%d] not signaled!\n", i);
509 err = -EINVAL;
510 goto err;
511 }
512
513 if (i + 1 < fc.chain_length &&
514 dma_fence_is_signaled(fc.chains[i + 1])) {
515 pr_err("chain[%d] is signaled!\n", i);
516 err = -EINVAL;
517 goto err;
518 }
519 }
520
521 err:
522 fence_chains_fini(&fc);
523 return err;
524 }
525
signal_backward(void * arg)526 static int signal_backward(void *arg)
527 {
528 struct fence_chains fc;
529 int err;
530 int i;
531
532 err = fence_chains_init(&fc, 64, seqno_inc);
533 if (err)
534 return err;
535
536 for (i = fc.chain_length; i--; ) {
537 dma_fence_signal(fc.fences[i]);
538
539 if (i > 0 && dma_fence_is_signaled(fc.chains[i])) {
540 pr_err("chain[%d] is signaled!\n", i);
541 err = -EINVAL;
542 goto err;
543 }
544 }
545
546 for (i = 0; i < fc.chain_length; i++) {
547 if (!dma_fence_is_signaled(fc.chains[i])) {
548 pr_err("chain[%d] was not signaled!\n", i);
549 err = -EINVAL;
550 goto err;
551 }
552 }
553
554 err:
555 fence_chains_fini(&fc);
556 return err;
557 }
558
__wait_fence_chains(void * arg)559 static int __wait_fence_chains(void *arg)
560 {
561 struct fence_chains *fc = arg;
562
563 if (dma_fence_wait(fc->tail, false))
564 return -EIO;
565
566 return 0;
567 }
568
wait_forward(void * arg)569 static int wait_forward(void *arg)
570 {
571 struct fence_chains fc;
572 struct task_struct *tsk;
573 int err;
574 int i;
575
576 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
577 if (err)
578 return err;
579
580 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
581 if (IS_ERR(tsk)) {
582 err = PTR_ERR(tsk);
583 goto err;
584 }
585 get_task_struct(tsk);
586 yield_to(tsk, true);
587
588 for (i = 0; i < fc.chain_length; i++)
589 dma_fence_signal(fc.fences[i]);
590
591 err = kthread_stop_put(tsk);
592
593 err:
594 fence_chains_fini(&fc);
595 return err;
596 }
597
wait_backward(void * arg)598 static int wait_backward(void *arg)
599 {
600 struct fence_chains fc;
601 struct task_struct *tsk;
602 int err;
603 int i;
604
605 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
606 if (err)
607 return err;
608
609 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
610 if (IS_ERR(tsk)) {
611 err = PTR_ERR(tsk);
612 goto err;
613 }
614 get_task_struct(tsk);
615 yield_to(tsk, true);
616
617 for (i = fc.chain_length; i--; )
618 dma_fence_signal(fc.fences[i]);
619
620 err = kthread_stop_put(tsk);
621
622 err:
623 fence_chains_fini(&fc);
624 return err;
625 }
626
randomise_fences(struct fence_chains * fc)627 static void randomise_fences(struct fence_chains *fc)
628 {
629 unsigned int count = fc->chain_length;
630
631 /* Fisher-Yates shuffle courtesy of Knuth */
632 while (--count) {
633 unsigned int swp;
634
635 swp = get_random_u32_below(count + 1);
636 if (swp == count)
637 continue;
638
639 swap(fc->fences[count], fc->fences[swp]);
640 }
641 }
642
wait_random(void * arg)643 static int wait_random(void *arg)
644 {
645 struct fence_chains fc;
646 struct task_struct *tsk;
647 int err;
648 int i;
649
650 err = fence_chains_init(&fc, CHAIN_SZ, seqno_inc);
651 if (err)
652 return err;
653
654 randomise_fences(&fc);
655
656 tsk = kthread_run(__wait_fence_chains, &fc, "dmabuf/wait");
657 if (IS_ERR(tsk)) {
658 err = PTR_ERR(tsk);
659 goto err;
660 }
661 get_task_struct(tsk);
662 yield_to(tsk, true);
663
664 for (i = 0; i < fc.chain_length; i++)
665 dma_fence_signal(fc.fences[i]);
666
667 err = kthread_stop_put(tsk);
668
669 err:
670 fence_chains_fini(&fc);
671 return err;
672 }
673
dma_fence_chain(void)674 int dma_fence_chain(void)
675 {
676 static const struct subtest tests[] = {
677 SUBTEST(sanitycheck),
678 SUBTEST(find_seqno),
679 SUBTEST(find_signaled),
680 SUBTEST(find_out_of_order),
681 SUBTEST(find_gap),
682 SUBTEST(find_race),
683 SUBTEST(signal_forward),
684 SUBTEST(signal_backward),
685 SUBTEST(wait_forward),
686 SUBTEST(wait_backward),
687 SUBTEST(wait_random),
688 };
689 int ret;
690
691 pr_info("sizeof(dma_fence_chain)=%zu\n",
692 sizeof(struct dma_fence_chain));
693
694 slab_fences = KMEM_CACHE(mock_fence,
695 SLAB_TYPESAFE_BY_RCU |
696 SLAB_HWCACHE_ALIGN);
697 if (!slab_fences)
698 return -ENOMEM;
699
700 ret = subtests(tests, NULL);
701
702 kmem_cache_destroy(slab_fences);
703 return ret;
704 }
705