1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3 * Copyright (C) 2013-2016 Red Hat
4 * Author: Rob Clark <robdclark@gmail.com>
5 */
6
7 #include <linux/dma-fence.h>
8
9 #include "msm_drv.h"
10 #include "msm_fence.h"
11 #include "msm_gpu.h"
12
fctx2gpu(struct msm_fence_context * fctx)13 static struct msm_gpu *fctx2gpu(struct msm_fence_context *fctx)
14 {
15 struct msm_drm_private *priv = fctx->dev->dev_private;
16 return priv->gpu;
17 }
18
deadline_timer(struct hrtimer * t)19 static enum hrtimer_restart deadline_timer(struct hrtimer *t)
20 {
21 struct msm_fence_context *fctx = container_of(t,
22 struct msm_fence_context, deadline_timer);
23
24 kthread_queue_work(fctx2gpu(fctx)->worker, &fctx->deadline_work);
25
26 return HRTIMER_NORESTART;
27 }
28
deadline_work(struct kthread_work * work)29 static void deadline_work(struct kthread_work *work)
30 {
31 struct msm_fence_context *fctx = container_of(work,
32 struct msm_fence_context, deadline_work);
33
34 /* If deadline fence has already passed, nothing to do: */
35 if (msm_fence_completed(fctx, fctx->next_deadline_fence))
36 return;
37
38 msm_devfreq_boost(fctx2gpu(fctx), 2);
39 }
40
41
42 struct msm_fence_context *
msm_fence_context_alloc(struct drm_device * dev,volatile uint32_t * fenceptr,const char * name)43 msm_fence_context_alloc(struct drm_device *dev, volatile uint32_t *fenceptr,
44 const char *name)
45 {
46 struct msm_fence_context *fctx;
47 static int index = 0;
48
49 fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
50 if (!fctx)
51 return ERR_PTR(-ENOMEM);
52
53 fctx->dev = dev;
54 strscpy(fctx->name, name, sizeof(fctx->name));
55 fctx->context = dma_fence_context_alloc(1);
56 fctx->index = index++;
57 fctx->fenceptr = fenceptr;
58 spin_lock_init(&fctx->spinlock);
59
60 /*
61 * Start out close to the 32b fence rollover point, so we can
62 * catch bugs with fence comparisons.
63 */
64 fctx->last_fence = 0xffffff00;
65 fctx->completed_fence = fctx->last_fence;
66 *fctx->fenceptr = fctx->last_fence;
67
68 hrtimer_setup(&fctx->deadline_timer, deadline_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
69
70 kthread_init_work(&fctx->deadline_work, deadline_work);
71
72 fctx->next_deadline = ktime_get();
73
74 return fctx;
75 }
76
msm_fence_context_free(struct msm_fence_context * fctx)77 void msm_fence_context_free(struct msm_fence_context *fctx)
78 {
79 kfree(fctx);
80 }
81
msm_fence_completed(struct msm_fence_context * fctx,uint32_t fence)82 bool msm_fence_completed(struct msm_fence_context *fctx, uint32_t fence)
83 {
84 /*
85 * Note: Check completed_fence first, as fenceptr is in a write-combine
86 * mapping, so it will be more expensive to read.
87 */
88 return (int32_t)(fctx->completed_fence - fence) >= 0 ||
89 (int32_t)(*fctx->fenceptr - fence) >= 0;
90 }
91
92 /* called from irq handler and workqueue (in recover path) */
msm_update_fence(struct msm_fence_context * fctx,uint32_t fence)93 void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence)
94 {
95 unsigned long flags;
96
97 spin_lock_irqsave(&fctx->spinlock, flags);
98 if (fence_after(fence, fctx->completed_fence))
99 fctx->completed_fence = fence;
100 if (msm_fence_completed(fctx, fctx->next_deadline_fence))
101 hrtimer_cancel(&fctx->deadline_timer);
102 spin_unlock_irqrestore(&fctx->spinlock, flags);
103 }
104
105 struct msm_fence {
106 struct dma_fence base;
107 struct msm_fence_context *fctx;
108 };
109
to_msm_fence(struct dma_fence * fence)110 static inline struct msm_fence *to_msm_fence(struct dma_fence *fence)
111 {
112 return container_of(fence, struct msm_fence, base);
113 }
114
msm_fence_get_driver_name(struct dma_fence * fence)115 static const char *msm_fence_get_driver_name(struct dma_fence *fence)
116 {
117 return "msm";
118 }
119
msm_fence_get_timeline_name(struct dma_fence * fence)120 static const char *msm_fence_get_timeline_name(struct dma_fence *fence)
121 {
122 struct msm_fence *f = to_msm_fence(fence);
123 return f->fctx->name;
124 }
125
msm_fence_signaled(struct dma_fence * fence)126 static bool msm_fence_signaled(struct dma_fence *fence)
127 {
128 struct msm_fence *f = to_msm_fence(fence);
129 return msm_fence_completed(f->fctx, f->base.seqno);
130 }
131
msm_fence_set_deadline(struct dma_fence * fence,ktime_t deadline)132 static void msm_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
133 {
134 struct msm_fence *f = to_msm_fence(fence);
135 struct msm_fence_context *fctx = f->fctx;
136 unsigned long flags;
137 ktime_t now;
138
139 spin_lock_irqsave(&fctx->spinlock, flags);
140 now = ktime_get();
141
142 if (ktime_after(now, fctx->next_deadline) ||
143 ktime_before(deadline, fctx->next_deadline)) {
144 fctx->next_deadline = deadline;
145 fctx->next_deadline_fence =
146 max(fctx->next_deadline_fence, (uint32_t)fence->seqno);
147
148 /*
149 * Set timer to trigger boost 3ms before deadline, or
150 * if we are already less than 3ms before the deadline
151 * schedule boost work immediately.
152 */
153 deadline = ktime_sub(deadline, ms_to_ktime(3));
154
155 if (ktime_after(now, deadline)) {
156 kthread_queue_work(fctx2gpu(fctx)->worker,
157 &fctx->deadline_work);
158 } else {
159 hrtimer_start(&fctx->deadline_timer, deadline,
160 HRTIMER_MODE_ABS);
161 }
162 }
163
164 spin_unlock_irqrestore(&fctx->spinlock, flags);
165 }
166
167 static const struct dma_fence_ops msm_fence_ops = {
168 .get_driver_name = msm_fence_get_driver_name,
169 .get_timeline_name = msm_fence_get_timeline_name,
170 .signaled = msm_fence_signaled,
171 .set_deadline = msm_fence_set_deadline,
172 };
173
174 struct dma_fence *
msm_fence_alloc(void)175 msm_fence_alloc(void)
176 {
177 struct msm_fence *f;
178
179 f = kzalloc(sizeof(*f), GFP_KERNEL);
180 if (!f)
181 return ERR_PTR(-ENOMEM);
182
183 return &f->base;
184 }
185
186 void
msm_fence_init(struct dma_fence * fence,struct msm_fence_context * fctx)187 msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx)
188 {
189 struct msm_fence *f = to_msm_fence(fence);
190
191 f->fctx = fctx;
192
193 /*
194 * Until this point, the fence was just some pre-allocated memory,
195 * no-one should have taken a reference to it yet.
196 */
197 WARN_ON(kref_read(&fence->refcount));
198
199 dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
200 fctx->context, ++fctx->last_fence);
201 }
202