1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Copyright (C) 2013-2016 Red Hat
4  * Author: Rob Clark <robdclark@gmail.com>
5  */
6 
7 #include <linux/dma-fence.h>
8 
9 #include "msm_drv.h"
10 #include "msm_fence.h"
11 #include "msm_gpu.h"
12 
fctx2gpu(struct msm_fence_context * fctx)13 static struct msm_gpu *fctx2gpu(struct msm_fence_context *fctx)
14 {
15 	struct msm_drm_private *priv = fctx->dev->dev_private;
16 	return priv->gpu;
17 }
18 
deadline_timer(struct hrtimer * t)19 static enum hrtimer_restart deadline_timer(struct hrtimer *t)
20 {
21 	struct msm_fence_context *fctx = container_of(t,
22 			struct msm_fence_context, deadline_timer);
23 
24 	kthread_queue_work(fctx2gpu(fctx)->worker, &fctx->deadline_work);
25 
26 	return HRTIMER_NORESTART;
27 }
28 
deadline_work(struct kthread_work * work)29 static void deadline_work(struct kthread_work *work)
30 {
31 	struct msm_fence_context *fctx = container_of(work,
32 			struct msm_fence_context, deadline_work);
33 
34 	/* If deadline fence has already passed, nothing to do: */
35 	if (msm_fence_completed(fctx, fctx->next_deadline_fence))
36 		return;
37 
38 	msm_devfreq_boost(fctx2gpu(fctx), 2);
39 }
40 
41 
42 struct msm_fence_context *
msm_fence_context_alloc(struct drm_device * dev,volatile uint32_t * fenceptr,const char * name)43 msm_fence_context_alloc(struct drm_device *dev, volatile uint32_t *fenceptr,
44 		const char *name)
45 {
46 	struct msm_fence_context *fctx;
47 	static int index = 0;
48 
49 	fctx = kzalloc(sizeof(*fctx), GFP_KERNEL);
50 	if (!fctx)
51 		return ERR_PTR(-ENOMEM);
52 
53 	fctx->dev = dev;
54 	strscpy(fctx->name, name, sizeof(fctx->name));
55 	fctx->context = dma_fence_context_alloc(1);
56 	fctx->index = index++;
57 	fctx->fenceptr = fenceptr;
58 	spin_lock_init(&fctx->spinlock);
59 
60 	/*
61 	 * Start out close to the 32b fence rollover point, so we can
62 	 * catch bugs with fence comparisons.
63 	 */
64 	fctx->last_fence = 0xffffff00;
65 	fctx->completed_fence = fctx->last_fence;
66 	*fctx->fenceptr = fctx->last_fence;
67 
68 	hrtimer_setup(&fctx->deadline_timer, deadline_timer, CLOCK_MONOTONIC, HRTIMER_MODE_ABS);
69 
70 	kthread_init_work(&fctx->deadline_work, deadline_work);
71 
72 	fctx->next_deadline = ktime_get();
73 
74 	return fctx;
75 }
76 
msm_fence_context_free(struct msm_fence_context * fctx)77 void msm_fence_context_free(struct msm_fence_context *fctx)
78 {
79 	kfree(fctx);
80 }
81 
msm_fence_completed(struct msm_fence_context * fctx,uint32_t fence)82 bool msm_fence_completed(struct msm_fence_context *fctx, uint32_t fence)
83 {
84 	/*
85 	 * Note: Check completed_fence first, as fenceptr is in a write-combine
86 	 * mapping, so it will be more expensive to read.
87 	 */
88 	return (int32_t)(fctx->completed_fence - fence) >= 0 ||
89 		(int32_t)(*fctx->fenceptr - fence) >= 0;
90 }
91 
92 /* called from irq handler and workqueue (in recover path) */
msm_update_fence(struct msm_fence_context * fctx,uint32_t fence)93 void msm_update_fence(struct msm_fence_context *fctx, uint32_t fence)
94 {
95 	unsigned long flags;
96 
97 	spin_lock_irqsave(&fctx->spinlock, flags);
98 	if (fence_after(fence, fctx->completed_fence))
99 		fctx->completed_fence = fence;
100 	if (msm_fence_completed(fctx, fctx->next_deadline_fence))
101 		hrtimer_cancel(&fctx->deadline_timer);
102 	spin_unlock_irqrestore(&fctx->spinlock, flags);
103 }
104 
105 struct msm_fence {
106 	struct dma_fence base;
107 	struct msm_fence_context *fctx;
108 };
109 
to_msm_fence(struct dma_fence * fence)110 static inline struct msm_fence *to_msm_fence(struct dma_fence *fence)
111 {
112 	return container_of(fence, struct msm_fence, base);
113 }
114 
msm_fence_get_driver_name(struct dma_fence * fence)115 static const char *msm_fence_get_driver_name(struct dma_fence *fence)
116 {
117 	return "msm";
118 }
119 
msm_fence_get_timeline_name(struct dma_fence * fence)120 static const char *msm_fence_get_timeline_name(struct dma_fence *fence)
121 {
122 	struct msm_fence *f = to_msm_fence(fence);
123 	return f->fctx->name;
124 }
125 
msm_fence_signaled(struct dma_fence * fence)126 static bool msm_fence_signaled(struct dma_fence *fence)
127 {
128 	struct msm_fence *f = to_msm_fence(fence);
129 	return msm_fence_completed(f->fctx, f->base.seqno);
130 }
131 
msm_fence_set_deadline(struct dma_fence * fence,ktime_t deadline)132 static void msm_fence_set_deadline(struct dma_fence *fence, ktime_t deadline)
133 {
134 	struct msm_fence *f = to_msm_fence(fence);
135 	struct msm_fence_context *fctx = f->fctx;
136 	unsigned long flags;
137 	ktime_t now;
138 
139 	spin_lock_irqsave(&fctx->spinlock, flags);
140 	now = ktime_get();
141 
142 	if (ktime_after(now, fctx->next_deadline) ||
143 			ktime_before(deadline, fctx->next_deadline)) {
144 		fctx->next_deadline = deadline;
145 		fctx->next_deadline_fence =
146 			max(fctx->next_deadline_fence, (uint32_t)fence->seqno);
147 
148 		/*
149 		 * Set timer to trigger boost 3ms before deadline, or
150 		 * if we are already less than 3ms before the deadline
151 		 * schedule boost work immediately.
152 		 */
153 		deadline = ktime_sub(deadline, ms_to_ktime(3));
154 
155 		if (ktime_after(now, deadline)) {
156 			kthread_queue_work(fctx2gpu(fctx)->worker,
157 					&fctx->deadline_work);
158 		} else {
159 			hrtimer_start(&fctx->deadline_timer, deadline,
160 					HRTIMER_MODE_ABS);
161 		}
162 	}
163 
164 	spin_unlock_irqrestore(&fctx->spinlock, flags);
165 }
166 
167 static const struct dma_fence_ops msm_fence_ops = {
168 	.get_driver_name = msm_fence_get_driver_name,
169 	.get_timeline_name = msm_fence_get_timeline_name,
170 	.signaled = msm_fence_signaled,
171 	.set_deadline = msm_fence_set_deadline,
172 };
173 
174 struct dma_fence *
msm_fence_alloc(void)175 msm_fence_alloc(void)
176 {
177 	struct msm_fence *f;
178 
179 	f = kzalloc(sizeof(*f), GFP_KERNEL);
180 	if (!f)
181 		return ERR_PTR(-ENOMEM);
182 
183 	return &f->base;
184 }
185 
186 void
msm_fence_init(struct dma_fence * fence,struct msm_fence_context * fctx)187 msm_fence_init(struct dma_fence *fence, struct msm_fence_context *fctx)
188 {
189 	struct msm_fence *f = to_msm_fence(fence);
190 
191 	f->fctx = fctx;
192 
193 	/*
194 	 * Until this point, the fence was just some pre-allocated memory,
195 	 * no-one should have taken a reference to it yet.
196 	 */
197 	WARN_ON(kref_read(&fence->refcount));
198 
199 	dma_fence_init(&f->base, &msm_fence_ops, &fctx->spinlock,
200 		       fctx->context, ++fctx->last_fence);
201 }
202