1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2
3 #include <drm/drm_exec.h>
4 #include <drm/drm_gem.h>
5
6 #include <linux/dma-resv.h>
7 #include <linux/export.h>
8
9 /**
10 * DOC: Overview
11 *
12 * This component mainly abstracts the retry loop necessary for locking
13 * multiple GEM objects while preparing hardware operations (e.g. command
14 * submissions, page table updates etc..).
15 *
16 * If a contention is detected while locking a GEM object the cleanup procedure
17 * unlocks all previously locked GEM objects and locks the contended one first
18 * before locking any further objects.
19 *
20 * After an object is locked fences slots can optionally be reserved on the
21 * dma_resv object inside the GEM object.
22 *
23 * A typical usage pattern should look like this::
24 *
25 * struct drm_gem_object *obj;
26 * struct drm_exec exec;
27 * unsigned long index;
28 * int ret;
29 *
30 * drm_exec_init(&exec, DRM_EXEC_INTERRUPTIBLE_WAIT);
31 * drm_exec_until_all_locked(&exec) {
32 * ret = drm_exec_prepare_obj(&exec, boA, 1);
33 * drm_exec_retry_on_contention(&exec);
34 * if (ret)
35 * goto error;
36 *
37 * ret = drm_exec_prepare_obj(&exec, boB, 1);
38 * drm_exec_retry_on_contention(&exec);
39 * if (ret)
40 * goto error;
41 * }
42 *
43 * drm_exec_for_each_locked_object(&exec, index, obj) {
44 * dma_resv_add_fence(obj->resv, fence, DMA_RESV_USAGE_READ);
45 * ...
46 * }
47 * drm_exec_fini(&exec);
48 *
49 * See struct dma_exec for more details.
50 */
51
52 /* Dummy value used to initially enter the retry loop */
53 #define DRM_EXEC_DUMMY ((void *)~0)
54
55 /* Unlock all objects and drop references */
drm_exec_unlock_all(struct drm_exec * exec)56 static void drm_exec_unlock_all(struct drm_exec *exec)
57 {
58 struct drm_gem_object *obj;
59 unsigned long index;
60
61 drm_exec_for_each_locked_object_reverse(exec, index, obj) {
62 dma_resv_unlock(obj->resv);
63 drm_gem_object_put(obj);
64 }
65
66 drm_gem_object_put(exec->prelocked);
67 exec->prelocked = NULL;
68 }
69
70 /**
71 * drm_exec_init - initialize a drm_exec object
72 * @exec: the drm_exec object to initialize
73 * @flags: controls locking behavior, see DRM_EXEC_* defines
74 * @nr: the initial # of objects
75 *
76 * Initialize the object and make sure that we can track locked objects.
77 *
78 * If nr is non-zero then it is used as the initial objects table size.
79 * In either case, the table will grow (be re-allocated) on demand.
80 */
drm_exec_init(struct drm_exec * exec,u32 flags,unsigned nr)81 void drm_exec_init(struct drm_exec *exec, u32 flags, unsigned nr)
82 {
83 if (!nr)
84 nr = PAGE_SIZE / sizeof(void *);
85
86 exec->flags = flags;
87 exec->objects = kvmalloc_array(nr, sizeof(void *), GFP_KERNEL);
88
89 /* If allocation here fails, just delay that till the first use */
90 exec->max_objects = exec->objects ? nr : 0;
91 exec->num_objects = 0;
92 exec->contended = DRM_EXEC_DUMMY;
93 exec->prelocked = NULL;
94 }
95 EXPORT_SYMBOL(drm_exec_init);
96
97 /**
98 * drm_exec_fini - finalize a drm_exec object
99 * @exec: the drm_exec object to finalize
100 *
101 * Unlock all locked objects, drop the references to objects and free all memory
102 * used for tracking the state.
103 */
drm_exec_fini(struct drm_exec * exec)104 void drm_exec_fini(struct drm_exec *exec)
105 {
106 drm_exec_unlock_all(exec);
107 kvfree(exec->objects);
108 if (exec->contended != DRM_EXEC_DUMMY) {
109 drm_gem_object_put(exec->contended);
110 ww_acquire_fini(&exec->ticket);
111 }
112 }
113 EXPORT_SYMBOL(drm_exec_fini);
114
115 /**
116 * drm_exec_cleanup - cleanup when contention is detected
117 * @exec: the drm_exec object to cleanup
118 *
119 * Cleanup the current state and return true if we should stay inside the retry
120 * loop, false if there wasn't any contention detected and we can keep the
121 * objects locked.
122 */
drm_exec_cleanup(struct drm_exec * exec)123 bool drm_exec_cleanup(struct drm_exec *exec)
124 {
125 if (likely(!exec->contended)) {
126 ww_acquire_done(&exec->ticket);
127 return false;
128 }
129
130 if (likely(exec->contended == DRM_EXEC_DUMMY)) {
131 exec->contended = NULL;
132 ww_acquire_init(&exec->ticket, &reservation_ww_class);
133 return true;
134 }
135
136 drm_exec_unlock_all(exec);
137 exec->num_objects = 0;
138 return true;
139 }
140 EXPORT_SYMBOL(drm_exec_cleanup);
141
142 /* Track the locked object in the array */
drm_exec_obj_locked(struct drm_exec * exec,struct drm_gem_object * obj)143 static int drm_exec_obj_locked(struct drm_exec *exec,
144 struct drm_gem_object *obj)
145 {
146 if (unlikely(exec->num_objects == exec->max_objects)) {
147 size_t size = exec->max_objects * sizeof(void *);
148 void *tmp;
149
150 tmp = kvrealloc(exec->objects, size + PAGE_SIZE, GFP_KERNEL);
151 if (!tmp)
152 return -ENOMEM;
153
154 exec->objects = tmp;
155 exec->max_objects += PAGE_SIZE / sizeof(void *);
156 }
157 drm_gem_object_get(obj);
158 exec->objects[exec->num_objects++] = obj;
159
160 return 0;
161 }
162
163 /* Make sure the contended object is locked first */
drm_exec_lock_contended(struct drm_exec * exec)164 static int drm_exec_lock_contended(struct drm_exec *exec)
165 {
166 struct drm_gem_object *obj = exec->contended;
167 int ret;
168
169 if (likely(!obj))
170 return 0;
171
172 /* Always cleanup the contention so that error handling can kick in */
173 exec->contended = NULL;
174 if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT) {
175 ret = dma_resv_lock_slow_interruptible(obj->resv,
176 &exec->ticket);
177 if (unlikely(ret))
178 goto error_dropref;
179 } else {
180 dma_resv_lock_slow(obj->resv, &exec->ticket);
181 }
182
183 ret = drm_exec_obj_locked(exec, obj);
184 if (unlikely(ret))
185 goto error_unlock;
186
187 exec->prelocked = obj;
188 return 0;
189
190 error_unlock:
191 dma_resv_unlock(obj->resv);
192
193 error_dropref:
194 drm_gem_object_put(obj);
195 return ret;
196 }
197
198 /**
199 * drm_exec_lock_obj - lock a GEM object for use
200 * @exec: the drm_exec object with the state
201 * @obj: the GEM object to lock
202 *
203 * Lock a GEM object for use and grab a reference to it.
204 *
205 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
206 * already locked (can be suppressed by setting the DRM_EXEC_IGNORE_DUPLICATES
207 * flag), -ENOMEM when memory allocation failed and zero for success.
208 */
drm_exec_lock_obj(struct drm_exec * exec,struct drm_gem_object * obj)209 int drm_exec_lock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
210 {
211 int ret;
212
213 ret = drm_exec_lock_contended(exec);
214 if (unlikely(ret))
215 return ret;
216
217 if (exec->prelocked == obj) {
218 drm_gem_object_put(exec->prelocked);
219 exec->prelocked = NULL;
220 return 0;
221 }
222
223 if (exec->flags & DRM_EXEC_INTERRUPTIBLE_WAIT)
224 ret = dma_resv_lock_interruptible(obj->resv, &exec->ticket);
225 else
226 ret = dma_resv_lock(obj->resv, &exec->ticket);
227
228 if (unlikely(ret == -EDEADLK)) {
229 drm_gem_object_get(obj);
230 exec->contended = obj;
231 return -EDEADLK;
232 }
233
234 if (unlikely(ret == -EALREADY) &&
235 exec->flags & DRM_EXEC_IGNORE_DUPLICATES)
236 return 0;
237
238 if (unlikely(ret))
239 return ret;
240
241 ret = drm_exec_obj_locked(exec, obj);
242 if (ret)
243 goto error_unlock;
244
245 return 0;
246
247 error_unlock:
248 dma_resv_unlock(obj->resv);
249 return ret;
250 }
251 EXPORT_SYMBOL(drm_exec_lock_obj);
252
253 /**
254 * drm_exec_unlock_obj - unlock a GEM object in this exec context
255 * @exec: the drm_exec object with the state
256 * @obj: the GEM object to unlock
257 *
258 * Unlock the GEM object and remove it from the collection of locked objects.
259 * Should only be used to unlock the most recently locked objects. It's not time
260 * efficient to unlock objects locked long ago.
261 */
drm_exec_unlock_obj(struct drm_exec * exec,struct drm_gem_object * obj)262 void drm_exec_unlock_obj(struct drm_exec *exec, struct drm_gem_object *obj)
263 {
264 unsigned int i;
265
266 for (i = exec->num_objects; i--;) {
267 if (exec->objects[i] == obj) {
268 dma_resv_unlock(obj->resv);
269 for (++i; i < exec->num_objects; ++i)
270 exec->objects[i - 1] = exec->objects[i];
271 --exec->num_objects;
272 drm_gem_object_put(obj);
273 return;
274 }
275
276 }
277 }
278 EXPORT_SYMBOL(drm_exec_unlock_obj);
279
280 /**
281 * drm_exec_prepare_obj - prepare a GEM object for use
282 * @exec: the drm_exec object with the state
283 * @obj: the GEM object to prepare
284 * @num_fences: how many fences to reserve
285 *
286 * Prepare a GEM object for use by locking it and reserving fence slots.
287 *
288 * Returns: -EDEADLK if a contention is detected, -EALREADY when object is
289 * already locked, -ENOMEM when memory allocation failed and zero for success.
290 */
drm_exec_prepare_obj(struct drm_exec * exec,struct drm_gem_object * obj,unsigned int num_fences)291 int drm_exec_prepare_obj(struct drm_exec *exec, struct drm_gem_object *obj,
292 unsigned int num_fences)
293 {
294 int ret;
295
296 ret = drm_exec_lock_obj(exec, obj);
297 if (ret)
298 return ret;
299
300 ret = dma_resv_reserve_fences(obj->resv, num_fences);
301 if (ret) {
302 drm_exec_unlock_obj(exec, obj);
303 return ret;
304 }
305
306 return 0;
307 }
308 EXPORT_SYMBOL(drm_exec_prepare_obj);
309
310 /**
311 * drm_exec_prepare_array - helper to prepare an array of objects
312 * @exec: the drm_exec object with the state
313 * @objects: array of GEM object to prepare
314 * @num_objects: number of GEM objects in the array
315 * @num_fences: number of fences to reserve on each GEM object
316 *
317 * Prepares all GEM objects in an array, aborts on first error.
318 * Reserves @num_fences on each GEM object after locking it.
319 *
320 * Returns: -EDEADLOCK on contention, -EALREADY when object is already locked,
321 * -ENOMEM when memory allocation failed and zero for success.
322 */
drm_exec_prepare_array(struct drm_exec * exec,struct drm_gem_object ** objects,unsigned int num_objects,unsigned int num_fences)323 int drm_exec_prepare_array(struct drm_exec *exec,
324 struct drm_gem_object **objects,
325 unsigned int num_objects,
326 unsigned int num_fences)
327 {
328 int ret;
329
330 for (unsigned int i = 0; i < num_objects; ++i) {
331 ret = drm_exec_prepare_obj(exec, objects[i], num_fences);
332 if (unlikely(ret))
333 return ret;
334 }
335
336 return 0;
337 }
338 EXPORT_SYMBOL(drm_exec_prepare_array);
339
340 MODULE_DESCRIPTION("DRM execution context");
341 MODULE_LICENSE("Dual MIT/GPL");
342