1 // SPDX-License-Identifier: GPL-2.0 OR MIT
2 /**************************************************************************
3 *
4 * Copyright (c) 2019-2025 Broadcom. All Rights Reserved. The term
5 * “Broadcom” refers to Broadcom Inc. and/or its subsidiaries.
6 *
7 **************************************************************************/
8 #include "vmwgfx_bo.h"
9 #include "vmwgfx_drv.h"
10
11 /*
12 * Different methods for tracking dirty:
13 * VMW_BO_DIRTY_PAGETABLE - Scan the pagetable for hardware dirty bits
14 * VMW_BO_DIRTY_MKWRITE - Write-protect page table entries and record write-
15 * accesses in the VM mkwrite() callback
16 */
17 enum vmw_bo_dirty_method {
18 VMW_BO_DIRTY_PAGETABLE,
19 VMW_BO_DIRTY_MKWRITE,
20 };
21
22 /*
23 * No dirtied pages at scan trigger a transition to the _MKWRITE method,
24 * similarly a certain percentage of dirty pages trigger a transition to
25 * the _PAGETABLE method. How many triggers should we wait for before
26 * changing method?
27 */
28 #define VMW_DIRTY_NUM_CHANGE_TRIGGERS 2
29
30 /* Percentage to trigger a transition to the _PAGETABLE method */
31 #define VMW_DIRTY_PERCENTAGE 10
32
33 /**
34 * struct vmw_bo_dirty - Dirty information for buffer objects
35 * @start: First currently dirty bit
36 * @end: Last currently dirty bit + 1
37 * @method: The currently used dirty method
38 * @change_count: Number of consecutive method change triggers
39 * @ref_count: Reference count for this structure
40 * @bitmap_size: The size of the bitmap in bits. Typically equal to the
41 * nuber of pages in the bo.
42 * @bitmap: A bitmap where each bit represents a page. A set bit means a
43 * dirty page.
44 */
45 struct vmw_bo_dirty {
46 unsigned long start;
47 unsigned long end;
48 enum vmw_bo_dirty_method method;
49 unsigned int change_count;
50 unsigned int ref_count;
51 unsigned long bitmap_size;
52 unsigned long bitmap[];
53 };
54
vmw_bo_is_dirty(struct vmw_bo * vbo)55 bool vmw_bo_is_dirty(struct vmw_bo *vbo)
56 {
57 return vbo->dirty && (vbo->dirty->start < vbo->dirty->end);
58 }
59
60 /**
61 * vmw_bo_dirty_scan_pagetable - Perform a pagetable scan for dirty bits
62 * @vbo: The buffer object to scan
63 *
64 * Scans the pagetable for dirty bits. Clear those bits and modify the
65 * dirty structure with the results. This function may change the
66 * dirty-tracking method.
67 */
vmw_bo_dirty_scan_pagetable(struct vmw_bo * vbo)68 static void vmw_bo_dirty_scan_pagetable(struct vmw_bo *vbo)
69 {
70 struct vmw_bo_dirty *dirty = vbo->dirty;
71 pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
72 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
73 pgoff_t num_marked;
74
75 num_marked = clean_record_shared_mapping_range
76 (mapping,
77 offset, dirty->bitmap_size,
78 offset, &dirty->bitmap[0],
79 &dirty->start, &dirty->end);
80 if (num_marked == 0)
81 dirty->change_count++;
82 else
83 dirty->change_count = 0;
84
85 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
86 dirty->change_count = 0;
87 dirty->method = VMW_BO_DIRTY_MKWRITE;
88 wp_shared_mapping_range(mapping,
89 offset, dirty->bitmap_size);
90 clean_record_shared_mapping_range(mapping,
91 offset, dirty->bitmap_size,
92 offset, &dirty->bitmap[0],
93 &dirty->start, &dirty->end);
94 }
95 }
96
97 /**
98 * vmw_bo_dirty_scan_mkwrite - Reset the mkwrite dirty-tracking method
99 * @vbo: The buffer object to scan
100 *
101 * Write-protect pages written to so that consecutive write accesses will
102 * trigger a call to mkwrite.
103 *
104 * This function may change the dirty-tracking method.
105 */
vmw_bo_dirty_scan_mkwrite(struct vmw_bo * vbo)106 static void vmw_bo_dirty_scan_mkwrite(struct vmw_bo *vbo)
107 {
108 struct vmw_bo_dirty *dirty = vbo->dirty;
109 unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
110 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
111 pgoff_t num_marked;
112
113 if (dirty->end <= dirty->start)
114 return;
115
116 num_marked = wp_shared_mapping_range(vbo->tbo.bdev->dev_mapping,
117 dirty->start + offset,
118 dirty->end - dirty->start);
119
120 if (100UL * num_marked / dirty->bitmap_size >
121 VMW_DIRTY_PERCENTAGE)
122 dirty->change_count++;
123 else
124 dirty->change_count = 0;
125
126 if (dirty->change_count > VMW_DIRTY_NUM_CHANGE_TRIGGERS) {
127 pgoff_t start = 0;
128 pgoff_t end = dirty->bitmap_size;
129
130 dirty->method = VMW_BO_DIRTY_PAGETABLE;
131 clean_record_shared_mapping_range(mapping, offset, end, offset,
132 &dirty->bitmap[0],
133 &start, &end);
134 bitmap_clear(&dirty->bitmap[0], 0, dirty->bitmap_size);
135 if (dirty->start < dirty->end)
136 bitmap_set(&dirty->bitmap[0], dirty->start,
137 dirty->end - dirty->start);
138 dirty->change_count = 0;
139 }
140 }
141
142 /**
143 * vmw_bo_dirty_scan - Scan for dirty pages and add them to the dirty
144 * tracking structure
145 * @vbo: The buffer object to scan
146 *
147 * This function may change the dirty tracking method.
148 */
vmw_bo_dirty_scan(struct vmw_bo * vbo)149 void vmw_bo_dirty_scan(struct vmw_bo *vbo)
150 {
151 struct vmw_bo_dirty *dirty = vbo->dirty;
152
153 if (dirty->method == VMW_BO_DIRTY_PAGETABLE)
154 vmw_bo_dirty_scan_pagetable(vbo);
155 else
156 vmw_bo_dirty_scan_mkwrite(vbo);
157 }
158
159 /**
160 * vmw_bo_dirty_pre_unmap - write-protect and pick up dirty pages before
161 * an unmap_mapping_range operation.
162 * @vbo: The buffer object,
163 * @start: First page of the range within the buffer object.
164 * @end: Last page of the range within the buffer object + 1.
165 *
166 * If we're using the _PAGETABLE scan method, we may leak dirty pages
167 * when calling unmap_mapping_range(). This function makes sure we pick
168 * up all dirty pages.
169 */
vmw_bo_dirty_pre_unmap(struct vmw_bo * vbo,pgoff_t start,pgoff_t end)170 static void vmw_bo_dirty_pre_unmap(struct vmw_bo *vbo,
171 pgoff_t start, pgoff_t end)
172 {
173 struct vmw_bo_dirty *dirty = vbo->dirty;
174 unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
175 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
176
177 if (dirty->method != VMW_BO_DIRTY_PAGETABLE || start >= end)
178 return;
179
180 wp_shared_mapping_range(mapping, start + offset, end - start);
181 clean_record_shared_mapping_range(mapping, start + offset,
182 end - start, offset,
183 &dirty->bitmap[0], &dirty->start,
184 &dirty->end);
185 }
186
187 /**
188 * vmw_bo_dirty_unmap - Clear all ptes pointing to a range within a bo
189 * @vbo: The buffer object,
190 * @start: First page of the range within the buffer object.
191 * @end: Last page of the range within the buffer object + 1.
192 *
193 * This is similar to ttm_bo_unmap_virtual() except it takes a subrange.
194 */
vmw_bo_dirty_unmap(struct vmw_bo * vbo,pgoff_t start,pgoff_t end)195 void vmw_bo_dirty_unmap(struct vmw_bo *vbo,
196 pgoff_t start, pgoff_t end)
197 {
198 unsigned long offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
199 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
200
201 vmw_bo_dirty_pre_unmap(vbo, start, end);
202 unmap_shared_mapping_range(mapping, (offset + start) << PAGE_SHIFT,
203 (loff_t) (end - start) << PAGE_SHIFT);
204 }
205
206 /**
207 * vmw_bo_dirty_add - Add a dirty-tracking user to a buffer object
208 * @vbo: The buffer object
209 *
210 * This function registers a dirty-tracking user to a buffer object.
211 * A user can be for example a resource or a vma in a special user-space
212 * mapping.
213 *
214 * Return: Zero on success, -ENOMEM on memory allocation failure.
215 */
vmw_bo_dirty_add(struct vmw_bo * vbo)216 int vmw_bo_dirty_add(struct vmw_bo *vbo)
217 {
218 struct vmw_bo_dirty *dirty = vbo->dirty;
219 pgoff_t num_pages = PFN_UP(vbo->tbo.resource->size);
220 size_t size;
221 int ret;
222
223 if (dirty) {
224 dirty->ref_count++;
225 return 0;
226 }
227
228 size = sizeof(*dirty) + BITS_TO_LONGS(num_pages) * sizeof(long);
229 dirty = kvzalloc(size, GFP_KERNEL);
230 if (!dirty) {
231 ret = -ENOMEM;
232 goto out_no_dirty;
233 }
234
235 dirty->bitmap_size = num_pages;
236 dirty->start = dirty->bitmap_size;
237 dirty->end = 0;
238 dirty->ref_count = 1;
239 if (num_pages < PAGE_SIZE / sizeof(pte_t)) {
240 dirty->method = VMW_BO_DIRTY_PAGETABLE;
241 } else {
242 struct address_space *mapping = vbo->tbo.bdev->dev_mapping;
243 pgoff_t offset = drm_vma_node_start(&vbo->tbo.base.vma_node);
244
245 dirty->method = VMW_BO_DIRTY_MKWRITE;
246
247 /* Write-protect and then pick up already dirty bits */
248 wp_shared_mapping_range(mapping, offset, num_pages);
249 clean_record_shared_mapping_range(mapping, offset, num_pages,
250 offset,
251 &dirty->bitmap[0],
252 &dirty->start, &dirty->end);
253 }
254
255 vbo->dirty = dirty;
256
257 return 0;
258
259 out_no_dirty:
260 return ret;
261 }
262
263 /**
264 * vmw_bo_dirty_release - Release a dirty-tracking user from a buffer object
265 * @vbo: The buffer object
266 *
267 * This function releases a dirty-tracking user from a buffer object.
268 * If the reference count reaches zero, then the dirty-tracking object is
269 * freed and the pointer to it cleared.
270 *
271 * Return: Zero on success, -ENOMEM on memory allocation failure.
272 */
vmw_bo_dirty_release(struct vmw_bo * vbo)273 void vmw_bo_dirty_release(struct vmw_bo *vbo)
274 {
275 struct vmw_bo_dirty *dirty = vbo->dirty;
276
277 if (dirty && --dirty->ref_count == 0) {
278 kvfree(dirty);
279 vbo->dirty = NULL;
280 }
281 }
282
283 /**
284 * vmw_bo_dirty_transfer_to_res - Pick up a resource's dirty region from
285 * its backing mob.
286 * @res: The resource
287 *
288 * This function will pick up all dirty ranges affecting the resource from
289 * it's backup mob, and call vmw_resource_dirty_update() once for each
290 * range. The transferred ranges will be cleared from the backing mob's
291 * dirty tracking.
292 */
vmw_bo_dirty_transfer_to_res(struct vmw_resource * res)293 void vmw_bo_dirty_transfer_to_res(struct vmw_resource *res)
294 {
295 struct vmw_bo *vbo = res->guest_memory_bo;
296 struct vmw_bo_dirty *dirty = vbo->dirty;
297 pgoff_t start, cur, end;
298 unsigned long res_start = res->guest_memory_offset;
299 unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
300
301 WARN_ON_ONCE(res_start & ~PAGE_MASK);
302 res_start >>= PAGE_SHIFT;
303 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
304
305 if (res_start >= dirty->end || res_end <= dirty->start)
306 return;
307
308 cur = max(res_start, dirty->start);
309 res_end = max(res_end, dirty->end);
310 while (cur < res_end) {
311 unsigned long num;
312
313 start = find_next_bit(&dirty->bitmap[0], res_end, cur);
314 if (start >= res_end)
315 break;
316
317 end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
318 cur = end + 1;
319 num = end - start;
320 bitmap_clear(&dirty->bitmap[0], start, num);
321 vmw_resource_dirty_update(res, start, end);
322 }
323
324 if (res_start <= dirty->start && res_end > dirty->start)
325 dirty->start = res_end;
326 if (res_start < dirty->end && res_end >= dirty->end)
327 dirty->end = res_start;
328 }
329
vmw_bo_dirty_clear(struct vmw_bo * vbo)330 void vmw_bo_dirty_clear(struct vmw_bo *vbo)
331 {
332 struct vmw_bo_dirty *dirty = vbo->dirty;
333 pgoff_t start, cur, end;
334 unsigned long res_start = 0;
335 unsigned long res_end = vbo->tbo.base.size;
336
337 WARN_ON_ONCE(res_start & ~PAGE_MASK);
338 res_start >>= PAGE_SHIFT;
339 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
340
341 if (res_start >= dirty->end || res_end <= dirty->start)
342 return;
343
344 cur = max(res_start, dirty->start);
345 res_end = max(res_end, dirty->end);
346 while (cur < res_end) {
347 unsigned long num;
348
349 start = find_next_bit(&dirty->bitmap[0], res_end, cur);
350 if (start >= res_end)
351 break;
352
353 end = find_next_zero_bit(&dirty->bitmap[0], res_end, start + 1);
354 cur = end + 1;
355 num = end - start;
356 bitmap_clear(&dirty->bitmap[0], start, num);
357 }
358
359 if (res_start <= dirty->start && res_end > dirty->start)
360 dirty->start = res_end;
361 if (res_start < dirty->end && res_end >= dirty->end)
362 dirty->end = res_start;
363 }
364
365 /**
366 * vmw_bo_dirty_clear_res - Clear a resource's dirty region from
367 * its backing mob.
368 * @res: The resource
369 *
370 * This function will clear all dirty ranges affecting the resource from
371 * it's backup mob's dirty tracking.
372 */
vmw_bo_dirty_clear_res(struct vmw_resource * res)373 void vmw_bo_dirty_clear_res(struct vmw_resource *res)
374 {
375 unsigned long res_start = res->guest_memory_offset;
376 unsigned long res_end = res->guest_memory_offset + res->guest_memory_size;
377 struct vmw_bo *vbo = res->guest_memory_bo;
378 struct vmw_bo_dirty *dirty = vbo->dirty;
379
380 res_start >>= PAGE_SHIFT;
381 res_end = DIV_ROUND_UP(res_end, PAGE_SIZE);
382
383 if (res_start >= dirty->end || res_end <= dirty->start)
384 return;
385
386 res_start = max(res_start, dirty->start);
387 res_end = min(res_end, dirty->end);
388 bitmap_clear(&dirty->bitmap[0], res_start, res_end - res_start);
389
390 if (res_start <= dirty->start && res_end > dirty->start)
391 dirty->start = res_end;
392 if (res_start < dirty->end && res_end >= dirty->end)
393 dirty->end = res_start;
394 }
395
vmw_bo_vm_mkwrite(struct vm_fault * vmf)396 vm_fault_t vmw_bo_vm_mkwrite(struct vm_fault *vmf)
397 {
398 struct vm_area_struct *vma = vmf->vma;
399 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
400 vma->vm_private_data;
401 vm_fault_t ret;
402 unsigned long page_offset;
403 unsigned int save_flags;
404 struct vmw_bo *vbo = to_vmw_bo(&bo->base);
405
406 /*
407 * mkwrite() doesn't handle the VM_FAULT_RETRY return value correctly.
408 * So make sure the TTM helpers are aware.
409 */
410 save_flags = vmf->flags;
411 vmf->flags &= ~FAULT_FLAG_ALLOW_RETRY;
412 ret = ttm_bo_vm_reserve(bo, vmf);
413 vmf->flags = save_flags;
414 if (ret)
415 return ret;
416
417 page_offset = vmf->pgoff - drm_vma_node_start(&bo->base.vma_node);
418 if (unlikely(page_offset >= PFN_UP(bo->resource->size))) {
419 ret = VM_FAULT_SIGBUS;
420 goto out_unlock;
421 }
422
423 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE &&
424 !test_bit(page_offset, &vbo->dirty->bitmap[0])) {
425 struct vmw_bo_dirty *dirty = vbo->dirty;
426
427 __set_bit(page_offset, &dirty->bitmap[0]);
428 dirty->start = min(dirty->start, page_offset);
429 dirty->end = max(dirty->end, page_offset + 1);
430 }
431
432 out_unlock:
433 dma_resv_unlock(bo->base.resv);
434 return ret;
435 }
436
vmw_bo_vm_fault(struct vm_fault * vmf)437 vm_fault_t vmw_bo_vm_fault(struct vm_fault *vmf)
438 {
439 struct vm_area_struct *vma = vmf->vma;
440 struct ttm_buffer_object *bo = (struct ttm_buffer_object *)
441 vma->vm_private_data;
442 struct vmw_bo *vbo = to_vmw_bo(&bo->base);
443 pgoff_t num_prefault;
444 pgprot_t prot;
445 vm_fault_t ret;
446
447 ret = ttm_bo_vm_reserve(bo, vmf);
448 if (ret)
449 return ret;
450
451 num_prefault = (vma->vm_flags & VM_RAND_READ) ? 1 :
452 TTM_BO_VM_NUM_PREFAULT;
453
454 if (vbo->dirty) {
455 pgoff_t allowed_prefault;
456 unsigned long page_offset;
457
458 page_offset = vmf->pgoff -
459 drm_vma_node_start(&bo->base.vma_node);
460 if (page_offset >= PFN_UP(bo->resource->size) ||
461 vmw_resources_clean(vbo, page_offset,
462 page_offset + PAGE_SIZE,
463 &allowed_prefault)) {
464 ret = VM_FAULT_SIGBUS;
465 goto out_unlock;
466 }
467
468 num_prefault = min(num_prefault, allowed_prefault);
469 }
470
471 /*
472 * If we don't track dirty using the MKWRITE method, make sure
473 * sure the page protection is write-enabled so we don't get
474 * a lot of unnecessary write faults.
475 */
476 if (vbo->dirty && vbo->dirty->method == VMW_BO_DIRTY_MKWRITE)
477 prot = vm_get_page_prot(vma->vm_flags & ~VM_SHARED);
478 else
479 prot = vm_get_page_prot(vma->vm_flags);
480
481 ret = ttm_bo_vm_fault_reserved(vmf, prot, num_prefault);
482 if (ret == VM_FAULT_RETRY && !(vmf->flags & FAULT_FLAG_RETRY_NOWAIT))
483 return ret;
484
485 out_unlock:
486 dma_resv_unlock(bo->base.resv);
487
488 return ret;
489 }
490