1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22 /*
23 * Copyright (c) 2014 by Chunwei Chen. All rights reserved.
24 * Copyright (c) 2019 by Delphix. All rights reserved.
25 * Copyright (c) 2023, 2024, Klara Inc.
26 * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
27 */
28
29 /*
30 * See abd.c for a general overview of the arc buffered data (ABD).
31 *
32 * Linear buffers act exactly like normal buffers and are always mapped into the
33 * kernel's virtual memory space, while scattered ABD data chunks are allocated
34 * as physical pages and then mapped in only while they are actually being
35 * accessed through one of the abd_* library functions. Using scattered ABDs
36 * provides several benefits:
37 *
38 * (1) They avoid use of kmem_*, preventing performance problems where running
39 * kmem_reap on very large memory systems never finishes and causes
40 * constant TLB shootdowns.
41 *
42 * (2) Fragmentation is less of an issue since when we are at the limit of
43 * allocatable space, we won't have to search around for a long free
44 * hole in the VA space for large ARC allocations. Each chunk is mapped in
45 * individually, so even if we are using HIGHMEM (see next point) we
46 * wouldn't need to worry about finding a contiguous address range.
47 *
48 * (3) If we are not using HIGHMEM, then all physical memory is always
49 * mapped into the kernel's address space, so we also avoid the map /
50 * unmap costs on each ABD access.
51 *
52 * If we are not using HIGHMEM, scattered buffers which have only one chunk
53 * can be treated as linear buffers, because they are contiguous in the
54 * kernel's virtual address space. See abd_alloc_chunks() for details.
55 */
56
57 #include <sys/abd_impl.h>
58 #include <sys/param.h>
59 #include <sys/zio.h>
60 #include <sys/arc.h>
61 #include <sys/zfs_context.h>
62 #include <sys/zfs_znode.h>
63 #include <linux/kmap_compat.h>
64 #include <linux/mm_compat.h>
65 #include <linux/scatterlist.h>
66 #include <linux/version.h>
67
68 #if defined(MAX_ORDER)
69 #define ABD_MAX_ORDER (MAX_ORDER)
70 #elif defined(MAX_PAGE_ORDER)
71 #define ABD_MAX_ORDER (MAX_PAGE_ORDER)
72 #endif
73
74 typedef struct abd_stats {
75 kstat_named_t abdstat_struct_size;
76 kstat_named_t abdstat_linear_cnt;
77 kstat_named_t abdstat_linear_data_size;
78 kstat_named_t abdstat_scatter_cnt;
79 kstat_named_t abdstat_scatter_data_size;
80 kstat_named_t abdstat_scatter_chunk_waste;
81 kstat_named_t abdstat_scatter_orders[ABD_MAX_ORDER];
82 kstat_named_t abdstat_scatter_page_multi_chunk;
83 kstat_named_t abdstat_scatter_page_multi_zone;
84 kstat_named_t abdstat_scatter_page_alloc_retry;
85 kstat_named_t abdstat_scatter_sg_table_retry;
86 } abd_stats_t;
87
88 static abd_stats_t abd_stats = {
89 /* Amount of memory occupied by all of the abd_t struct allocations */
90 { "struct_size", KSTAT_DATA_UINT64 },
91 /*
92 * The number of linear ABDs which are currently allocated, excluding
93 * ABDs which don't own their data (for instance the ones which were
94 * allocated through abd_get_offset() and abd_get_from_buf()). If an
95 * ABD takes ownership of its buf then it will become tracked.
96 */
97 { "linear_cnt", KSTAT_DATA_UINT64 },
98 /* Amount of data stored in all linear ABDs tracked by linear_cnt */
99 { "linear_data_size", KSTAT_DATA_UINT64 },
100 /*
101 * The number of scatter ABDs which are currently allocated, excluding
102 * ABDs which don't own their data (for instance the ones which were
103 * allocated through abd_get_offset()).
104 */
105 { "scatter_cnt", KSTAT_DATA_UINT64 },
106 /* Amount of data stored in all scatter ABDs tracked by scatter_cnt */
107 { "scatter_data_size", KSTAT_DATA_UINT64 },
108 /*
109 * The amount of space wasted at the end of the last chunk across all
110 * scatter ABDs tracked by scatter_cnt.
111 */
112 { "scatter_chunk_waste", KSTAT_DATA_UINT64 },
113 /*
114 * The number of compound allocations of a given order. These
115 * allocations are spread over all currently allocated ABDs, and
116 * act as a measure of memory fragmentation.
117 */
118 { { "scatter_order_N", KSTAT_DATA_UINT64 } },
119 /*
120 * The number of scatter ABDs which contain multiple chunks.
121 * ABDs are preferentially allocated from the minimum number of
122 * contiguous multi-page chunks, a single chunk is optimal.
123 */
124 { "scatter_page_multi_chunk", KSTAT_DATA_UINT64 },
125 /*
126 * The number of scatter ABDs which are split across memory zones.
127 * ABDs are preferentially allocated using pages from a single zone.
128 */
129 { "scatter_page_multi_zone", KSTAT_DATA_UINT64 },
130 /*
131 * The total number of retries encountered when attempting to
132 * allocate the pages to populate the scatter ABD.
133 */
134 { "scatter_page_alloc_retry", KSTAT_DATA_UINT64 },
135 /*
136 * The total number of retries encountered when attempting to
137 * allocate the sg table for an ABD.
138 */
139 { "scatter_sg_table_retry", KSTAT_DATA_UINT64 },
140 };
141
142 static struct {
143 wmsum_t abdstat_struct_size;
144 wmsum_t abdstat_linear_cnt;
145 wmsum_t abdstat_linear_data_size;
146 wmsum_t abdstat_scatter_cnt;
147 wmsum_t abdstat_scatter_data_size;
148 wmsum_t abdstat_scatter_chunk_waste;
149 wmsum_t abdstat_scatter_orders[ABD_MAX_ORDER];
150 wmsum_t abdstat_scatter_page_multi_chunk;
151 wmsum_t abdstat_scatter_page_multi_zone;
152 wmsum_t abdstat_scatter_page_alloc_retry;
153 wmsum_t abdstat_scatter_sg_table_retry;
154 } abd_sums;
155
156 #define abd_for_each_sg(abd, sg, n, i) \
157 for_each_sg(ABD_SCATTER(abd).abd_sgl, sg, n, i)
158
159 /*
160 * zfs_abd_scatter_min_size is the minimum allocation size to use scatter
161 * ABD's. Smaller allocations will use linear ABD's which uses
162 * zio_[data_]buf_alloc().
163 *
164 * Scatter ABD's use at least one page each, so sub-page allocations waste
165 * some space when allocated as scatter (e.g. 2KB scatter allocation wastes
166 * half of each page). Using linear ABD's for small allocations means that
167 * they will be put on slabs which contain many allocations. This can
168 * improve memory efficiency, but it also makes it much harder for ARC
169 * evictions to actually free pages, because all the buffers on one slab need
170 * to be freed in order for the slab (and underlying pages) to be freed.
171 * Typically, 512B and 1KB kmem caches have 16 buffers per slab, so it's
172 * possible for them to actually waste more memory than scatter (one page per
173 * buf = wasting 3/4 or 7/8th; one buf per slab = wasting 15/16th).
174 *
175 * Spill blocks are typically 512B and are heavily used on systems running
176 * selinux with the default dnode size and the `xattr=sa` property set.
177 *
178 * By default we use linear allocations for 512B and 1KB, and scatter
179 * allocations for larger (1.5KB and up).
180 */
181 static int zfs_abd_scatter_min_size = 512 * 3;
182
183 /*
184 * We use a scattered SPA_MAXBLOCKSIZE sized ABD whose pages are
185 * just a single zero'd page. This allows us to conserve memory by
186 * only using a single zero page for the scatterlist.
187 */
188 abd_t *abd_zero_scatter = NULL;
189
190 struct page;
191
192 /*
193 * abd_zero_page is assigned to each of the pages of abd_zero_scatter. It will
194 * point to ZERO_PAGE if it is available or it will be an allocated zero'd
195 * PAGESIZE buffer.
196 */
197 static struct page *abd_zero_page = NULL;
198
199 static kmem_cache_t *abd_cache = NULL;
200 static kstat_t *abd_ksp;
201
202 static uint_t
abd_chunkcnt_for_bytes(size_t size)203 abd_chunkcnt_for_bytes(size_t size)
204 {
205 return (P2ROUNDUP(size, PAGESIZE) / PAGESIZE);
206 }
207
208 abd_t *
abd_alloc_struct_impl(size_t size)209 abd_alloc_struct_impl(size_t size)
210 {
211 /*
212 * In Linux we do not use the size passed in during ABD
213 * allocation, so we just ignore it.
214 */
215 (void) size;
216 abd_t *abd = kmem_cache_alloc(abd_cache, KM_PUSHPAGE);
217 ASSERT3P(abd, !=, NULL);
218 ABDSTAT_INCR(abdstat_struct_size, sizeof (abd_t));
219
220 return (abd);
221 }
222
223 void
abd_free_struct_impl(abd_t * abd)224 abd_free_struct_impl(abd_t *abd)
225 {
226 kmem_cache_free(abd_cache, abd);
227 ABDSTAT_INCR(abdstat_struct_size, -(int)sizeof (abd_t));
228 }
229
230 static unsigned zfs_abd_scatter_max_order = ABD_MAX_ORDER - 1;
231
232 /*
233 * Mark zfs data pages so they can be excluded from kernel crash dumps
234 */
235 #ifdef _LP64
236 #define ABD_FILE_CACHE_PAGE 0x2F5ABDF11ECAC4E
237
238 static inline void
abd_mark_zfs_page(struct page * page)239 abd_mark_zfs_page(struct page *page)
240 {
241 get_page(page);
242 SetPagePrivate(page);
243 set_page_private(page, ABD_FILE_CACHE_PAGE);
244 }
245
246 static inline void
abd_unmark_zfs_page(struct page * page)247 abd_unmark_zfs_page(struct page *page)
248 {
249 set_page_private(page, 0UL);
250 ClearPagePrivate(page);
251 put_page(page);
252 }
253 #else
254 #define abd_mark_zfs_page(page)
255 #define abd_unmark_zfs_page(page)
256 #endif /* _LP64 */
257
258 #ifndef CONFIG_HIGHMEM
259
260 /*
261 * The goal is to minimize fragmentation by preferentially populating ABDs
262 * with higher order compound pages from a single zone. Allocation size is
263 * progressively decreased until it can be satisfied without performing
264 * reclaim or compaction. When necessary this function will degenerate to
265 * allocating individual pages and allowing reclaim to satisfy allocations.
266 */
267 void
abd_alloc_chunks(abd_t * abd,size_t size)268 abd_alloc_chunks(abd_t *abd, size_t size)
269 {
270 struct list_head pages;
271 struct sg_table table;
272 struct scatterlist *sg;
273 struct page *page, *tmp_page = NULL;
274 gfp_t gfp = __GFP_RECLAIMABLE | __GFP_NOWARN | GFP_NOIO;
275 gfp_t gfp_comp = (gfp | __GFP_NORETRY | __GFP_COMP) & ~__GFP_RECLAIM;
276 unsigned int max_order = MIN(zfs_abd_scatter_max_order,
277 ABD_MAX_ORDER - 1);
278 unsigned int nr_pages = abd_chunkcnt_for_bytes(size);
279 unsigned int chunks = 0, zones = 0;
280 size_t remaining_size;
281 int nid = NUMA_NO_NODE;
282 unsigned int alloc_pages = 0;
283
284 INIT_LIST_HEAD(&pages);
285
286 ASSERT3U(alloc_pages, <, nr_pages);
287
288 while (alloc_pages < nr_pages) {
289 unsigned int chunk_pages;
290 unsigned int order;
291
292 order = MIN(highbit64(nr_pages - alloc_pages) - 1, max_order);
293 chunk_pages = (1U << order);
294
295 page = alloc_pages_node(nid, order ? gfp_comp : gfp, order);
296 if (page == NULL) {
297 if (order == 0) {
298 ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
299 schedule_timeout_interruptible(1);
300 } else {
301 max_order = MAX(0, order - 1);
302 }
303 continue;
304 }
305
306 list_add_tail(&page->lru, &pages);
307
308 if ((nid != NUMA_NO_NODE) && (page_to_nid(page) != nid))
309 zones++;
310
311 nid = page_to_nid(page);
312 ABDSTAT_BUMP(abdstat_scatter_orders[order]);
313 chunks++;
314 alloc_pages += chunk_pages;
315 }
316
317 ASSERT3S(alloc_pages, ==, nr_pages);
318
319 while (sg_alloc_table(&table, chunks, gfp)) {
320 ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
321 schedule_timeout_interruptible(1);
322 }
323
324 sg = table.sgl;
325 remaining_size = size;
326 list_for_each_entry_safe(page, tmp_page, &pages, lru) {
327 size_t sg_size = MIN(PAGESIZE << compound_order(page),
328 remaining_size);
329 sg_set_page(sg, page, sg_size, 0);
330 abd_mark_zfs_page(page);
331 remaining_size -= sg_size;
332
333 sg = sg_next(sg);
334 list_del(&page->lru);
335 }
336
337 /*
338 * These conditions ensure that a possible transformation to a linear
339 * ABD would be valid.
340 */
341 ASSERT(!PageHighMem(sg_page(table.sgl)));
342 ASSERT0(ABD_SCATTER(abd).abd_offset);
343
344 if (table.nents == 1) {
345 /*
346 * Since there is only one entry, this ABD can be represented
347 * as a linear buffer. All single-page (4K) ABD's can be
348 * represented this way. Some multi-page ABD's can also be
349 * represented this way, if we were able to allocate a single
350 * "chunk" (higher-order "page" which represents a power-of-2
351 * series of physically-contiguous pages). This is often the
352 * case for 2-page (8K) ABD's.
353 *
354 * Representing a single-entry scatter ABD as a linear ABD
355 * has the performance advantage of avoiding the copy (and
356 * allocation) in abd_borrow_buf_copy / abd_return_buf_copy.
357 * A performance increase of around 5% has been observed for
358 * ARC-cached reads (of small blocks which can take advantage
359 * of this).
360 *
361 * Note that this optimization is only possible because the
362 * pages are always mapped into the kernel's address space.
363 * This is not the case for highmem pages, so the
364 * optimization can not be made there.
365 */
366 abd->abd_flags |= ABD_FLAG_LINEAR;
367 abd->abd_flags |= ABD_FLAG_LINEAR_PAGE;
368 abd->abd_u.abd_linear.abd_sgl = table.sgl;
369 ABD_LINEAR_BUF(abd) = page_address(sg_page(table.sgl));
370 } else if (table.nents > 1) {
371 ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
372 abd->abd_flags |= ABD_FLAG_MULTI_CHUNK;
373
374 if (zones) {
375 ABDSTAT_BUMP(abdstat_scatter_page_multi_zone);
376 abd->abd_flags |= ABD_FLAG_MULTI_ZONE;
377 }
378
379 ABD_SCATTER(abd).abd_sgl = table.sgl;
380 ABD_SCATTER(abd).abd_nents = table.nents;
381 }
382 }
383 #else
384
385 /*
386 * Allocate N individual pages to construct a scatter ABD. This function
387 * makes no attempt to request contiguous pages and requires the minimal
388 * number of kernel interfaces. It's designed for maximum compatibility.
389 */
390 void
abd_alloc_chunks(abd_t * abd,size_t size)391 abd_alloc_chunks(abd_t *abd, size_t size)
392 {
393 struct scatterlist *sg = NULL;
394 struct sg_table table;
395 struct page *page;
396 gfp_t gfp = __GFP_RECLAIMABLE | __GFP_NOWARN | GFP_NOIO;
397 int nr_pages = abd_chunkcnt_for_bytes(size);
398 int i = 0;
399
400 while (sg_alloc_table(&table, nr_pages, gfp)) {
401 ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
402 schedule_timeout_interruptible(1);
403 }
404
405 ASSERT3U(table.nents, ==, nr_pages);
406 ABD_SCATTER(abd).abd_sgl = table.sgl;
407 ABD_SCATTER(abd).abd_nents = nr_pages;
408
409 abd_for_each_sg(abd, sg, nr_pages, i) {
410 while ((page = __page_cache_alloc(gfp)) == NULL) {
411 ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
412 schedule_timeout_interruptible(1);
413 }
414
415 ABDSTAT_BUMP(abdstat_scatter_orders[0]);
416 sg_set_page(sg, page, PAGESIZE, 0);
417 abd_mark_zfs_page(page);
418 }
419
420 if (nr_pages > 1) {
421 ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
422 abd->abd_flags |= ABD_FLAG_MULTI_CHUNK;
423 }
424 }
425 #endif /* !CONFIG_HIGHMEM */
426
427 /*
428 * This must be called if any of the sg_table allocation functions
429 * are called.
430 */
431 static void
abd_free_sg_table(abd_t * abd)432 abd_free_sg_table(abd_t *abd)
433 {
434 struct sg_table table;
435
436 table.sgl = ABD_SCATTER(abd).abd_sgl;
437 table.nents = table.orig_nents = ABD_SCATTER(abd).abd_nents;
438 sg_free_table(&table);
439 }
440
441 void
abd_free_chunks(abd_t * abd)442 abd_free_chunks(abd_t *abd)
443 {
444 struct scatterlist *sg = NULL;
445 struct page *page;
446 int nr_pages = ABD_SCATTER(abd).abd_nents;
447 int order, i = 0;
448
449 if (abd->abd_flags & ABD_FLAG_MULTI_ZONE)
450 ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_zone);
451
452 if (abd->abd_flags & ABD_FLAG_MULTI_CHUNK)
453 ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_chunk);
454
455 /*
456 * Scatter ABDs may be constructed by abd_alloc_from_pages() from
457 * an array of pages. In which case they should not be freed.
458 */
459 if (!abd_is_from_pages(abd)) {
460 abd_for_each_sg(abd, sg, nr_pages, i) {
461 page = sg_page(sg);
462 abd_unmark_zfs_page(page);
463 order = compound_order(page);
464 __free_pages(page, order);
465 ASSERT3U(sg->length, <=, PAGE_SIZE << order);
466 ABDSTAT_BUMPDOWN(abdstat_scatter_orders[order]);
467 }
468 }
469
470 abd_free_sg_table(abd);
471 }
472
473 /*
474 * Allocate scatter ABD of size SPA_MAXBLOCKSIZE, where each page in
475 * the scatterlist will be set to the zero'd out buffer abd_zero_page.
476 */
477 static void
abd_alloc_zero_scatter(void)478 abd_alloc_zero_scatter(void)
479 {
480 struct scatterlist *sg = NULL;
481 struct sg_table table;
482 gfp_t gfp = __GFP_NOWARN | GFP_NOIO;
483 int nr_pages = abd_chunkcnt_for_bytes(SPA_MAXBLOCKSIZE);
484 int i = 0;
485
486 #if defined(HAVE_ZERO_PAGE_GPL_ONLY)
487 gfp_t gfp_zero_page = gfp | __GFP_ZERO;
488 while ((abd_zero_page = __page_cache_alloc(gfp_zero_page)) == NULL) {
489 ABDSTAT_BUMP(abdstat_scatter_page_alloc_retry);
490 schedule_timeout_interruptible(1);
491 }
492 abd_mark_zfs_page(abd_zero_page);
493 #else
494 abd_zero_page = ZERO_PAGE(0);
495 #endif /* HAVE_ZERO_PAGE_GPL_ONLY */
496
497 while (sg_alloc_table(&table, nr_pages, gfp)) {
498 ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
499 schedule_timeout_interruptible(1);
500 }
501 ASSERT3U(table.nents, ==, nr_pages);
502
503 abd_zero_scatter = abd_alloc_struct(SPA_MAXBLOCKSIZE);
504 abd_zero_scatter->abd_flags |= ABD_FLAG_OWNER;
505 ABD_SCATTER(abd_zero_scatter).abd_offset = 0;
506 ABD_SCATTER(abd_zero_scatter).abd_sgl = table.sgl;
507 ABD_SCATTER(abd_zero_scatter).abd_nents = nr_pages;
508 abd_zero_scatter->abd_size = SPA_MAXBLOCKSIZE;
509 abd_zero_scatter->abd_flags |= ABD_FLAG_MULTI_CHUNK;
510
511 abd_for_each_sg(abd_zero_scatter, sg, nr_pages, i) {
512 sg_set_page(sg, abd_zero_page, PAGESIZE, 0);
513 }
514
515 ABDSTAT_BUMP(abdstat_scatter_cnt);
516 ABDSTAT_INCR(abdstat_scatter_data_size, PAGESIZE);
517 ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
518 }
519
520 boolean_t
abd_size_alloc_linear(size_t size)521 abd_size_alloc_linear(size_t size)
522 {
523 return (!zfs_abd_scatter_enabled || size < zfs_abd_scatter_min_size);
524 }
525
526 void
abd_update_scatter_stats(abd_t * abd,abd_stats_op_t op)527 abd_update_scatter_stats(abd_t *abd, abd_stats_op_t op)
528 {
529 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR);
530 int waste = P2ROUNDUP(abd->abd_size, PAGESIZE) - abd->abd_size;
531 if (op == ABDSTAT_INCR) {
532 ABDSTAT_BUMP(abdstat_scatter_cnt);
533 ABDSTAT_INCR(abdstat_scatter_data_size, abd->abd_size);
534 ABDSTAT_INCR(abdstat_scatter_chunk_waste, waste);
535 arc_space_consume(waste, ARC_SPACE_ABD_CHUNK_WASTE);
536 } else {
537 ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
538 ABDSTAT_INCR(abdstat_scatter_data_size, -(int)abd->abd_size);
539 ABDSTAT_INCR(abdstat_scatter_chunk_waste, -waste);
540 arc_space_return(waste, ARC_SPACE_ABD_CHUNK_WASTE);
541 }
542 }
543
544 void
abd_update_linear_stats(abd_t * abd,abd_stats_op_t op)545 abd_update_linear_stats(abd_t *abd, abd_stats_op_t op)
546 {
547 ASSERT(op == ABDSTAT_INCR || op == ABDSTAT_DECR);
548 if (op == ABDSTAT_INCR) {
549 ABDSTAT_BUMP(abdstat_linear_cnt);
550 ABDSTAT_INCR(abdstat_linear_data_size, abd->abd_size);
551 } else {
552 ABDSTAT_BUMPDOWN(abdstat_linear_cnt);
553 ABDSTAT_INCR(abdstat_linear_data_size, -(int)abd->abd_size);
554 }
555 }
556
557 void
abd_verify_scatter(abd_t * abd)558 abd_verify_scatter(abd_t *abd)
559 {
560 ASSERT3U(ABD_SCATTER(abd).abd_nents, >, 0);
561 ASSERT3U(ABD_SCATTER(abd).abd_offset, <,
562 ABD_SCATTER(abd).abd_sgl->length);
563
564 #ifdef ZFS_DEBUG
565 struct scatterlist *sg = NULL;
566 size_t n = ABD_SCATTER(abd).abd_nents;
567 int i = 0;
568
569 abd_for_each_sg(abd, sg, n, i) {
570 ASSERT3P(sg_page(sg), !=, NULL);
571 }
572 #endif
573 }
574
575 static void
abd_free_zero_scatter(void)576 abd_free_zero_scatter(void)
577 {
578 ABDSTAT_BUMPDOWN(abdstat_scatter_cnt);
579 ABDSTAT_INCR(abdstat_scatter_data_size, -(int)PAGESIZE);
580 ABDSTAT_BUMPDOWN(abdstat_scatter_page_multi_chunk);
581
582 abd_free_sg_table(abd_zero_scatter);
583 abd_free_struct(abd_zero_scatter);
584 abd_zero_scatter = NULL;
585 ASSERT3P(abd_zero_page, !=, NULL);
586 #if defined(HAVE_ZERO_PAGE_GPL_ONLY)
587 abd_unmark_zfs_page(abd_zero_page);
588 __free_page(abd_zero_page);
589 #endif /* HAVE_ZERO_PAGE_GPL_ONLY */
590 }
591
592 static int
abd_kstats_update(kstat_t * ksp,int rw)593 abd_kstats_update(kstat_t *ksp, int rw)
594 {
595 abd_stats_t *as = ksp->ks_data;
596
597 if (rw == KSTAT_WRITE)
598 return (EACCES);
599 as->abdstat_struct_size.value.ui64 =
600 wmsum_value(&abd_sums.abdstat_struct_size);
601 as->abdstat_linear_cnt.value.ui64 =
602 wmsum_value(&abd_sums.abdstat_linear_cnt);
603 as->abdstat_linear_data_size.value.ui64 =
604 wmsum_value(&abd_sums.abdstat_linear_data_size);
605 as->abdstat_scatter_cnt.value.ui64 =
606 wmsum_value(&abd_sums.abdstat_scatter_cnt);
607 as->abdstat_scatter_data_size.value.ui64 =
608 wmsum_value(&abd_sums.abdstat_scatter_data_size);
609 as->abdstat_scatter_chunk_waste.value.ui64 =
610 wmsum_value(&abd_sums.abdstat_scatter_chunk_waste);
611 for (int i = 0; i < ABD_MAX_ORDER; i++) {
612 as->abdstat_scatter_orders[i].value.ui64 =
613 wmsum_value(&abd_sums.abdstat_scatter_orders[i]);
614 }
615 as->abdstat_scatter_page_multi_chunk.value.ui64 =
616 wmsum_value(&abd_sums.abdstat_scatter_page_multi_chunk);
617 as->abdstat_scatter_page_multi_zone.value.ui64 =
618 wmsum_value(&abd_sums.abdstat_scatter_page_multi_zone);
619 as->abdstat_scatter_page_alloc_retry.value.ui64 =
620 wmsum_value(&abd_sums.abdstat_scatter_page_alloc_retry);
621 as->abdstat_scatter_sg_table_retry.value.ui64 =
622 wmsum_value(&abd_sums.abdstat_scatter_sg_table_retry);
623 return (0);
624 }
625
626 void
abd_init(void)627 abd_init(void)
628 {
629 int i;
630
631 abd_cache = kmem_cache_create("abd_t", sizeof (abd_t),
632 0, NULL, NULL, NULL, NULL, NULL, KMC_RECLAIMABLE);
633
634 wmsum_init(&abd_sums.abdstat_struct_size, 0);
635 wmsum_init(&abd_sums.abdstat_linear_cnt, 0);
636 wmsum_init(&abd_sums.abdstat_linear_data_size, 0);
637 wmsum_init(&abd_sums.abdstat_scatter_cnt, 0);
638 wmsum_init(&abd_sums.abdstat_scatter_data_size, 0);
639 wmsum_init(&abd_sums.abdstat_scatter_chunk_waste, 0);
640 for (i = 0; i < ABD_MAX_ORDER; i++)
641 wmsum_init(&abd_sums.abdstat_scatter_orders[i], 0);
642 wmsum_init(&abd_sums.abdstat_scatter_page_multi_chunk, 0);
643 wmsum_init(&abd_sums.abdstat_scatter_page_multi_zone, 0);
644 wmsum_init(&abd_sums.abdstat_scatter_page_alloc_retry, 0);
645 wmsum_init(&abd_sums.abdstat_scatter_sg_table_retry, 0);
646
647 abd_ksp = kstat_create("zfs", 0, "abdstats", "misc", KSTAT_TYPE_NAMED,
648 sizeof (abd_stats) / sizeof (kstat_named_t), KSTAT_FLAG_VIRTUAL);
649 if (abd_ksp != NULL) {
650 for (i = 0; i < ABD_MAX_ORDER; i++) {
651 snprintf(abd_stats.abdstat_scatter_orders[i].name,
652 KSTAT_STRLEN, "scatter_order_%d", i);
653 abd_stats.abdstat_scatter_orders[i].data_type =
654 KSTAT_DATA_UINT64;
655 }
656 abd_ksp->ks_data = &abd_stats;
657 abd_ksp->ks_update = abd_kstats_update;
658 kstat_install(abd_ksp);
659 }
660
661 abd_alloc_zero_scatter();
662 }
663
664 void
abd_fini(void)665 abd_fini(void)
666 {
667 abd_free_zero_scatter();
668
669 if (abd_ksp != NULL) {
670 kstat_delete(abd_ksp);
671 abd_ksp = NULL;
672 }
673
674 wmsum_fini(&abd_sums.abdstat_struct_size);
675 wmsum_fini(&abd_sums.abdstat_linear_cnt);
676 wmsum_fini(&abd_sums.abdstat_linear_data_size);
677 wmsum_fini(&abd_sums.abdstat_scatter_cnt);
678 wmsum_fini(&abd_sums.abdstat_scatter_data_size);
679 wmsum_fini(&abd_sums.abdstat_scatter_chunk_waste);
680 for (int i = 0; i < ABD_MAX_ORDER; i++)
681 wmsum_fini(&abd_sums.abdstat_scatter_orders[i]);
682 wmsum_fini(&abd_sums.abdstat_scatter_page_multi_chunk);
683 wmsum_fini(&abd_sums.abdstat_scatter_page_multi_zone);
684 wmsum_fini(&abd_sums.abdstat_scatter_page_alloc_retry);
685 wmsum_fini(&abd_sums.abdstat_scatter_sg_table_retry);
686
687 if (abd_cache) {
688 kmem_cache_destroy(abd_cache);
689 abd_cache = NULL;
690 }
691 }
692
693 void
abd_free_linear_page(abd_t * abd)694 abd_free_linear_page(abd_t *abd)
695 {
696 /* Transform it back into a scatter ABD for freeing */
697 struct scatterlist *sg = abd->abd_u.abd_linear.abd_sgl;
698
699 /* When backed by user page unmap it */
700 if (abd_is_from_pages(abd))
701 zfs_kunmap(sg_page(sg));
702 else
703 abd_update_scatter_stats(abd, ABDSTAT_DECR);
704
705 abd->abd_flags &= ~ABD_FLAG_LINEAR;
706 abd->abd_flags &= ~ABD_FLAG_LINEAR_PAGE;
707 ABD_SCATTER(abd).abd_nents = 1;
708 ABD_SCATTER(abd).abd_offset = 0;
709 ABD_SCATTER(abd).abd_sgl = sg;
710 abd_free_chunks(abd);
711 }
712
713 /*
714 * Allocate a scatter ABD structure from user pages. The pages must be
715 * pinned with get_user_pages, or similiar, but need not be mapped via
716 * the kmap interfaces.
717 */
718 abd_t *
abd_alloc_from_pages(struct page ** pages,unsigned long offset,uint64_t size)719 abd_alloc_from_pages(struct page **pages, unsigned long offset, uint64_t size)
720 {
721 uint_t npages = DIV_ROUND_UP(size, PAGE_SIZE);
722 struct sg_table table;
723
724 VERIFY3U(size, <=, DMU_MAX_ACCESS);
725 ASSERT3U(offset, <, PAGE_SIZE);
726 ASSERT3P(pages, !=, NULL);
727
728 /*
729 * Even if this buf is filesystem metadata, we only track that we
730 * own the underlying data buffer, which is not true in this case.
731 * Therefore, we don't ever use ABD_FLAG_META here.
732 */
733 abd_t *abd = abd_alloc_struct(0);
734 abd->abd_flags |= ABD_FLAG_FROM_PAGES | ABD_FLAG_OWNER;
735 abd->abd_size = size;
736
737 while (sg_alloc_table_from_pages(&table, pages, npages, offset,
738 size, __GFP_NOWARN | GFP_NOIO) != 0) {
739 ABDSTAT_BUMP(abdstat_scatter_sg_table_retry);
740 schedule_timeout_interruptible(1);
741 }
742
743 if ((offset + size) <= PAGE_SIZE) {
744 /*
745 * Since there is only one entry, this ABD can be represented
746 * as a linear buffer. All single-page (4K) ABD's constructed
747 * from a user page can be represented this way as long as the
748 * page is mapped to a virtual address. This allows us to
749 * apply an offset in to the mapped page.
750 *
751 * Note that kmap() must be used, not kmap_atomic(), because
752 * the mapping needs to bet set up on all CPUs. Using kmap()
753 * also enables the user of highmem pages when required.
754 */
755 abd->abd_flags |= ABD_FLAG_LINEAR | ABD_FLAG_LINEAR_PAGE;
756 abd->abd_u.abd_linear.abd_sgl = table.sgl;
757 zfs_kmap(sg_page(table.sgl));
758 ABD_LINEAR_BUF(abd) = sg_virt(table.sgl);
759 } else {
760 ABDSTAT_BUMP(abdstat_scatter_page_multi_chunk);
761 abd->abd_flags |= ABD_FLAG_MULTI_CHUNK;
762
763 ABD_SCATTER(abd).abd_offset = offset;
764 ABD_SCATTER(abd).abd_sgl = table.sgl;
765 ABD_SCATTER(abd).abd_nents = table.nents;
766
767 ASSERT0(ABD_SCATTER(abd).abd_offset);
768 }
769
770 return (abd);
771 }
772
773 /*
774 * If we're going to use this ABD for doing I/O using the block layer, the
775 * consumer of the ABD data doesn't care if it's scattered or not, and we don't
776 * plan to store this ABD in memory for a long period of time, we should
777 * allocate the ABD type that requires the least data copying to do the I/O.
778 *
779 * On Linux the optimal thing to do would be to use abd_get_offset() and
780 * construct a new ABD which shares the original pages thereby eliminating
781 * the copy. But for the moment a new linear ABD is allocated until this
782 * performance optimization can be implemented.
783 */
784 abd_t *
abd_alloc_for_io(size_t size,boolean_t is_metadata)785 abd_alloc_for_io(size_t size, boolean_t is_metadata)
786 {
787 return (abd_alloc(size, is_metadata));
788 }
789
790 abd_t *
abd_get_offset_scatter(abd_t * abd,abd_t * sabd,size_t off,size_t size)791 abd_get_offset_scatter(abd_t *abd, abd_t *sabd, size_t off,
792 size_t size)
793 {
794 (void) size;
795 int i = 0;
796 struct scatterlist *sg = NULL;
797
798 abd_verify(sabd);
799 ASSERT3U(off, <=, sabd->abd_size);
800
801 size_t new_offset = ABD_SCATTER(sabd).abd_offset + off;
802
803 if (abd == NULL)
804 abd = abd_alloc_struct(0);
805
806 /*
807 * Even if this buf is filesystem metadata, we only track that
808 * if we own the underlying data buffer, which is not true in
809 * this case. Therefore, we don't ever use ABD_FLAG_META here.
810 */
811
812 abd_for_each_sg(sabd, sg, ABD_SCATTER(sabd).abd_nents, i) {
813 if (new_offset < sg->length)
814 break;
815 new_offset -= sg->length;
816 }
817
818 ABD_SCATTER(abd).abd_sgl = sg;
819 ABD_SCATTER(abd).abd_offset = new_offset;
820 ABD_SCATTER(abd).abd_nents = ABD_SCATTER(sabd).abd_nents - i;
821
822 if (abd_is_from_pages(sabd))
823 abd->abd_flags |= ABD_FLAG_FROM_PAGES;
824
825 return (abd);
826 }
827
828 /*
829 * Initialize the abd_iter.
830 */
831 void
abd_iter_init(struct abd_iter * aiter,abd_t * abd)832 abd_iter_init(struct abd_iter *aiter, abd_t *abd)
833 {
834 ASSERT(!abd_is_gang(abd));
835 abd_verify(abd);
836 memset(aiter, 0, sizeof (struct abd_iter));
837 aiter->iter_abd = abd;
838 if (!abd_is_linear(abd)) {
839 aiter->iter_offset = ABD_SCATTER(abd).abd_offset;
840 aiter->iter_sg = ABD_SCATTER(abd).abd_sgl;
841 }
842 }
843
844 /*
845 * This is just a helper function to see if we have exhausted the
846 * abd_iter and reached the end.
847 */
848 boolean_t
abd_iter_at_end(struct abd_iter * aiter)849 abd_iter_at_end(struct abd_iter *aiter)
850 {
851 ASSERT3U(aiter->iter_pos, <=, aiter->iter_abd->abd_size);
852 return (aiter->iter_pos == aiter->iter_abd->abd_size);
853 }
854
855 /*
856 * Advance the iterator by a certain amount. Cannot be called when a chunk is
857 * in use. This can be safely called when the aiter has already exhausted, in
858 * which case this does nothing.
859 */
860 void
abd_iter_advance(struct abd_iter * aiter,size_t amount)861 abd_iter_advance(struct abd_iter *aiter, size_t amount)
862 {
863 /*
864 * Ensure that last chunk is not in use. abd_iterate_*() must clear
865 * this state (directly or abd_iter_unmap()) before advancing.
866 */
867 ASSERT0P(aiter->iter_mapaddr);
868 ASSERT0(aiter->iter_mapsize);
869 ASSERT0P(aiter->iter_page);
870 ASSERT0(aiter->iter_page_doff);
871 ASSERT0(aiter->iter_page_dsize);
872
873 /* There's nothing left to advance to, so do nothing */
874 if (abd_iter_at_end(aiter))
875 return;
876
877 aiter->iter_pos += amount;
878 aiter->iter_offset += amount;
879 if (!abd_is_linear(aiter->iter_abd)) {
880 while (aiter->iter_offset >= aiter->iter_sg->length) {
881 aiter->iter_offset -= aiter->iter_sg->length;
882 aiter->iter_sg = sg_next(aiter->iter_sg);
883 if (aiter->iter_sg == NULL) {
884 ASSERT0(aiter->iter_offset);
885 break;
886 }
887 }
888 }
889 }
890
891 #ifndef nth_page
892 /*
893 * Since 6.18 nth_page() no longer exists, and is no longer required to iterate
894 * within a single SG entry, so we replace it with a simple addition.
895 */
896 #define nth_page(p, n) ((p)+(n))
897 #endif
898
899 /*
900 * Map the current chunk into aiter. This can be safely called when the aiter
901 * has already exhausted, in which case this does nothing.
902 */
903 void
abd_iter_map(struct abd_iter * aiter)904 abd_iter_map(struct abd_iter *aiter)
905 {
906 void *paddr;
907 size_t offset = 0;
908
909 ASSERT0P(aiter->iter_mapaddr);
910 ASSERT0(aiter->iter_mapsize);
911
912 /* There's nothing left to iterate over, so do nothing */
913 if (abd_iter_at_end(aiter))
914 return;
915
916 if (abd_is_linear(aiter->iter_abd)) {
917 ASSERT3U(aiter->iter_pos, ==, aiter->iter_offset);
918 offset = aiter->iter_offset;
919 aiter->iter_mapsize = aiter->iter_abd->abd_size - offset;
920 paddr = ABD_LINEAR_BUF(aiter->iter_abd);
921 } else {
922 offset = aiter->iter_offset;
923 aiter->iter_mapsize = MIN(aiter->iter_sg->length - offset,
924 aiter->iter_abd->abd_size - aiter->iter_pos);
925
926 struct page *page = sg_page(aiter->iter_sg);
927 if (PageHighMem(page)) {
928 page = nth_page(page, offset / PAGE_SIZE);
929 offset &= PAGE_SIZE - 1;
930 aiter->iter_mapsize = MIN(aiter->iter_mapsize,
931 PAGE_SIZE - offset);
932 }
933 paddr = zfs_kmap_local(page);
934 }
935
936 aiter->iter_mapaddr = (char *)paddr + offset;
937 }
938
939 /*
940 * Unmap the current chunk from aiter. This can be safely called when the aiter
941 * has already exhausted, in which case this does nothing.
942 */
943 void
abd_iter_unmap(struct abd_iter * aiter)944 abd_iter_unmap(struct abd_iter *aiter)
945 {
946 /* There's nothing left to unmap, so do nothing */
947 if (abd_iter_at_end(aiter))
948 return;
949
950 if (!abd_is_linear(aiter->iter_abd)) {
951 size_t offset = aiter->iter_offset;
952
953 struct page *page = sg_page(aiter->iter_sg);
954 if (PageHighMem(page))
955 offset &= PAGE_SIZE - 1;
956
957 /* LINTED E_FUNC_SET_NOT_USED */
958 zfs_kunmap_local(aiter->iter_mapaddr - offset);
959 }
960
961 ASSERT3P(aiter->iter_mapaddr, !=, NULL);
962 ASSERT3U(aiter->iter_mapsize, >, 0);
963
964 aiter->iter_mapaddr = NULL;
965 aiter->iter_mapsize = 0;
966 }
967
968 void
abd_cache_reap_now(void)969 abd_cache_reap_now(void)
970 {
971 }
972
973 /*
974 * Borrow a raw buffer from an ABD without copying the contents of the ABD
975 * into the buffer. If the ABD is scattered, this will allocate a raw buffer
976 * whose contents are undefined. To copy over the existing data in the ABD, use
977 * abd_borrow_buf_copy() instead.
978 */
979 void *
abd_borrow_buf(abd_t * abd,size_t n)980 abd_borrow_buf(abd_t *abd, size_t n)
981 {
982 void *buf;
983 abd_verify(abd);
984 ASSERT3U(abd->abd_size, >=, 0);
985 /*
986 * In the event the ABD is composed of a single user page from Direct
987 * I/O we can not direclty return the raw buffer. This is a consequence
988 * of not being able to write protect the page and the contents of the
989 * page can be changed at any time by the user.
990 */
991 if (abd_is_from_pages(abd)) {
992 buf = zio_buf_alloc(n);
993 } else if (abd_is_linear(abd)) {
994 buf = abd_to_buf(abd);
995 } else {
996 buf = zio_buf_alloc(n);
997 }
998
999 #ifdef ZFS_DEBUG
1000 (void) zfs_refcount_add_many(&abd->abd_children, n, buf);
1001 #endif
1002 return (buf);
1003 }
1004
1005 void *
abd_borrow_buf_copy(abd_t * abd,size_t n)1006 abd_borrow_buf_copy(abd_t *abd, size_t n)
1007 {
1008 void *buf = abd_borrow_buf(abd, n);
1009
1010 /*
1011 * In the event the ABD is composed of a single user page from Direct
1012 * I/O we must make sure copy the data over into the newly allocated
1013 * buffer. This is a consequence of the fact that we can not write
1014 * protect the user page and there is a risk the contents of the page
1015 * could be changed by the user at any moment.
1016 */
1017 if (!abd_is_linear(abd) || abd_is_from_pages(abd)) {
1018 abd_copy_to_buf(buf, abd, n);
1019 }
1020 return (buf);
1021 }
1022
1023 /*
1024 * Return a borrowed raw buffer to an ABD. If the ABD is scatterd, this will
1025 * not change the contents of the ABD. If you want any changes you made to
1026 * buf to be copied back to abd, use abd_return_buf_copy() instead. If the
1027 * ABD is not constructed from user pages for Direct I/O then an ASSERT
1028 * checks to make sure the contents of buffer have not changed since it was
1029 * borrowed. We can not ASSERT that the contents of the buffer have not changed
1030 * if it is composed of user pages because the pages can not be placed under
1031 * write protection and the user could have possibly changed the contents in
1032 * the pages at any time. This is also an issue for Direct I/O reads. Checksum
1033 * verifications in the ZIO pipeline check for this issue and handle it by
1034 * returning an error on checksum verification failure.
1035 */
1036 void
abd_return_buf(abd_t * abd,void * buf,size_t n)1037 abd_return_buf(abd_t *abd, void *buf, size_t n)
1038 {
1039 abd_verify(abd);
1040 ASSERT3U(abd->abd_size, >=, n);
1041 #ifdef ZFS_DEBUG
1042 (void) zfs_refcount_remove_many(&abd->abd_children, n, buf);
1043 #endif
1044 if (abd_is_from_pages(abd)) {
1045 zio_buf_free(buf, n);
1046 } else if (abd_is_linear(abd)) {
1047 ASSERT3P(buf, ==, abd_to_buf(abd));
1048 } else if (abd_is_gang(abd)) {
1049 #ifdef ZFS_DEBUG
1050 /*
1051 * We have to be careful with gang ABD's that we do not ASSERT0
1052 * for any ABD's that contain user pages from Direct I/O. In
1053 * order to handle this, we just iterate through the gang ABD
1054 * and only verify ABDs that are not from user pages.
1055 */
1056 void *cmp_buf = buf;
1057
1058 for (abd_t *cabd = list_head(&ABD_GANG(abd).abd_gang_chain);
1059 cabd != NULL;
1060 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
1061 if (!abd_is_from_pages(cabd)) {
1062 ASSERT0(abd_cmp_buf(cabd, cmp_buf,
1063 cabd->abd_size));
1064 }
1065 cmp_buf = (char *)cmp_buf + cabd->abd_size;
1066 }
1067 #endif
1068 zio_buf_free(buf, n);
1069 } else {
1070 ASSERT0(abd_cmp_buf(abd, buf, n));
1071 zio_buf_free(buf, n);
1072 }
1073 }
1074
1075 void
abd_return_buf_copy(abd_t * abd,void * buf,size_t n)1076 abd_return_buf_copy(abd_t *abd, void *buf, size_t n)
1077 {
1078 if (!abd_is_linear(abd) || abd_is_from_pages(abd)) {
1079 abd_copy_from_buf(abd, buf, n);
1080 }
1081 abd_return_buf(abd, buf, n);
1082 }
1083
1084 /*
1085 * This is abd_iter_page(), the function underneath abd_iterate_page_func().
1086 * It yields the next page struct and data offset and size within it, without
1087 * mapping it into the address space.
1088 */
1089
1090 /*
1091 * "Compound pages" are a group of pages that can be referenced from a single
1092 * struct page *. Its organised as a "head" page, followed by a series of
1093 * "tail" pages.
1094 *
1095 * In OpenZFS, compound pages are allocated using the __GFP_COMP flag, which we
1096 * get from scatter ABDs and SPL vmalloc slabs (ie >16K allocations). So a
1097 * great many of the IO buffers we get are going to be of this type.
1098 *
1099 * The tail pages are just regular PAGESIZE pages, and can be safely used
1100 * as-is. However, the head page has length covering itself and all the tail
1101 * pages. If the ABD chunk spans multiple pages, then we can use the head page
1102 * and a >PAGESIZE length, which is far more efficient.
1103 *
1104 * Before kernel 4.5 however, compound page heads were refcounted separately
1105 * from tail pages, such that moving back to the head page would require us to
1106 * take a reference to it and releasing it once we're completely finished with
1107 * it. In practice, that meant when our caller is done with the ABD, which we
1108 * have no insight into from here. Rather than contort this API to track head
1109 * page references on such ancient kernels, we disabled this special compound
1110 * page handling on kernels before 4.5, instead just using treating each page
1111 * within it as a regular PAGESIZE page (which it is). This is slightly less
1112 * efficient, but makes everything far simpler.
1113 *
1114 * We no longer support kernels before 4.5, so in theory none of this is
1115 * necessary. However, this code is still relatively new in the grand scheme of
1116 * things, so I'm leaving the ability to compile this out for the moment.
1117 *
1118 * Setting/clearing ABD_ITER_COMPOUND_PAGES below enables/disables the special
1119 * handling, by defining the ABD_ITER_PAGE_SIZE(page) macro to understand
1120 * compound pages, or not, and compiling in/out the support to detect compound
1121 * tail pages and move back to the start.
1122 */
1123
1124 /* On by default */
1125 #define ABD_ITER_COMPOUND_PAGES
1126
1127 #ifdef ABD_ITER_COMPOUND_PAGES
1128 #define ABD_ITER_PAGE_SIZE(page) \
1129 (PageCompound(page) ? page_size(page) : PAGESIZE)
1130 #else
1131 #define ABD_ITER_PAGE_SIZE(page) (PAGESIZE)
1132 #endif
1133
1134 void
abd_iter_page(struct abd_iter * aiter)1135 abd_iter_page(struct abd_iter *aiter)
1136 {
1137 if (abd_iter_at_end(aiter)) {
1138 aiter->iter_page = NULL;
1139 aiter->iter_page_doff = 0;
1140 aiter->iter_page_dsize = 0;
1141 return;
1142 }
1143
1144 struct page *page;
1145 size_t doff, dsize;
1146
1147 /*
1148 * Find the page, and the start of the data within it. This is computed
1149 * differently for linear and scatter ABDs; linear is referenced by
1150 * virtual memory location, while scatter is referenced by page
1151 * pointer.
1152 */
1153 if (abd_is_linear(aiter->iter_abd)) {
1154 ASSERT3U(aiter->iter_pos, ==, aiter->iter_offset);
1155
1156 /* memory address at iter_pos */
1157 void *paddr = ABD_LINEAR_BUF(aiter->iter_abd) + aiter->iter_pos;
1158
1159 /* struct page for address */
1160 page = is_vmalloc_addr(paddr) ?
1161 vmalloc_to_page(paddr) : virt_to_page(paddr);
1162
1163 /* offset of address within the page */
1164 doff = offset_in_page(paddr);
1165 } else {
1166 ASSERT(!abd_is_gang(aiter->iter_abd));
1167
1168 /* current scatter page */
1169 page = nth_page(sg_page(aiter->iter_sg),
1170 aiter->iter_offset >> PAGE_SHIFT);
1171
1172 /* position within page */
1173 doff = aiter->iter_offset & (PAGESIZE - 1);
1174 }
1175
1176 #ifdef ABD_ITER_COMPOUND_PAGES
1177 if (PageTail(page)) {
1178 /*
1179 * If this is a compound tail page, move back to the head, and
1180 * adjust the offset to match. This may let us yield a much
1181 * larger amount of data from a single logical page, and so
1182 * leave our caller with fewer pages to process.
1183 */
1184 struct page *head = compound_head(page);
1185 doff += ((page - head) * PAGESIZE);
1186 page = head;
1187 }
1188 #endif
1189
1190 ASSERT(page);
1191
1192 /*
1193 * Compute the maximum amount of data we can take from this page. This
1194 * is the smaller of:
1195 * - the remaining space in the page
1196 * - the remaining space in this scatterlist entry (which may not cover
1197 * the entire page)
1198 * - the remaining space in the abd (which may not cover the entire
1199 * scatterlist entry)
1200 */
1201 dsize = MIN(ABD_ITER_PAGE_SIZE(page) - doff,
1202 aiter->iter_abd->abd_size - aiter->iter_pos);
1203 if (!abd_is_linear(aiter->iter_abd))
1204 dsize = MIN(dsize, aiter->iter_sg->length - aiter->iter_offset);
1205 ASSERT3U(dsize, >, 0);
1206
1207 /* final iterator outputs */
1208 aiter->iter_page = page;
1209 aiter->iter_page_doff = doff;
1210 aiter->iter_page_dsize = dsize;
1211 }
1212
1213 /*
1214 * Note: ABD BIO functions only needed to support vdev_classic. See comments in
1215 * vdev_disk.c.
1216 */
1217
1218 /*
1219 * bio_nr_pages for ABD.
1220 * @off is the offset in @abd
1221 */
1222 unsigned long
abd_nr_pages_off(abd_t * abd,unsigned int size,size_t off)1223 abd_nr_pages_off(abd_t *abd, unsigned int size, size_t off)
1224 {
1225 unsigned long pos;
1226
1227 if (abd_is_gang(abd)) {
1228 unsigned long count = 0;
1229
1230 for (abd_t *cabd = abd_gang_get_offset(abd, &off);
1231 cabd != NULL && size != 0;
1232 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
1233 ASSERT3U(off, <, cabd->abd_size);
1234 int mysize = MIN(size, cabd->abd_size - off);
1235 count += abd_nr_pages_off(cabd, mysize, off);
1236 size -= mysize;
1237 off = 0;
1238 }
1239 return (count);
1240 }
1241
1242 if (abd_is_linear(abd))
1243 pos = (unsigned long)abd_to_buf(abd) + off;
1244 else
1245 pos = ABD_SCATTER(abd).abd_offset + off;
1246
1247 return (((pos + size + PAGESIZE - 1) >> PAGE_SHIFT) -
1248 (pos >> PAGE_SHIFT));
1249 }
1250
1251 static unsigned int
bio_map(struct bio * bio,void * buf_ptr,unsigned int bio_size)1252 bio_map(struct bio *bio, void *buf_ptr, unsigned int bio_size)
1253 {
1254 unsigned int offset, size, i;
1255 struct page *page;
1256
1257 offset = offset_in_page(buf_ptr);
1258 for (i = 0; i < bio->bi_max_vecs; i++) {
1259 size = PAGE_SIZE - offset;
1260
1261 if (bio_size <= 0)
1262 break;
1263
1264 if (size > bio_size)
1265 size = bio_size;
1266
1267 if (is_vmalloc_addr(buf_ptr))
1268 page = vmalloc_to_page(buf_ptr);
1269 else
1270 page = virt_to_page(buf_ptr);
1271
1272 /*
1273 * Some network related block device uses tcp_sendpage, which
1274 * doesn't behave well when using 0-count page, this is a
1275 * safety net to catch them.
1276 */
1277 ASSERT3S(page_count(page), >, 0);
1278
1279 if (bio_add_page(bio, page, size, offset) != size)
1280 break;
1281
1282 buf_ptr += size;
1283 bio_size -= size;
1284 offset = 0;
1285 }
1286
1287 return (bio_size);
1288 }
1289
1290 /*
1291 * bio_map for gang ABD.
1292 */
1293 static unsigned int
abd_gang_bio_map_off(struct bio * bio,abd_t * abd,unsigned int io_size,size_t off)1294 abd_gang_bio_map_off(struct bio *bio, abd_t *abd,
1295 unsigned int io_size, size_t off)
1296 {
1297 ASSERT(abd_is_gang(abd));
1298
1299 for (abd_t *cabd = abd_gang_get_offset(abd, &off);
1300 cabd != NULL;
1301 cabd = list_next(&ABD_GANG(abd).abd_gang_chain, cabd)) {
1302 ASSERT3U(off, <, cabd->abd_size);
1303 int size = MIN(io_size, cabd->abd_size - off);
1304 int remainder = abd_bio_map_off(bio, cabd, size, off);
1305 io_size -= (size - remainder);
1306 if (io_size == 0 || remainder > 0)
1307 return (io_size);
1308 off = 0;
1309 }
1310 ASSERT0(io_size);
1311 return (io_size);
1312 }
1313
1314 /*
1315 * bio_map for ABD.
1316 * @off is the offset in @abd
1317 * Remaining IO size is returned
1318 */
1319 unsigned int
abd_bio_map_off(struct bio * bio,abd_t * abd,unsigned int io_size,size_t off)1320 abd_bio_map_off(struct bio *bio, abd_t *abd,
1321 unsigned int io_size, size_t off)
1322 {
1323 struct abd_iter aiter;
1324
1325 ASSERT3U(io_size, <=, abd->abd_size - off);
1326 if (abd_is_linear(abd))
1327 return (bio_map(bio, ((char *)abd_to_buf(abd)) + off, io_size));
1328
1329 ASSERT(!abd_is_linear(abd));
1330 if (abd_is_gang(abd))
1331 return (abd_gang_bio_map_off(bio, abd, io_size, off));
1332
1333 abd_iter_init(&aiter, abd);
1334 abd_iter_advance(&aiter, off);
1335
1336 for (int i = 0; i < bio->bi_max_vecs; i++) {
1337 struct page *pg;
1338 size_t len, sgoff, pgoff;
1339 struct scatterlist *sg;
1340
1341 if (io_size <= 0)
1342 break;
1343
1344 sg = aiter.iter_sg;
1345 sgoff = aiter.iter_offset;
1346 pgoff = sgoff & (PAGESIZE - 1);
1347 len = MIN(io_size, PAGESIZE - pgoff);
1348 ASSERT(len > 0);
1349
1350 pg = nth_page(sg_page(sg), sgoff >> PAGE_SHIFT);
1351 if (bio_add_page(bio, pg, len, pgoff) != len)
1352 break;
1353
1354 io_size -= len;
1355 abd_iter_advance(&aiter, len);
1356 }
1357
1358 return (io_size);
1359 }
1360
1361 EXPORT_SYMBOL(abd_alloc_from_pages);
1362
1363 /* Tunable Parameters */
1364 module_param(zfs_abd_scatter_enabled, int, 0644);
1365 MODULE_PARM_DESC(zfs_abd_scatter_enabled,
1366 "Toggle whether ABD allocations must be linear.");
1367 module_param(zfs_abd_scatter_min_size, int, 0644);
1368 MODULE_PARM_DESC(zfs_abd_scatter_min_size,
1369 "Minimum size of scatter allocations.");
1370 module_param(zfs_abd_scatter_max_order, uint, 0644);
1371 MODULE_PARM_DESC(zfs_abd_scatter_max_order,
1372 "Maximum order allocation used for a scatter ABD.");
1373