1 // SPDX-License-Identifier: GPL-2.0
2 /*
3 * Functions related to generic helpers functions
4 */
5 #include <linux/kernel.h>
6 #include <linux/module.h>
7 #include <linux/bio.h>
8 #include <linux/blkdev.h>
9 #include <linux/scatterlist.h>
10
11 #include "blk.h"
12
bio_discard_limit(struct block_device * bdev,sector_t sector)13 static sector_t bio_discard_limit(struct block_device *bdev, sector_t sector)
14 {
15 unsigned int discard_granularity = bdev_discard_granularity(bdev);
16 sector_t granularity_aligned_sector;
17
18 if (bdev_is_partition(bdev))
19 sector += bdev->bd_start_sect;
20
21 granularity_aligned_sector =
22 round_up(sector, discard_granularity >> SECTOR_SHIFT);
23
24 /*
25 * Make sure subsequent bios start aligned to the discard granularity if
26 * it needs to be split.
27 */
28 if (granularity_aligned_sector != sector)
29 return granularity_aligned_sector - sector;
30
31 /*
32 * Align the bio size to the discard granularity to make splitting the bio
33 * at discard granularity boundaries easier in the driver if needed.
34 */
35 return round_down(BIO_MAX_SIZE, discard_granularity) >> SECTOR_SHIFT;
36 }
37
blk_alloc_discard_bio(struct block_device * bdev,sector_t * sector,sector_t * nr_sects,gfp_t gfp_mask)38 struct bio *blk_alloc_discard_bio(struct block_device *bdev,
39 sector_t *sector, sector_t *nr_sects, gfp_t gfp_mask)
40 {
41 sector_t bio_sects = min(*nr_sects, bio_discard_limit(bdev, *sector));
42 struct bio *bio;
43
44 if (!bio_sects)
45 return NULL;
46
47 bio = bio_alloc(bdev, 0, REQ_OP_DISCARD, gfp_mask);
48 if (!bio)
49 return NULL;
50 bio->bi_iter.bi_sector = *sector;
51 bio->bi_iter.bi_size = bio_sects << SECTOR_SHIFT;
52 *sector += bio_sects;
53 *nr_sects -= bio_sects;
54 /*
55 * We can loop for a long time in here if someone does full device
56 * discards (like mkfs). Be nice and allow us to schedule out to avoid
57 * softlocking if preempt is disabled.
58 */
59 cond_resched();
60 return bio;
61 }
62
__blkdev_issue_discard(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop)63 void __blkdev_issue_discard(struct block_device *bdev, sector_t sector,
64 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop)
65 {
66 struct bio *bio;
67
68 while ((bio = blk_alloc_discard_bio(bdev, §or, &nr_sects,
69 gfp_mask)))
70 *biop = bio_chain_and_submit(*biop, bio);
71 }
72 EXPORT_SYMBOL(__blkdev_issue_discard);
73
74 /**
75 * blkdev_issue_discard - queue a discard
76 * @bdev: blockdev to issue discard for
77 * @sector: start sector
78 * @nr_sects: number of sectors to discard
79 * @gfp_mask: memory allocation flags (for bio_alloc)
80 *
81 * Description:
82 * Issue a discard request for the sectors in question.
83 */
blkdev_issue_discard(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask)84 int blkdev_issue_discard(struct block_device *bdev, sector_t sector,
85 sector_t nr_sects, gfp_t gfp_mask)
86 {
87 struct bio *bio = NULL;
88 struct blk_plug plug;
89 int ret = 0;
90
91 blk_start_plug(&plug);
92 __blkdev_issue_discard(bdev, sector, nr_sects, gfp_mask, &bio);
93 if (bio) {
94 ret = submit_bio_wait(bio);
95 if (ret == -EOPNOTSUPP)
96 ret = 0;
97 bio_put(bio);
98 }
99 blk_finish_plug(&plug);
100
101 return ret;
102 }
103 EXPORT_SYMBOL(blkdev_issue_discard);
104
bio_write_zeroes_limit(struct block_device * bdev)105 static sector_t bio_write_zeroes_limit(struct block_device *bdev)
106 {
107 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
108
109 return min(bdev_write_zeroes_sectors(bdev), BIO_MAX_SECTORS & ~bs_mask);
110 }
111
112 /*
113 * There is no reliable way for the SCSI subsystem to determine whether a
114 * device supports a WRITE SAME operation without actually performing a write
115 * to media. As a result, write_zeroes is enabled by default and will be
116 * disabled if a zeroing operation subsequently fails. This means that this
117 * queue limit is likely to change at runtime.
118 */
__blkdev_issue_write_zeroes(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop,unsigned flags,sector_t limit)119 static void __blkdev_issue_write_zeroes(struct block_device *bdev,
120 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
121 struct bio **biop, unsigned flags, sector_t limit)
122 {
123
124 while (nr_sects) {
125 unsigned int len = min(nr_sects, limit);
126 struct bio *bio;
127
128 if ((flags & BLKDEV_ZERO_KILLABLE) &&
129 fatal_signal_pending(current))
130 break;
131
132 bio = bio_alloc(bdev, 0, REQ_OP_WRITE_ZEROES, gfp_mask);
133 bio->bi_iter.bi_sector = sector;
134 if (flags & BLKDEV_ZERO_NOUNMAP)
135 bio->bi_opf |= REQ_NOUNMAP;
136
137 bio->bi_iter.bi_size = len << SECTOR_SHIFT;
138 *biop = bio_chain_and_submit(*biop, bio);
139
140 nr_sects -= len;
141 sector += len;
142 cond_resched();
143 }
144 }
145
blkdev_issue_write_zeroes(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp,unsigned flags)146 static int blkdev_issue_write_zeroes(struct block_device *bdev, sector_t sector,
147 sector_t nr_sects, gfp_t gfp, unsigned flags)
148 {
149 sector_t limit = bio_write_zeroes_limit(bdev);
150 struct bio *bio = NULL;
151 struct blk_plug plug;
152 int ret = 0;
153
154 blk_start_plug(&plug);
155 __blkdev_issue_write_zeroes(bdev, sector, nr_sects, gfp, &bio,
156 flags, limit);
157 if (bio) {
158 ret = bio_submit_or_kill(bio, flags);
159 bio_put(bio);
160 }
161 blk_finish_plug(&plug);
162
163 /*
164 * For some devices there is no non-destructive way to verify whether
165 * WRITE ZEROES is actually supported. These will clear the capability
166 * on an I/O error, in which case we'll turn any error into
167 * "not supported" here.
168 */
169 if (ret && !bdev_write_zeroes_sectors(bdev))
170 return -EOPNOTSUPP;
171 return ret;
172 }
173
174 /*
175 * Convert a number of 512B sectors to a number of pages.
176 * The result is limited to a number of pages that can fit into a BIO.
177 * Also make sure that the result is always at least 1 (page) for the cases
178 * where nr_sects is lower than the number of sectors in a page.
179 */
__blkdev_sectors_to_bio_pages(sector_t nr_sects)180 static unsigned int __blkdev_sectors_to_bio_pages(sector_t nr_sects)
181 {
182 sector_t pages = DIV_ROUND_UP_SECTOR_T(nr_sects, PAGE_SIZE / 512);
183
184 return min(pages, (sector_t)BIO_MAX_VECS);
185 }
186
__blkdev_issue_zero_pages(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop,unsigned int flags)187 static void __blkdev_issue_zero_pages(struct block_device *bdev,
188 sector_t sector, sector_t nr_sects, gfp_t gfp_mask,
189 struct bio **biop, unsigned int flags)
190 {
191 struct folio *zero_folio = largest_zero_folio();
192
193 while (nr_sects) {
194 unsigned int nr_vecs = __blkdev_sectors_to_bio_pages(nr_sects);
195 struct bio *bio;
196
197 if ((flags & BLKDEV_ZERO_KILLABLE) &&
198 fatal_signal_pending(current))
199 break;
200
201 bio = bio_alloc(bdev, nr_vecs, REQ_OP_WRITE, gfp_mask);
202 bio->bi_iter.bi_sector = sector;
203
204 do {
205 unsigned int len;
206
207 len = min_t(sector_t, folio_size(zero_folio),
208 nr_sects << SECTOR_SHIFT);
209 if (!bio_add_folio(bio, zero_folio, len, 0))
210 break;
211 nr_sects -= len >> SECTOR_SHIFT;
212 sector += len >> SECTOR_SHIFT;
213 } while (nr_sects);
214
215 *biop = bio_chain_and_submit(*biop, bio);
216 cond_resched();
217 }
218 }
219
blkdev_issue_zero_pages(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp,unsigned flags)220 static int blkdev_issue_zero_pages(struct block_device *bdev, sector_t sector,
221 sector_t nr_sects, gfp_t gfp, unsigned flags)
222 {
223 struct bio *bio = NULL;
224 struct blk_plug plug;
225 int ret = 0;
226
227 if (flags & BLKDEV_ZERO_NOFALLBACK)
228 return -EOPNOTSUPP;
229
230 blk_start_plug(&plug);
231 __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp, &bio, flags);
232 if (bio) {
233 ret = bio_submit_or_kill(bio, flags);
234 bio_put(bio);
235 }
236 blk_finish_plug(&plug);
237
238 return ret;
239 }
240
241 /**
242 * __blkdev_issue_zeroout - generate number of zero filed write bios
243 * @bdev: blockdev to issue
244 * @sector: start sector
245 * @nr_sects: number of sectors to write
246 * @gfp_mask: memory allocation flags (for bio_alloc)
247 * @biop: pointer to anchor bio
248 * @flags: controls detailed behavior
249 *
250 * Description:
251 * Zero-fill a block range, either using hardware offload or by explicitly
252 * writing zeroes to the device.
253 *
254 * If a device is using logical block provisioning, the underlying space will
255 * not be released if %flags contains BLKDEV_ZERO_NOUNMAP.
256 *
257 * If %flags contains BLKDEV_ZERO_NOFALLBACK, the function will return
258 * -EOPNOTSUPP if no explicit hardware offload for zeroing is provided.
259 */
__blkdev_issue_zeroout(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,struct bio ** biop,unsigned flags)260 int __blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
261 sector_t nr_sects, gfp_t gfp_mask, struct bio **biop,
262 unsigned flags)
263 {
264 sector_t limit = bio_write_zeroes_limit(bdev);
265
266 if (bdev_read_only(bdev))
267 return -EPERM;
268
269 if (limit) {
270 __blkdev_issue_write_zeroes(bdev, sector, nr_sects,
271 gfp_mask, biop, flags, limit);
272 } else {
273 if (flags & BLKDEV_ZERO_NOFALLBACK)
274 return -EOPNOTSUPP;
275 __blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask,
276 biop, flags);
277 }
278 return 0;
279 }
280 EXPORT_SYMBOL(__blkdev_issue_zeroout);
281
282 /**
283 * blkdev_issue_zeroout - zero-fill a block range
284 * @bdev: blockdev to write
285 * @sector: start sector
286 * @nr_sects: number of sectors to write
287 * @gfp_mask: memory allocation flags (for bio_alloc)
288 * @flags: controls detailed behavior
289 *
290 * Description:
291 * Zero-fill a block range, either using hardware offload or by explicitly
292 * writing zeroes to the device. See __blkdev_issue_zeroout() for the
293 * valid values for %flags.
294 */
blkdev_issue_zeroout(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp_mask,unsigned flags)295 int blkdev_issue_zeroout(struct block_device *bdev, sector_t sector,
296 sector_t nr_sects, gfp_t gfp_mask, unsigned flags)
297 {
298 int ret;
299
300 if ((sector | nr_sects) & ((bdev_logical_block_size(bdev) >> 9) - 1))
301 return -EINVAL;
302 if (bdev_read_only(bdev))
303 return -EPERM;
304
305 if (bdev_write_zeroes_sectors(bdev)) {
306 ret = blkdev_issue_write_zeroes(bdev, sector, nr_sects,
307 gfp_mask, flags);
308 if (ret != -EOPNOTSUPP)
309 return ret;
310 }
311
312 return blkdev_issue_zero_pages(bdev, sector, nr_sects, gfp_mask, flags);
313 }
314 EXPORT_SYMBOL(blkdev_issue_zeroout);
315
blkdev_issue_secure_erase(struct block_device * bdev,sector_t sector,sector_t nr_sects,gfp_t gfp)316 int blkdev_issue_secure_erase(struct block_device *bdev, sector_t sector,
317 sector_t nr_sects, gfp_t gfp)
318 {
319 sector_t bs_mask = (bdev_logical_block_size(bdev) >> 9) - 1;
320 unsigned int max_sectors = bdev_max_secure_erase_sectors(bdev);
321 struct bio *bio = NULL;
322 struct blk_plug plug;
323 int ret = 0;
324
325 /* make sure that "len << SECTOR_SHIFT" doesn't overflow */
326 if (max_sectors > BIO_MAX_SECTORS)
327 max_sectors = BIO_MAX_SECTORS;
328 max_sectors &= ~bs_mask;
329
330 if (max_sectors == 0)
331 return -EOPNOTSUPP;
332 if ((sector | nr_sects) & bs_mask)
333 return -EINVAL;
334 if (bdev_read_only(bdev))
335 return -EPERM;
336
337 blk_start_plug(&plug);
338 while (nr_sects) {
339 unsigned int len = min_t(sector_t, nr_sects, max_sectors);
340
341 bio = blk_next_bio(bio, bdev, 0, REQ_OP_SECURE_ERASE, gfp);
342 bio->bi_iter.bi_sector = sector;
343 bio->bi_iter.bi_size = len << SECTOR_SHIFT;
344
345 sector += len;
346 nr_sects -= len;
347 cond_resched();
348 }
349 if (bio) {
350 ret = submit_bio_wait(bio);
351 bio_put(bio);
352 }
353 blk_finish_plug(&plug);
354
355 return ret;
356 }
357 EXPORT_SYMBOL(blkdev_issue_secure_erase);
358