1 // SPDX-License-Identifier: CDDL-1.0
2 /*
3 * CDDL HEADER START
4 *
5 * The contents of this file are subject to the terms of the
6 * Common Development and Distribution License (the "License").
7 * You may not use this file except in compliance with the License.
8 *
9 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
10 * or https://opensource.org/licenses/CDDL-1.0.
11 * See the License for the specific language governing permissions
12 * and limitations under the License.
13 *
14 * When distributing Covered Code, include this CDDL HEADER in each
15 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
16 * If applicable, add the following below this CDDL HEADER, with the
17 * fields enclosed by brackets "[]" replaced with your own identifying
18 * information: Portions Copyright [yyyy] [name of copyright owner]
19 *
20 * CDDL HEADER END
21 */
22
23 /*
24 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
25 * Copyright (c) 2012, 2018 by Delphix. All rights reserved.
26 * Copyright (c) 2015 by Chunwei Chen. All rights reserved.
27 * Copyright 2017 Nexenta Systems, Inc.
28 * Copyright (c) 2021, 2022 by Pawel Jakub Dawidek
29 * Copyright (c) 2025, Rob Norris <robn@despairlabs.com>
30 * Copyright (c) 2025, Klara, Inc.
31 */
32
33 /* Portions Copyright 2007 Jeremy Teo */
34 /* Portions Copyright 2010 Robert Milkowski */
35
36 #include <sys/types.h>
37 #include <sys/param.h>
38 #include <sys/time.h>
39 #include <sys/sysmacros.h>
40 #include <sys/vfs.h>
41 #include <sys/file.h>
42 #include <sys/stat.h>
43 #include <sys/kmem.h>
44 #include <sys/cmn_err.h>
45 #include <sys/errno.h>
46 #include <sys/zfs_dir.h>
47 #include <sys/zfs_acl.h>
48 #include <sys/zfs_ioctl.h>
49 #include <sys/fs/zfs.h>
50 #include <sys/dmu.h>
51 #include <sys/dmu_objset.h>
52 #include <sys/dsl_crypt.h>
53 #include <sys/dsl_dataset.h>
54 #include <sys/spa.h>
55 #include <sys/txg.h>
56 #include <sys/brt.h>
57 #include <sys/dbuf.h>
58 #include <sys/policy.h>
59 #include <sys/zfeature.h>
60 #include <sys/zfs_vnops.h>
61 #include <sys/zfs_quota.h>
62 #include <sys/zfs_vfsops.h>
63 #include <sys/zfs_znode.h>
64
65 /*
66 * Enables access to the block cloning feature. If this setting is 0, then even
67 * if feature@block_cloning is enabled, using functions and system calls that
68 * attempt to clone blocks will act as though the feature is disabled.
69 */
70 int zfs_bclone_enabled = 1;
71
72 /*
73 * Restricts block cloning between datasets with different properties
74 * (checksum, compression, copies, dedup, or special_small_blocks).
75 */
76 int zfs_bclone_strict_properties = 1;
77
78 /*
79 * When set to 1 the FICLONE and FICLONERANGE ioctls will wait for any dirty
80 * data to be written to disk before proceeding. This ensures that the clone
81 * operation reliably succeeds, even if a file is modified and then immediately
82 * cloned. Note that for small files this may be slower than simply copying
83 * the file. When set to 0 the clone operation will immediately fail if it
84 * encounters any dirty blocks. By default waiting is enabled.
85 */
86 int zfs_bclone_wait_dirty = 1;
87
88 /*
89 * Enable Direct I/O. If this setting is 0, then all I/O requests will be
90 * directed through the ARC acting as though the dataset property direct was
91 * set to disabled.
92 *
93 * Disabled by default on FreeBSD until a potential range locking issue in
94 * zfs_getpages() can be resolved.
95 */
96 #ifdef __FreeBSD__
97 static int zfs_dio_enabled = 0;
98 #else
99 static int zfs_dio_enabled = 1;
100 #endif
101
102 /*
103 * Strictly enforce alignment for Direct I/O requests, returning EINVAL
104 * if not page-aligned instead of silently falling back to uncached I/O.
105 */
106 static int zfs_dio_strict = 0;
107
108
109 /*
110 * Maximum bytes to read per chunk in zfs_read().
111 */
112 #ifdef _ILP32
113 static uint64_t zfs_vnops_read_chunk_size = 1024 * 1024;
114 #else
115 static uint64_t zfs_vnops_read_chunk_size = DMU_MAX_ACCESS / 2;
116 #endif
117
118 int
zfs_fsync(znode_t * zp,int syncflag,cred_t * cr)119 zfs_fsync(znode_t *zp, int syncflag, cred_t *cr)
120 {
121 int error = 0;
122 zfsvfs_t *zfsvfs = ZTOZSB(zp);
123
124 if (zfsvfs->z_os->os_sync != ZFS_SYNC_DISABLED) {
125 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
126 return (error);
127 error = zil_commit(zfsvfs->z_log, zp->z_id);
128 zfs_exit(zfsvfs, FTAG);
129 }
130 return (error);
131 }
132
133
134 #if defined(SEEK_HOLE) && defined(SEEK_DATA)
135 /*
136 * Lseek support for finding holes (cmd == SEEK_HOLE) and
137 * data (cmd == SEEK_DATA). "off" is an in/out parameter.
138 */
139 static int
zfs_holey_common(znode_t * zp,ulong_t cmd,loff_t * off)140 zfs_holey_common(znode_t *zp, ulong_t cmd, loff_t *off)
141 {
142 zfs_locked_range_t *lr;
143 uint64_t noff = (uint64_t)*off; /* new offset */
144 uint64_t file_sz;
145 int error;
146 boolean_t hole;
147
148 file_sz = zp->z_size;
149 if (noff >= file_sz) {
150 return (SET_ERROR(ENXIO));
151 }
152
153 if (cmd == F_SEEK_HOLE)
154 hole = B_TRUE;
155 else
156 hole = B_FALSE;
157
158 /* Flush any mmap()'d data to disk */
159 if (zn_has_cached_data(zp, 0, file_sz - 1))
160 zn_flush_cached_data(zp, B_TRUE);
161
162 lr = zfs_rangelock_enter(&zp->z_rangelock, 0, UINT64_MAX, RL_READER);
163 error = dmu_offset_next(ZTOZSB(zp)->z_os, zp->z_id, hole, &noff);
164 zfs_rangelock_exit(lr);
165
166 if (error == ESRCH)
167 return (SET_ERROR(ENXIO));
168
169 /* File was dirty, so fall back to using generic logic */
170 if (error == EBUSY) {
171 if (hole)
172 *off = file_sz;
173
174 return (0);
175 }
176
177 /*
178 * We could find a hole that begins after the logical end-of-file,
179 * because dmu_offset_next() only works on whole blocks. If the
180 * EOF falls mid-block, then indicate that the "virtual hole"
181 * at the end of the file begins at the logical EOF, rather than
182 * at the end of the last block.
183 */
184 if (noff > file_sz) {
185 ASSERT(hole);
186 noff = file_sz;
187 }
188
189 if (noff < *off)
190 return (error);
191 *off = noff;
192 return (error);
193 }
194
195 int
zfs_holey(znode_t * zp,ulong_t cmd,loff_t * off)196 zfs_holey(znode_t *zp, ulong_t cmd, loff_t *off)
197 {
198 zfsvfs_t *zfsvfs = ZTOZSB(zp);
199 int error;
200
201 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
202 return (error);
203
204 error = zfs_holey_common(zp, cmd, off);
205
206 zfs_exit(zfsvfs, FTAG);
207 return (error);
208 }
209 #endif /* SEEK_HOLE && SEEK_DATA */
210
211 int
zfs_access(znode_t * zp,int mode,int flag,cred_t * cr)212 zfs_access(znode_t *zp, int mode, int flag, cred_t *cr)
213 {
214 zfsvfs_t *zfsvfs = ZTOZSB(zp);
215 int error;
216
217 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
218 return (error);
219
220 if (flag & V_ACE_MASK)
221 #if defined(__linux__)
222 error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
223 zfs_init_idmap);
224 #else
225 error = zfs_zaccess(zp, mode, flag, B_FALSE, cr,
226 NULL);
227 #endif
228 else
229 #if defined(__linux__)
230 error = zfs_zaccess_rwx(zp, mode, flag, cr, zfs_init_idmap);
231 #else
232 error = zfs_zaccess_rwx(zp, mode, flag, cr, NULL);
233 #endif
234
235 zfs_exit(zfsvfs, FTAG);
236 return (error);
237 }
238
239 /*
240 * Determine if Direct I/O has been requested (either via the O_DIRECT flag or
241 * the "direct" dataset property). When inherited by the property only apply
242 * the O_DIRECT flag to correctly aligned IO requests. The rational for this
243 * is it allows the property to be safely set on a dataset without forcing
244 * all of the applications to be aware of the alignment restrictions. When
245 * O_DIRECT is explicitly requested by an application return EINVAL if the
246 * request is unaligned. In all cases, if the range for this request has
247 * been mmap'ed then we will perform buffered I/O to keep the mapped region
248 * synhronized with the ARC.
249 *
250 * It is possible that a file's pages could be mmap'ed after it is checked
251 * here. If so, that is handled coorarding in zfs_write(). See comments in the
252 * following area for how this is handled:
253 * zfs_write() -> update_pages()
254 */
255 static int
zfs_setup_direct(struct znode * zp,zfs_uio_t * uio,zfs_uio_rw_t rw,int * ioflagp)256 zfs_setup_direct(struct znode *zp, zfs_uio_t *uio, zfs_uio_rw_t rw,
257 int *ioflagp)
258 {
259 zfsvfs_t *zfsvfs = ZTOZSB(zp);
260 objset_t *os = zfsvfs->z_os;
261 int ioflag = *ioflagp;
262 int error = 0;
263
264 if (os->os_direct == ZFS_DIRECT_ALWAYS) {
265 /* Force either direct or uncached I/O. */
266 ioflag |= O_DIRECT;
267 }
268
269 if ((ioflag & O_DIRECT) == 0)
270 goto out;
271
272 if (!zfs_dio_enabled || os->os_direct == ZFS_DIRECT_DISABLED) {
273 /*
274 * Direct I/O is disabled. The I/O request will be directed
275 * through the ARC as uncached I/O.
276 */
277 goto out;
278 }
279
280 if (!zfs_uio_page_aligned(uio) ||
281 !zfs_uio_aligned(uio, PAGE_SIZE)) {
282 /*
283 * Misaligned requests can be executed through the ARC as
284 * uncached I/O. But if O_DIRECT was set by user and we
285 * were set to be strict, then it is a failure.
286 */
287 if ((*ioflagp & O_DIRECT) && zfs_dio_strict)
288 error = SET_ERROR(EINVAL);
289 goto out;
290 }
291
292 if (zn_has_cached_data(zp, zfs_uio_offset(uio),
293 zfs_uio_offset(uio) + zfs_uio_resid(uio) - 1)) {
294 /*
295 * The region is mmap'ed. The I/O request will be directed
296 * through the ARC as uncached I/O.
297 */
298 goto out;
299 }
300
301 /*
302 * For short writes the page mapping of Direct I/O makes no sense.
303 * Direct them through the ARC as uncached I/O.
304 */
305 if (rw == UIO_WRITE && zfs_uio_resid(uio) < zp->z_blksz)
306 goto out;
307
308 error = zfs_uio_get_dio_pages_alloc(uio, rw);
309 if (error)
310 goto out;
311 ASSERT(uio->uio_extflg & UIO_DIRECT);
312
313 out:
314 *ioflagp = ioflag;
315 return (error);
316 }
317
318 /*
319 * Read bytes from specified file into supplied buffer.
320 *
321 * IN: zp - inode of file to be read from.
322 * uio - structure supplying read location, range info,
323 * and return buffer.
324 * ioflag - O_SYNC flags; used to provide FRSYNC semantics.
325 * O_DIRECT flag; used to bypass page cache.
326 * cr - credentials of caller.
327 *
328 * OUT: uio - updated offset and range, buffer filled.
329 *
330 * RETURN: 0 on success, error code on failure.
331 *
332 * Side Effects:
333 * inode - atime updated if byte count > 0
334 */
335 int
zfs_read(struct znode * zp,zfs_uio_t * uio,int ioflag,cred_t * cr)336 zfs_read(struct znode *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
337 {
338 (void) cr;
339 int error = 0;
340 boolean_t frsync = B_FALSE;
341 boolean_t dio_checksum_failure = B_FALSE;
342
343 zfsvfs_t *zfsvfs = ZTOZSB(zp);
344 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
345 return (error);
346
347 if (zp->z_pflags & ZFS_AV_QUARANTINED) {
348 zfs_exit(zfsvfs, FTAG);
349 return (SET_ERROR(EACCES));
350 }
351
352 /* We don't copy out anything useful for directories. */
353 if (Z_ISDIR(ZTOTYPE(zp))) {
354 zfs_exit(zfsvfs, FTAG);
355 return (SET_ERROR(EISDIR));
356 }
357
358 /*
359 * Validate file offset
360 */
361 if (zfs_uio_offset(uio) < (offset_t)0) {
362 zfs_exit(zfsvfs, FTAG);
363 return (SET_ERROR(EINVAL));
364 }
365
366 /*
367 * Fasttrack empty reads
368 */
369 if (zfs_uio_resid(uio) == 0) {
370 zfs_exit(zfsvfs, FTAG);
371 return (0);
372 }
373
374 #ifdef FRSYNC
375 /*
376 * If we're in FRSYNC mode, sync out this znode before reading it.
377 * Only do this for non-snapshots.
378 *
379 * Some platforms do not support FRSYNC and instead map it
380 * to O_SYNC, which results in unnecessary calls to zil_commit. We
381 * only honor FRSYNC requests on platforms which support it.
382 */
383 frsync = !!(ioflag & FRSYNC);
384 #endif
385 if (zfsvfs->z_log &&
386 (frsync || zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)) {
387 error = zil_commit(zfsvfs->z_log, zp->z_id);
388 if (error != 0) {
389 zfs_exit(zfsvfs, FTAG);
390 return (error);
391 }
392 }
393
394 /*
395 * Lock the range against changes.
396 */
397 zfs_locked_range_t *lr = zfs_rangelock_enter(&zp->z_rangelock,
398 zfs_uio_offset(uio), zfs_uio_resid(uio), RL_READER);
399
400 /*
401 * If we are reading past end-of-file we can skip
402 * to the end; but we might still need to set atime.
403 */
404 if (zfs_uio_offset(uio) >= zp->z_size) {
405 error = 0;
406 goto out;
407 }
408 ASSERT(zfs_uio_offset(uio) < zp->z_size);
409
410 /*
411 * Setting up Direct I/O if requested.
412 */
413 error = zfs_setup_direct(zp, uio, UIO_READ, &ioflag);
414 if (error) {
415 goto out;
416 }
417
418 #if defined(__linux__)
419 ssize_t start_offset = zfs_uio_offset(uio);
420 #endif
421 uint_t blksz = zp->z_blksz;
422 ssize_t chunk_size;
423 ssize_t n = MIN(zfs_uio_resid(uio), zp->z_size - zfs_uio_offset(uio));
424 ssize_t start_resid = n;
425 ssize_t dio_remaining_resid = 0;
426
427 dmu_flags_t dflags = DMU_READ_PREFETCH;
428 if (ioflag & O_DIRECT)
429 dflags |= DMU_UNCACHEDIO;
430 if (uio->uio_extflg & UIO_DIRECT) {
431 /*
432 * All pages for an O_DIRECT request ahve already been mapped
433 * so there's no compelling reason to handle this uio in
434 * smaller chunks.
435 */
436 chunk_size = DMU_MAX_ACCESS;
437
438 /*
439 * In the event that the O_DIRECT request is reading the entire
440 * file, it is possible file's length is not page sized
441 * aligned. However, lower layers expect that the Direct I/O
442 * request is page-aligned. In this case, as much of the file
443 * that can be read using Direct I/O happens and the remaining
444 * amount will be read through the ARC.
445 *
446 * This is still consistent with the semantics of Direct I/O in
447 * ZFS as at a minimum the I/O request must be page-aligned.
448 */
449 dio_remaining_resid = n - P2ALIGN_TYPED(n, PAGE_SIZE, ssize_t);
450 if (dio_remaining_resid != 0)
451 n -= dio_remaining_resid;
452 dflags |= DMU_DIRECTIO;
453 } else {
454 chunk_size = MIN(MAX(zfs_vnops_read_chunk_size, blksz),
455 DMU_MAX_ACCESS / 2);
456 }
457
458 while (n > 0) {
459 ssize_t nbytes = MIN(n, chunk_size -
460 P2PHASE(zfs_uio_offset(uio), blksz));
461 #ifdef UIO_NOCOPY
462 if (zfs_uio_segflg(uio) == UIO_NOCOPY)
463 error = mappedread_sf(zp, nbytes, uio);
464 else
465 #endif
466 if (zn_has_cached_data(zp, zfs_uio_offset(uio),
467 zfs_uio_offset(uio) + nbytes - 1)) {
468 error = mappedread(zp, nbytes, uio);
469 } else {
470 error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl),
471 uio, nbytes, dflags);
472 }
473
474 if (error) {
475 /* convert checksum errors into IO errors */
476 if (error == ECKSUM) {
477 /*
478 * If a Direct I/O read returned a checksum
479 * verify error, then it must be treated as
480 * suspicious. The contents of the buffer could
481 * have beeen manipulated while the I/O was in
482 * flight. In this case, the remainder of I/O
483 * request will just be reissued through the
484 * ARC.
485 */
486 if (uio->uio_extflg & UIO_DIRECT) {
487 dio_checksum_failure = B_TRUE;
488 uio->uio_extflg &= ~UIO_DIRECT;
489 n += dio_remaining_resid;
490 dio_remaining_resid = 0;
491 continue;
492 } else {
493 error = SET_ERROR(EIO);
494 }
495 }
496
497 #if defined(__linux__)
498 /*
499 * if we actually read some bytes, bubbling EFAULT
500 * up to become EAGAIN isn't what we want here...
501 *
502 * ...on Linux, at least. On FBSD, doing this breaks.
503 */
504 if (error == EFAULT &&
505 (zfs_uio_offset(uio) - start_offset) != 0)
506 error = 0;
507 #endif
508 break;
509 }
510
511 n -= nbytes;
512 }
513
514 if (error == 0 && (uio->uio_extflg & UIO_DIRECT) &&
515 dio_remaining_resid != 0) {
516 /*
517 * Temporarily remove the UIO_DIRECT flag from the UIO so the
518 * remainder of the file can be read using the ARC.
519 */
520 uio->uio_extflg &= ~UIO_DIRECT;
521 dflags &= ~DMU_DIRECTIO;
522
523 if (zn_has_cached_data(zp, zfs_uio_offset(uio),
524 zfs_uio_offset(uio) + dio_remaining_resid - 1)) {
525 error = mappedread(zp, dio_remaining_resid, uio);
526 } else {
527 error = dmu_read_uio_dbuf(sa_get_db(zp->z_sa_hdl), uio,
528 dio_remaining_resid, dflags);
529 }
530 uio->uio_extflg |= UIO_DIRECT;
531 dflags |= DMU_DIRECTIO;
532
533 if (error != 0)
534 n += dio_remaining_resid;
535 } else if (error && (uio->uio_extflg & UIO_DIRECT)) {
536 n += dio_remaining_resid;
537 }
538 int64_t nread = start_resid - n;
539
540 dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nread);
541 out:
542 zfs_rangelock_exit(lr);
543
544 if (dio_checksum_failure == B_TRUE)
545 uio->uio_extflg |= UIO_DIRECT;
546
547 /*
548 * Cleanup for Direct I/O if requested.
549 */
550 if (uio->uio_extflg & UIO_DIRECT)
551 zfs_uio_free_dio_pages(uio, UIO_READ);
552
553 ZFS_ACCESSTIME_STAMP(zfsvfs, zp);
554 zfs_exit(zfsvfs, FTAG);
555 return (error);
556 }
557
558 static void
zfs_clear_setid_bits_if_necessary(zfsvfs_t * zfsvfs,znode_t * zp,cred_t * cr,uint64_t * clear_setid_bits_txgp,dmu_tx_t * tx)559 zfs_clear_setid_bits_if_necessary(zfsvfs_t *zfsvfs, znode_t *zp, cred_t *cr,
560 uint64_t *clear_setid_bits_txgp, dmu_tx_t *tx)
561 {
562 zilog_t *zilog = zfsvfs->z_log;
563 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
564
565 ASSERT(clear_setid_bits_txgp != NULL);
566 ASSERT(tx != NULL);
567
568 /*
569 * Clear Set-UID/Set-GID bits on successful write if not
570 * privileged and at least one of the execute bits is set.
571 *
572 * It would be nice to do this after all writes have
573 * been done, but that would still expose the ISUID/ISGID
574 * to another app after the partial write is committed.
575 *
576 * Note: we don't call zfs_fuid_map_id() here because
577 * user 0 is not an ephemeral uid.
578 */
579 mutex_enter(&zp->z_acl_lock);
580 if ((zp->z_mode & (S_IXUSR | (S_IXUSR >> 3) | (S_IXUSR >> 6))) != 0 &&
581 (zp->z_mode & (S_ISUID | S_ISGID)) != 0 &&
582 secpolicy_vnode_setid_retain(zp, cr,
583 ((zp->z_mode & S_ISUID) != 0 && uid == 0)) != 0) {
584 uint64_t newmode;
585
586 zp->z_mode &= ~(S_ISUID | S_ISGID);
587 newmode = zp->z_mode;
588 (void) sa_update(zp->z_sa_hdl, SA_ZPL_MODE(zfsvfs),
589 (void *)&newmode, sizeof (uint64_t), tx);
590
591 mutex_exit(&zp->z_acl_lock);
592
593 /*
594 * Make sure SUID/SGID bits will be removed when we replay the
595 * log. If the setid bits are keep coming back, don't log more
596 * than one TX_SETATTR per transaction group.
597 */
598 if (*clear_setid_bits_txgp != dmu_tx_get_txg(tx)) {
599 vattr_t va = {0};
600
601 va.va_mask = ATTR_MODE;
602 va.va_nodeid = zp->z_id;
603 va.va_mode = newmode;
604 zfs_log_setattr(zilog, tx, TX_SETATTR, zp, &va,
605 ATTR_MODE, NULL);
606 *clear_setid_bits_txgp = dmu_tx_get_txg(tx);
607 }
608 } else {
609 mutex_exit(&zp->z_acl_lock);
610 }
611 }
612
613 /*
614 * Write the bytes to a file.
615 *
616 * IN: zp - znode of file to be written to.
617 * uio - structure supplying write location, range info,
618 * and data buffer.
619 * ioflag - O_APPEND flag set if in append mode.
620 * O_DIRECT flag; used to bypass page cache.
621 * cr - credentials of caller.
622 *
623 * OUT: uio - updated offset and range.
624 *
625 * RETURN: 0 if success
626 * error code if failure
627 *
628 * Timestamps:
629 * ip - ctime|mtime updated if byte count > 0
630 */
631 int
zfs_write(znode_t * zp,zfs_uio_t * uio,int ioflag,cred_t * cr)632 zfs_write(znode_t *zp, zfs_uio_t *uio, int ioflag, cred_t *cr)
633 {
634 int error = 0, error1;
635 ssize_t start_resid = zfs_uio_resid(uio);
636 uint64_t clear_setid_bits_txg = 0;
637 boolean_t o_direct_defer = B_FALSE;
638
639 /*
640 * Fasttrack empty write
641 */
642 ssize_t n = start_resid;
643 if (n == 0)
644 return (0);
645
646 zfsvfs_t *zfsvfs = ZTOZSB(zp);
647 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
648 return (error);
649
650 sa_bulk_attr_t bulk[4];
651 int count = 0;
652 uint64_t mtime[2], ctime[2];
653 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
654 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
655 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
656 &zp->z_size, 8);
657 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_FLAGS(zfsvfs), NULL,
658 &zp->z_pflags, 8);
659
660 /*
661 * Callers might not be able to detect properly that we are read-only,
662 * so check it explicitly here.
663 */
664 if (zfs_is_readonly(zfsvfs)) {
665 zfs_exit(zfsvfs, FTAG);
666 return (SET_ERROR(EROFS));
667 }
668
669 /*
670 * If immutable or not appending then return EPERM.
671 * Intentionally allow ZFS_READONLY through here.
672 * See zfs_zaccess_common()
673 */
674 if ((zp->z_pflags & ZFS_IMMUTABLE) ||
675 ((zp->z_pflags & ZFS_APPENDONLY) && !(ioflag & O_APPEND) &&
676 (zfs_uio_offset(uio) < zp->z_size))) {
677 zfs_exit(zfsvfs, FTAG);
678 return (SET_ERROR(EPERM));
679 }
680
681 /*
682 * Validate file offset
683 */
684 offset_t woff = ioflag & O_APPEND ? zp->z_size : zfs_uio_offset(uio);
685 if (woff < 0) {
686 zfs_exit(zfsvfs, FTAG);
687 return (SET_ERROR(EINVAL));
688 }
689
690 /*
691 * Setting up Direct I/O if requested.
692 */
693 error = zfs_setup_direct(zp, uio, UIO_WRITE, &ioflag);
694 if (error) {
695 zfs_exit(zfsvfs, FTAG);
696 return (SET_ERROR(error));
697 }
698
699 /*
700 * Pre-fault the pages to ensure slow (eg NFS) pages
701 * don't hold up txg.
702 */
703 ssize_t pfbytes = MIN(n, DMU_MAX_ACCESS >> 1);
704 if (zfs_uio_prefaultpages(pfbytes, uio)) {
705 zfs_exit(zfsvfs, FTAG);
706 return (SET_ERROR(EFAULT));
707 }
708
709 /*
710 * If in append mode, set the io offset pointer to eof.
711 */
712 zfs_locked_range_t *lr;
713 if (ioflag & O_APPEND) {
714 /*
715 * Obtain an appending range lock to guarantee file append
716 * semantics. We reset the write offset once we have the lock.
717 */
718 lr = zfs_rangelock_enter(&zp->z_rangelock, 0, n, RL_APPEND);
719 woff = lr->lr_offset;
720 if (lr->lr_length == UINT64_MAX) {
721 /*
722 * We overlocked the file because this write will cause
723 * the file block size to increase.
724 * Note that zp_size cannot change with this lock held.
725 */
726 woff = zp->z_size;
727 }
728 zfs_uio_setoffset(uio, woff);
729 /*
730 * We need to update the starting offset as well because it is
731 * set previously in the ZPL (Linux) and VNOPS (FreeBSD)
732 * layers.
733 */
734 zfs_uio_setsoffset(uio, woff);
735 } else {
736 /*
737 * Note that if the file block size will change as a result of
738 * this write, then this range lock will lock the entire file
739 * so that we can re-write the block safely.
740 */
741 lr = zfs_rangelock_enter(&zp->z_rangelock, woff, n, RL_WRITER);
742 }
743
744 if (zn_rlimit_fsize_uio(zp, uio)) {
745 zfs_rangelock_exit(lr);
746 zfs_exit(zfsvfs, FTAG);
747 return (SET_ERROR(EFBIG));
748 }
749
750 const rlim64_t limit = MAXOFFSET_T;
751
752 if (woff >= limit) {
753 zfs_rangelock_exit(lr);
754 zfs_exit(zfsvfs, FTAG);
755 return (SET_ERROR(EFBIG));
756 }
757
758 if (n > limit - woff)
759 n = limit - woff;
760
761 uint64_t end_size = MAX(zp->z_size, woff + n);
762 zilog_t *zilog = zfsvfs->z_log;
763 boolean_t commit = (ioflag & (O_SYNC | O_DSYNC)) ||
764 (zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS);
765
766 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
767 const uint64_t gid = KGID_TO_SGID(ZTOGID(zp));
768 const uint64_t projid = zp->z_projid;
769
770 /*
771 * In the event we are increasing the file block size
772 * (lr_length == UINT64_MAX), we will direct the write to the ARC.
773 * Because zfs_grow_blocksize() will read from the ARC in order to
774 * grow the dbuf, we avoid doing Direct I/O here as that would cause
775 * data written to disk to be overwritten by data in the ARC during
776 * the sync phase. Besides writing data twice to disk, we also
777 * want to avoid consistency concerns between data in the the ARC and
778 * on disk while growing the file's blocksize.
779 *
780 * We will only temporarily remove Direct I/O and put it back after
781 * we have grown the blocksize. We do this in the event a request
782 * is larger than max_blksz, so further requests to
783 * dmu_write_uio_dbuf() will still issue the requests using Direct
784 * IO.
785 *
786 * As an example:
787 * The first block to file is being written as a 4k request with
788 * a recorsize of 1K. The first 1K issued in the loop below will go
789 * through the ARC; however, the following 3 1K requests will
790 * use Direct I/O.
791 */
792 if (uio->uio_extflg & UIO_DIRECT && lr->lr_length == UINT64_MAX) {
793 uio->uio_extflg &= ~UIO_DIRECT;
794 o_direct_defer = B_TRUE;
795 }
796
797 /*
798 * Write the file in reasonable size chunks. Each chunk is written
799 * in a separate transaction; this keeps the intent log records small
800 * and allows us to do more fine-grained space accounting.
801 */
802 while (n > 0) {
803 woff = zfs_uio_offset(uio);
804
805 if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) ||
806 zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) ||
807 (projid != ZFS_DEFAULT_PROJID &&
808 zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
809 projid))) {
810 error = SET_ERROR(EDQUOT);
811 break;
812 }
813
814 uint64_t blksz;
815 if (lr->lr_length == UINT64_MAX && zp->z_size <= zp->z_blksz) {
816 if (zp->z_blksz > zfsvfs->z_max_blksz &&
817 !ISP2(zp->z_blksz)) {
818 /*
819 * File's blocksize is already larger than the
820 * "recordsize" property. Only let it grow to
821 * the next power of 2.
822 */
823 blksz = 1 << highbit64(zp->z_blksz);
824 } else {
825 blksz = zfsvfs->z_max_blksz;
826 }
827 blksz = MIN(blksz, P2ROUNDUP(end_size,
828 SPA_MINBLOCKSIZE));
829 blksz = MAX(blksz, zp->z_blksz);
830 } else {
831 blksz = zp->z_blksz;
832 }
833
834 arc_buf_t *abuf = NULL;
835 ssize_t nbytes = n;
836 if (n >= blksz && woff >= zp->z_size &&
837 P2PHASE(woff, blksz) == 0 &&
838 !(uio->uio_extflg & UIO_DIRECT) &&
839 (blksz >= SPA_OLD_MAXBLOCKSIZE || n < 4 * blksz)) {
840 /*
841 * This write covers a full block. "Borrow" a buffer
842 * from the dmu so that we can fill it before we enter
843 * a transaction. This avoids the possibility of
844 * holding up the transaction if the data copy hangs
845 * up on a pagefault (e.g., from an NFS server mapping).
846 */
847 abuf = dmu_request_arcbuf(sa_get_db(zp->z_sa_hdl),
848 blksz);
849 ASSERT(abuf != NULL);
850 ASSERT(arc_buf_size(abuf) == blksz);
851 if ((error = zfs_uiocopy(abuf->b_data, blksz,
852 UIO_WRITE, uio, &nbytes))) {
853 dmu_return_arcbuf(abuf);
854 break;
855 }
856 ASSERT3S(nbytes, ==, blksz);
857 } else {
858 nbytes = MIN(n, (DMU_MAX_ACCESS >> 1) -
859 P2PHASE(woff, blksz));
860 if (pfbytes < nbytes) {
861 if (zfs_uio_prefaultpages(nbytes, uio)) {
862 error = SET_ERROR(EFAULT);
863 break;
864 }
865 pfbytes = nbytes;
866 }
867 }
868
869 /*
870 * Start a transaction.
871 */
872 dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
873 dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
874 dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
875 DB_DNODE_ENTER(db);
876 dmu_tx_hold_write_by_dnode(tx, DB_DNODE(db), woff, nbytes);
877 DB_DNODE_EXIT(db);
878 zfs_sa_upgrade_txholds(tx, zp);
879 error = dmu_tx_assign(tx, DMU_TX_WAIT);
880 if (error) {
881 dmu_tx_abort(tx);
882 if (abuf != NULL)
883 dmu_return_arcbuf(abuf);
884 break;
885 }
886
887 /*
888 * NB: We must call zfs_clear_setid_bits_if_necessary before
889 * committing the transaction!
890 */
891
892 /*
893 * If rangelock_enter() over-locked we grow the blocksize
894 * and then reduce the lock range. This will only happen
895 * on the first iteration since rangelock_reduce() will
896 * shrink down lr_length to the appropriate size.
897 */
898 if (lr->lr_length == UINT64_MAX) {
899 zfs_grow_blocksize(zp, blksz, tx);
900 zfs_rangelock_reduce(lr, woff, n);
901 }
902
903 dmu_flags_t dflags = DMU_READ_PREFETCH;
904 if (ioflag & O_DIRECT)
905 dflags |= DMU_UNCACHEDIO;
906 if (uio->uio_extflg & UIO_DIRECT)
907 dflags |= DMU_DIRECTIO;
908
909 ssize_t tx_bytes;
910 if (abuf == NULL) {
911 tx_bytes = zfs_uio_resid(uio);
912 zfs_uio_fault_disable(uio, B_TRUE);
913 error = dmu_write_uio_dbuf(sa_get_db(zp->z_sa_hdl),
914 uio, nbytes, tx, dflags);
915 zfs_uio_fault_disable(uio, B_FALSE);
916 #ifdef __linux__
917 if (error == EFAULT) {
918 zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
919 cr, &clear_setid_bits_txg, tx);
920 dmu_tx_commit(tx);
921 /*
922 * Account for partial writes before
923 * continuing the loop.
924 * Update needs to occur before the next
925 * zfs_uio_prefaultpages, or prefaultpages may
926 * error, and we may break the loop early.
927 */
928 n -= tx_bytes - zfs_uio_resid(uio);
929 pfbytes -= tx_bytes - zfs_uio_resid(uio);
930 continue;
931 }
932 #endif
933 /*
934 * On FreeBSD, EFAULT should be propagated back to the
935 * VFS, which will handle faulting and will retry.
936 */
937 if (error != 0 && error != EFAULT) {
938 zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
939 cr, &clear_setid_bits_txg, tx);
940 dmu_tx_commit(tx);
941 break;
942 }
943 tx_bytes -= zfs_uio_resid(uio);
944 } else {
945 /*
946 * Thus, we're writing a full block at a block-aligned
947 * offset and extending the file past EOF.
948 *
949 * dmu_assign_arcbuf_by_dbuf() will directly assign the
950 * arc buffer to a dbuf.
951 */
952 error = dmu_assign_arcbuf_by_dbuf(
953 sa_get_db(zp->z_sa_hdl), woff, abuf, tx, dflags);
954 if (error != 0) {
955 /*
956 * XXX This might not be necessary if
957 * dmu_assign_arcbuf_by_dbuf is guaranteed
958 * to be atomic.
959 */
960 zfs_clear_setid_bits_if_necessary(zfsvfs, zp,
961 cr, &clear_setid_bits_txg, tx);
962 dmu_return_arcbuf(abuf);
963 dmu_tx_commit(tx);
964 break;
965 }
966 ASSERT3S(nbytes, <=, zfs_uio_resid(uio));
967 zfs_uioskip(uio, nbytes);
968 tx_bytes = nbytes;
969 }
970 /*
971 * There is a window where a file's pages can be mmap'ed after
972 * zfs_setup_direct() is called. This is due to the fact that
973 * the rangelock in this function is acquired after calling
974 * zfs_setup_direct(). This is done so that
975 * zfs_uio_prefaultpages() does not attempt to fault in pages
976 * on Linux for Direct I/O requests. This is not necessary as
977 * the pages are pinned in memory and can not be faulted out.
978 * Ideally, the rangelock would be held before calling
979 * zfs_setup_direct() and zfs_uio_prefaultpages(); however,
980 * this can lead to a deadlock as zfs_getpage() also acquires
981 * the rangelock as a RL_WRITER and prefaulting the pages can
982 * lead to zfs_getpage() being called.
983 *
984 * In the case of the pages being mapped after
985 * zfs_setup_direct() is called, the call to update_pages()
986 * will still be made to make sure there is consistency between
987 * the ARC and the Linux page cache. This is an ufortunate
988 * situation as the data will be read back into the ARC after
989 * the Direct I/O write has completed, but this is the penality
990 * for writing to a mmap'ed region of a file using Direct I/O.
991 */
992 if (tx_bytes &&
993 zn_has_cached_data(zp, woff, woff + tx_bytes - 1)) {
994 update_pages(zp, woff, tx_bytes, zfsvfs->z_os);
995 }
996
997 /*
998 * If we made no progress, we're done. If we made even
999 * partial progress, update the znode and ZIL accordingly.
1000 */
1001 if (tx_bytes == 0) {
1002 (void) sa_update(zp->z_sa_hdl, SA_ZPL_SIZE(zfsvfs),
1003 (void *)&zp->z_size, sizeof (uint64_t), tx);
1004 dmu_tx_commit(tx);
1005 ASSERT(error != 0);
1006 break;
1007 }
1008
1009 zfs_clear_setid_bits_if_necessary(zfsvfs, zp, cr,
1010 &clear_setid_bits_txg, tx);
1011
1012 zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
1013
1014 /*
1015 * Update the file size (zp_size) if it has changed;
1016 * account for possible concurrent updates.
1017 */
1018 while ((end_size = zp->z_size) < zfs_uio_offset(uio)) {
1019 (void) atomic_cas_64(&zp->z_size, end_size,
1020 zfs_uio_offset(uio));
1021 ASSERT(error == 0 || error == EFAULT);
1022 }
1023 /*
1024 * If we are replaying and eof is non zero then force
1025 * the file size to the specified eof. Note, there's no
1026 * concurrency during replay.
1027 */
1028 if (zfsvfs->z_replay && zfsvfs->z_replay_eof != 0)
1029 zp->z_size = zfsvfs->z_replay_eof;
1030
1031 error1 = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
1032 if (error1 != 0)
1033 /* Avoid clobbering EFAULT. */
1034 error = error1;
1035
1036 /*
1037 * NB: During replay, the TX_SETATTR record logged by
1038 * zfs_clear_setid_bits_if_necessary must precede any of
1039 * the TX_WRITE records logged here.
1040 */
1041 zfs_log_write(zilog, tx, TX_WRITE, zp, woff, tx_bytes, commit,
1042 uio->uio_extflg & UIO_DIRECT ? B_TRUE : B_FALSE, NULL,
1043 NULL);
1044
1045 dmu_tx_commit(tx);
1046
1047 /*
1048 * Direct I/O was deferred in order to grow the first block.
1049 * At this point it can be re-enabled for subsequent writes.
1050 */
1051 if (o_direct_defer) {
1052 ASSERT(ioflag & O_DIRECT);
1053 uio->uio_extflg |= UIO_DIRECT;
1054 o_direct_defer = B_FALSE;
1055 }
1056
1057 if (error != 0)
1058 break;
1059 ASSERT3S(tx_bytes, ==, nbytes);
1060 n -= nbytes;
1061 pfbytes -= nbytes;
1062 }
1063
1064 if (o_direct_defer) {
1065 ASSERT(ioflag & O_DIRECT);
1066 uio->uio_extflg |= UIO_DIRECT;
1067 o_direct_defer = B_FALSE;
1068 }
1069
1070 zfs_znode_update_vfs(zp);
1071 zfs_rangelock_exit(lr);
1072
1073 /*
1074 * Cleanup for Direct I/O if requested.
1075 */
1076 if (uio->uio_extflg & UIO_DIRECT)
1077 zfs_uio_free_dio_pages(uio, UIO_WRITE);
1078
1079 /*
1080 * If we're in replay mode, or we made no progress, or the
1081 * uio data is inaccessible return an error. Otherwise, it's
1082 * at least a partial write, so it's successful.
1083 */
1084 if (zfsvfs->z_replay || zfs_uio_resid(uio) == start_resid ||
1085 error == EFAULT) {
1086 zfs_exit(zfsvfs, FTAG);
1087 return (error);
1088 }
1089
1090 if (commit) {
1091 error = zil_commit(zilog, zp->z_id);
1092 if (error != 0) {
1093 zfs_exit(zfsvfs, FTAG);
1094 return (error);
1095 }
1096 }
1097
1098 int64_t nwritten = start_resid - zfs_uio_resid(uio);
1099 dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nwritten);
1100
1101 zfs_exit(zfsvfs, FTAG);
1102 return (0);
1103 }
1104
1105 /*
1106 * Check if a block should be skipped during rewrite.
1107 * Returns B_TRUE if block should be skipped.
1108 */
1109 static boolean_t
zfs_rewrite_skip(dmu_buf_t * db,objset_t * os,uint64_t flags)1110 zfs_rewrite_skip(dmu_buf_t *db, objset_t *os, uint64_t flags)
1111 {
1112 /*
1113 * This may be slightly stale and racy, but should be OK for
1114 * the advisory use.
1115 */
1116 blkptr_t *bp = dmu_buf_get_blkptr(db);
1117 if (bp == NULL)
1118 return (B_TRUE);
1119
1120 if (flags & ZFS_REWRITE_SKIP_SNAPSHOT) {
1121 if (dmu_objset_block_is_shared(os, bp))
1122 return (B_TRUE);
1123 }
1124
1125 if (flags & ZFS_REWRITE_SKIP_BRT) {
1126 if (brt_maybe_exists(os->os_spa, bp))
1127 return (B_TRUE);
1128 }
1129
1130 return (B_FALSE);
1131 }
1132
1133 /*
1134 * Rewrite a range of file as-is without modification.
1135 *
1136 * IN: zp - znode of file to be rewritten.
1137 * off - Offset of the range to rewrite.
1138 * len - Length of the range to rewrite.
1139 * flags - Random rewrite parameters.
1140 * arg - flags-specific argument.
1141 *
1142 * RETURN: 0 if success
1143 * error code if failure
1144 */
1145 int
zfs_rewrite(znode_t * zp,uint64_t off,uint64_t len,uint64_t flags,uint64_t arg)1146 zfs_rewrite(znode_t *zp, uint64_t off, uint64_t len, uint64_t flags,
1147 uint64_t arg)
1148 {
1149 int error;
1150
1151 #define ZFS_REWRITE_VALID_FLAGS \
1152 (ZFS_REWRITE_PHYSICAL | ZFS_REWRITE_SKIP_SNAPSHOT | \
1153 ZFS_REWRITE_SKIP_BRT)
1154
1155 if ((flags & ~ZFS_REWRITE_VALID_FLAGS) != 0 || arg != 0)
1156 return (SET_ERROR(EINVAL));
1157
1158 zfsvfs_t *zfsvfs = ZTOZSB(zp);
1159 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1160 return (error);
1161
1162 /* Check if physical rewrite is allowed */
1163 spa_t *spa = zfsvfs->z_os->os_spa;
1164 if ((flags & ZFS_REWRITE_PHYSICAL) &&
1165 !spa_feature_is_enabled(spa, SPA_FEATURE_PHYSICAL_REWRITE)) {
1166 zfs_exit(zfsvfs, FTAG);
1167 return (SET_ERROR(ENOTSUP));
1168 }
1169
1170 if (zfs_is_readonly(zfsvfs)) {
1171 zfs_exit(zfsvfs, FTAG);
1172 return (SET_ERROR(EROFS));
1173 }
1174
1175 if (off >= zp->z_size) {
1176 zfs_exit(zfsvfs, FTAG);
1177 return (0);
1178 }
1179 if (len == 0 || len > zp->z_size - off)
1180 len = zp->z_size - off;
1181
1182 /* Flush any mmap()'d data to disk */
1183 if (zn_has_cached_data(zp, off, off + len - 1))
1184 zn_flush_cached_data(zp, B_TRUE);
1185
1186 zfs_locked_range_t *lr;
1187 lr = zfs_rangelock_enter(&zp->z_rangelock, off, len, RL_WRITER);
1188
1189 const uint64_t uid = KUID_TO_SUID(ZTOUID(zp));
1190 const uint64_t gid = KGID_TO_SGID(ZTOGID(zp));
1191 const uint64_t projid = zp->z_projid;
1192
1193 dmu_buf_impl_t *db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
1194 DB_DNODE_ENTER(db);
1195 dnode_t *dn = DB_DNODE(db);
1196
1197 uint64_t n, noff = off, nr = 0, nw = 0;
1198 while (len > 0) {
1199 /*
1200 * Rewrite only actual data, skipping any holes. This might
1201 * be inaccurate for dirty files, but we don't really care.
1202 */
1203 if (noff == off) {
1204 /* Find next data in the file. */
1205 error = dnode_next_offset(dn, 0, &noff, 1, 1, 0);
1206 if (error || noff >= off + len) {
1207 if (error == ESRCH) /* No more data. */
1208 error = 0;
1209 break;
1210 }
1211 ASSERT3U(noff, >=, off);
1212 len -= noff - off;
1213 off = noff;
1214
1215 /* Find where the data end. */
1216 error = dnode_next_offset(dn, DNODE_FIND_HOLE, &noff,
1217 1, 1, 0);
1218 if (error != 0)
1219 noff = off + len;
1220 }
1221 ASSERT3U(noff, >, off);
1222
1223 if (zfs_id_overblockquota(zfsvfs, DMU_USERUSED_OBJECT, uid) ||
1224 zfs_id_overblockquota(zfsvfs, DMU_GROUPUSED_OBJECT, gid) ||
1225 (projid != ZFS_DEFAULT_PROJID &&
1226 zfs_id_overblockquota(zfsvfs, DMU_PROJECTUSED_OBJECT,
1227 projid))) {
1228 error = SET_ERROR(EDQUOT);
1229 break;
1230 }
1231
1232 n = MIN(MIN(len, noff - off),
1233 DMU_MAX_ACCESS / 2 - P2PHASE(off, zp->z_blksz));
1234
1235 dmu_tx_t *tx = dmu_tx_create(zfsvfs->z_os);
1236 dmu_tx_hold_write_by_dnode(tx, dn, off, n);
1237 error = dmu_tx_assign(tx, DMU_TX_WAIT);
1238 if (error) {
1239 dmu_tx_abort(tx);
1240 break;
1241 }
1242
1243 /* Mark all dbufs within range as dirty to trigger rewrite. */
1244 dmu_buf_t **dbp;
1245 int numbufs;
1246 error = dmu_buf_hold_array_by_dnode(dn, off, n, TRUE, FTAG,
1247 &numbufs, &dbp, DMU_READ_PREFETCH | DMU_UNCACHEDIO);
1248 if (error) {
1249 dmu_tx_commit(tx);
1250 break;
1251 }
1252 for (int i = 0; i < numbufs; i++) {
1253 nr += dbp[i]->db_size;
1254 if (dmu_buf_is_dirty(dbp[i], tx))
1255 continue;
1256
1257 if (zfs_rewrite_skip(dbp[i], zfsvfs->z_os, flags))
1258 continue;
1259
1260 nw += dbp[i]->db_size;
1261 if (flags & ZFS_REWRITE_PHYSICAL)
1262 dmu_buf_will_rewrite(dbp[i], tx);
1263 else
1264 dmu_buf_will_dirty(dbp[i], tx);
1265 }
1266 dmu_buf_rele_array(dbp, numbufs, FTAG);
1267
1268 dmu_tx_commit(tx);
1269
1270 len -= n;
1271 off += n;
1272
1273 if (issig()) {
1274 error = SET_ERROR(EINTR);
1275 break;
1276 }
1277 }
1278
1279 DB_DNODE_EXIT(db);
1280
1281 dataset_kstats_update_read_kstats(&zfsvfs->z_kstat, nr);
1282 dataset_kstats_update_write_kstats(&zfsvfs->z_kstat, nw);
1283
1284 zfs_rangelock_exit(lr);
1285 zfs_exit(zfsvfs, FTAG);
1286 return (error);
1287 }
1288
1289 int
zfs_getsecattr(znode_t * zp,vsecattr_t * vsecp,int flag,cred_t * cr)1290 zfs_getsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
1291 {
1292 zfsvfs_t *zfsvfs = ZTOZSB(zp);
1293 int error;
1294 boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
1295
1296 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1297 return (error);
1298 error = zfs_getacl(zp, vsecp, skipaclchk, cr);
1299 zfs_exit(zfsvfs, FTAG);
1300
1301 return (error);
1302 }
1303
1304 int
zfs_setsecattr(znode_t * zp,vsecattr_t * vsecp,int flag,cred_t * cr)1305 zfs_setsecattr(znode_t *zp, vsecattr_t *vsecp, int flag, cred_t *cr)
1306 {
1307 zfsvfs_t *zfsvfs = ZTOZSB(zp);
1308 int error;
1309 boolean_t skipaclchk = (flag & ATTR_NOACLCHECK) ? B_TRUE : B_FALSE;
1310 zilog_t *zilog;
1311
1312 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
1313 return (error);
1314 zilog = zfsvfs->z_log;
1315 error = zfs_setacl(zp, vsecp, skipaclchk, cr);
1316
1317 if (error == 0 && zfsvfs->z_os->os_sync == ZFS_SYNC_ALWAYS)
1318 error = zil_commit(zilog, 0);
1319
1320 zfs_exit(zfsvfs, FTAG);
1321 return (error);
1322 }
1323
1324 /*
1325 * Get the optimal alignment to ensure direct IO can be performed without
1326 * incurring any RMW penalty on write. If direct IO is not enabled for this
1327 * file, returns an error.
1328 */
1329 int
zfs_get_direct_alignment(znode_t * zp,uint64_t * alignp)1330 zfs_get_direct_alignment(znode_t *zp, uint64_t *alignp)
1331 {
1332 zfsvfs_t *zfsvfs = ZTOZSB(zp);
1333
1334 if (!zfs_dio_enabled || zfsvfs->z_os->os_direct == ZFS_DIRECT_DISABLED)
1335 return (SET_ERROR(EOPNOTSUPP));
1336
1337 /*
1338 * If the file has multiple blocks, then its block size is fixed
1339 * forever, and so is the ideal alignment.
1340 *
1341 * If however it only has a single block, then we want to return the
1342 * max block size it could possibly grown to (ie, the dataset
1343 * recordsize). We do this so that a program querying alignment
1344 * immediately after the file is created gets a value that won't change
1345 * once the file has grown into the second block and beyond.
1346 *
1347 * Because we don't have a count of blocks easily available here, we
1348 * check if the apparent file size is smaller than its current block
1349 * size (meaning, the file hasn't yet grown into the current block
1350 * size) and then, check if the block size is smaller than the dataset
1351 * maximum (meaning, if the file grew past the current block size, the
1352 * block size could would be increased).
1353 */
1354 if (zp->z_size <= zp->z_blksz && zp->z_blksz < zfsvfs->z_max_blksz)
1355 *alignp = MAX(zfsvfs->z_max_blksz, PAGE_SIZE);
1356 else
1357 *alignp = MAX(zp->z_blksz, PAGE_SIZE);
1358
1359 return (0);
1360 }
1361
1362 #ifdef ZFS_DEBUG
1363 static int zil_fault_io = 0;
1364 #endif
1365
1366 static void zfs_get_done(zgd_t *zgd, int error);
1367
1368 /*
1369 * Get data to generate a TX_WRITE intent log record.
1370 */
1371 int
zfs_get_data(void * arg,uint64_t gen,lr_write_t * lr,char * buf,struct lwb * lwb,zio_t * zio)1372 zfs_get_data(void *arg, uint64_t gen, lr_write_t *lr, char *buf,
1373 struct lwb *lwb, zio_t *zio)
1374 {
1375 zfsvfs_t *zfsvfs = arg;
1376 objset_t *os = zfsvfs->z_os;
1377 znode_t *zp;
1378 uint64_t object = lr->lr_foid;
1379 uint64_t offset = lr->lr_offset;
1380 uint64_t size = lr->lr_length;
1381 zgd_t *zgd;
1382 int error = 0;
1383 uint64_t zp_gen;
1384
1385 ASSERT3P(lwb, !=, NULL);
1386 ASSERT3U(size, !=, 0);
1387
1388 /*
1389 * Nothing to do if the file has been removed
1390 */
1391 if (zfs_zget(zfsvfs, object, &zp) != 0)
1392 return (SET_ERROR(ENOENT));
1393 if (zp->z_unlinked) {
1394 /*
1395 * Release the vnode asynchronously as we currently have the
1396 * txg stopped from syncing.
1397 */
1398 zfs_zrele_async(zp);
1399 return (SET_ERROR(ENOENT));
1400 }
1401 /* check if generation number matches */
1402 if (sa_lookup(zp->z_sa_hdl, SA_ZPL_GEN(zfsvfs), &zp_gen,
1403 sizeof (zp_gen)) != 0) {
1404 zfs_zrele_async(zp);
1405 return (SET_ERROR(EIO));
1406 }
1407 if (zp_gen != gen) {
1408 zfs_zrele_async(zp);
1409 return (SET_ERROR(ENOENT));
1410 }
1411
1412 zgd = kmem_zalloc(sizeof (zgd_t), KM_SLEEP);
1413 zgd->zgd_lwb = lwb;
1414 zgd->zgd_private = zp;
1415
1416 /*
1417 * Write records come in two flavors: immediate and indirect.
1418 * For small writes it's cheaper to store the data with the
1419 * log record (immediate); for large writes it's cheaper to
1420 * sync the data and get a pointer to it (indirect) so that
1421 * we don't have to write the data twice.
1422 */
1423 if (buf != NULL) { /* immediate write */
1424 zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock, offset,
1425 size, RL_READER);
1426 /* test for truncation needs to be done while range locked */
1427 if (offset >= zp->z_size) {
1428 error = SET_ERROR(ENOENT);
1429 } else {
1430 error = dmu_read(os, object, offset, size, buf,
1431 DMU_READ_NO_PREFETCH | DMU_KEEP_CACHING);
1432 }
1433 ASSERT(error == 0 || error == ENOENT);
1434 } else { /* indirect write */
1435 ASSERT3P(zio, !=, NULL);
1436 /*
1437 * Have to lock the whole block to ensure when it's
1438 * written out and its checksum is being calculated
1439 * that no one can change the data. We need to re-check
1440 * blocksize after we get the lock in case it's changed!
1441 */
1442 for (;;) {
1443 uint64_t blkoff;
1444 size = zp->z_blksz;
1445 blkoff = ISP2(size) ? P2PHASE(offset, size) : offset;
1446 offset -= blkoff;
1447 zgd->zgd_lr = zfs_rangelock_enter(&zp->z_rangelock,
1448 offset, size, RL_READER);
1449 if (zp->z_blksz == size)
1450 break;
1451 offset += blkoff;
1452 zfs_rangelock_exit(zgd->zgd_lr);
1453 }
1454 /* test for truncation needs to be done while range locked */
1455 if (lr->lr_offset >= zp->z_size)
1456 error = SET_ERROR(ENOENT);
1457 #ifdef ZFS_DEBUG
1458 if (zil_fault_io) {
1459 error = SET_ERROR(EIO);
1460 zil_fault_io = 0;
1461 }
1462 #endif
1463
1464 dmu_buf_t *dbp;
1465 if (error == 0)
1466 error = dmu_buf_hold_noread(os, object, offset, zgd,
1467 &dbp);
1468
1469 if (error == 0) {
1470 zgd->zgd_db = dbp;
1471 dmu_buf_impl_t *db = (dmu_buf_impl_t *)dbp;
1472 boolean_t direct_write = B_FALSE;
1473 mutex_enter(&db->db_mtx);
1474 dbuf_dirty_record_t *dr =
1475 dbuf_find_dirty_eq(db, lr->lr_common.lrc_txg);
1476 if (dr != NULL && dr->dt.dl.dr_diowrite)
1477 direct_write = B_TRUE;
1478 mutex_exit(&db->db_mtx);
1479
1480 /*
1481 * All Direct I/O writes will have already completed and
1482 * the block pointer can be immediately stored in the
1483 * log record.
1484 */
1485 if (direct_write) {
1486 /*
1487 * A Direct I/O write always covers an entire
1488 * block.
1489 */
1490 ASSERT3U(dbp->db_size, ==, zp->z_blksz);
1491 lr->lr_blkptr = dr->dt.dl.dr_overridden_by;
1492 zfs_get_done(zgd, 0);
1493 return (0);
1494 }
1495
1496 blkptr_t *bp = &lr->lr_blkptr;
1497 zgd->zgd_bp = bp;
1498
1499 ASSERT3U(dbp->db_offset, ==, offset);
1500 ASSERT3U(dbp->db_size, ==, size);
1501
1502 error = dmu_sync(zio, lr->lr_common.lrc_txg,
1503 zfs_get_done, zgd);
1504 ASSERT(error || lr->lr_length <= size);
1505
1506 /*
1507 * On success, we need to wait for the write I/O
1508 * initiated by dmu_sync() to complete before we can
1509 * release this dbuf. We will finish everything up
1510 * in the zfs_get_done() callback.
1511 */
1512 if (error == 0)
1513 return (0);
1514
1515 if (error == EALREADY) {
1516 lr->lr_common.lrc_txtype = TX_WRITE2;
1517 /*
1518 * TX_WRITE2 relies on the data previously
1519 * written by the TX_WRITE that caused
1520 * EALREADY. We zero out the BP because
1521 * it is the old, currently-on-disk BP.
1522 */
1523 zgd->zgd_bp = NULL;
1524 BP_ZERO(bp);
1525 error = 0;
1526 }
1527 }
1528 }
1529
1530 zfs_get_done(zgd, error);
1531
1532 return (error);
1533 }
1534
1535 static void
zfs_get_done(zgd_t * zgd,int error)1536 zfs_get_done(zgd_t *zgd, int error)
1537 {
1538 (void) error;
1539 znode_t *zp = zgd->zgd_private;
1540
1541 if (zgd->zgd_db)
1542 dmu_buf_rele(zgd->zgd_db, zgd);
1543
1544 zfs_rangelock_exit(zgd->zgd_lr);
1545
1546 /*
1547 * Release the vnode asynchronously as we currently have the
1548 * txg stopped from syncing.
1549 */
1550 zfs_zrele_async(zp);
1551
1552 kmem_free(zgd, sizeof (zgd_t));
1553 }
1554
1555 static int
zfs_enter_two(zfsvfs_t * zfsvfs1,zfsvfs_t * zfsvfs2,const char * tag)1556 zfs_enter_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
1557 {
1558 int error;
1559
1560 /* Swap. Not sure if the order of zfs_enter()s is important. */
1561 if (zfsvfs1 > zfsvfs2) {
1562 zfsvfs_t *tmpzfsvfs;
1563
1564 tmpzfsvfs = zfsvfs2;
1565 zfsvfs2 = zfsvfs1;
1566 zfsvfs1 = tmpzfsvfs;
1567 }
1568
1569 error = zfs_enter(zfsvfs1, tag);
1570 if (error != 0)
1571 return (error);
1572 if (zfsvfs1 != zfsvfs2) {
1573 error = zfs_enter(zfsvfs2, tag);
1574 if (error != 0) {
1575 zfs_exit(zfsvfs1, tag);
1576 return (error);
1577 }
1578 }
1579
1580 return (0);
1581 }
1582
1583 static void
zfs_exit_two(zfsvfs_t * zfsvfs1,zfsvfs_t * zfsvfs2,const char * tag)1584 zfs_exit_two(zfsvfs_t *zfsvfs1, zfsvfs_t *zfsvfs2, const char *tag)
1585 {
1586
1587 zfs_exit(zfsvfs1, tag);
1588 if (zfsvfs1 != zfsvfs2)
1589 zfs_exit(zfsvfs2, tag);
1590 }
1591
1592 /*
1593 * We split each clone request in chunks that can fit into a single ZIL
1594 * log entry. Each ZIL log entry can fit 130816 bytes for a block cloning
1595 * operation (see zil_max_log_data() and zfs_log_clone_range()). This gives
1596 * us room for storing 1022 block pointers.
1597 *
1598 * On success, the function return the number of bytes copied in *lenp.
1599 * Note, it doesn't return how much bytes are left to be copied.
1600 * On errors which are caused by any file system limitations or
1601 * brt limitations `EINVAL` is returned. In the most cases a user
1602 * requested bad parameters, it could be possible to clone the file but
1603 * some parameters don't match the requirements.
1604 */
1605 int
zfs_clone_range(znode_t * inzp,uint64_t * inoffp,znode_t * outzp,uint64_t * outoffp,uint64_t * lenp,cred_t * cr)1606 zfs_clone_range(znode_t *inzp, uint64_t *inoffp, znode_t *outzp,
1607 uint64_t *outoffp, uint64_t *lenp, cred_t *cr)
1608 {
1609 zfsvfs_t *inzfsvfs, *outzfsvfs;
1610 objset_t *inos, *outos;
1611 zfs_locked_range_t *inlr, *outlr;
1612 dmu_buf_impl_t *db;
1613 dmu_tx_t *tx;
1614 zilog_t *zilog;
1615 uint64_t inoff, outoff, len, done;
1616 uint64_t outsize, size;
1617 int error;
1618 int count = 0;
1619 sa_bulk_attr_t bulk[3];
1620 uint64_t mtime[2], ctime[2];
1621 uint64_t uid, gid, projid;
1622 blkptr_t *bps;
1623 size_t maxblocks, nbps;
1624 uint_t inblksz;
1625 uint64_t clear_setid_bits_txg = 0;
1626 uint64_t last_synced_txg = 0;
1627
1628 inoff = *inoffp;
1629 outoff = *outoffp;
1630 len = *lenp;
1631 done = 0;
1632
1633 inzfsvfs = ZTOZSB(inzp);
1634 outzfsvfs = ZTOZSB(outzp);
1635
1636 /*
1637 * We need to call zfs_enter() potentially on two different datasets,
1638 * so we need a dedicated function for that.
1639 */
1640 error = zfs_enter_two(inzfsvfs, outzfsvfs, FTAG);
1641 if (error != 0)
1642 return (error);
1643
1644 inos = inzfsvfs->z_os;
1645 outos = outzfsvfs->z_os;
1646
1647 /*
1648 * Both source and destination have to belong to the same storage pool.
1649 */
1650 if (dmu_objset_spa(inos) != dmu_objset_spa(outos)) {
1651 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1652 return (SET_ERROR(EXDEV));
1653 }
1654
1655 /*
1656 * outos and inos belongs to the same storage pool.
1657 * see a few lines above, only one check.
1658 */
1659 if (!spa_feature_is_enabled(dmu_objset_spa(outos),
1660 SPA_FEATURE_BLOCK_CLONING)) {
1661 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1662 return (SET_ERROR(EOPNOTSUPP));
1663 }
1664
1665 ASSERT(!outzfsvfs->z_replay);
1666
1667 /*
1668 * Block cloning from an unencrypted dataset into an encrypted
1669 * dataset and vice versa is not supported.
1670 */
1671 if (inos->os_encrypted != outos->os_encrypted) {
1672 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1673 return (SET_ERROR(EXDEV));
1674 }
1675
1676 /*
1677 * Cloning across encrypted datasets is possible only if they
1678 * share the same master key.
1679 */
1680 if (inos != outos && inos->os_encrypted &&
1681 !dmu_objset_crypto_key_equal(inos, outos)) {
1682 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1683 return (SET_ERROR(EXDEV));
1684 }
1685
1686 /*
1687 * Cloning between datasets with different properties is possible,
1688 * but it may cause confusions when copying data between them and
1689 * expecting new properties to apply.
1690 */
1691 if (zfs_bclone_strict_properties && inos != outos &&
1692 !inzfsvfs->z_issnap &&
1693 (inos->os_checksum != outos->os_checksum ||
1694 inos->os_compress != outos->os_compress ||
1695 inos->os_copies != outos->os_copies ||
1696 inos->os_dedup_checksum != outos->os_dedup_checksum)) {
1697 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1698 return (SET_ERROR(EXDEV));
1699 }
1700
1701 error = zfs_verify_zp(inzp);
1702 if (error == 0)
1703 error = zfs_verify_zp(outzp);
1704 if (error != 0) {
1705 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1706 return (error);
1707 }
1708
1709 /*
1710 * We don't copy source file's flags that's why we don't allow to clone
1711 * files that are in quarantine.
1712 */
1713 if (inzp->z_pflags & ZFS_AV_QUARANTINED) {
1714 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1715 return (SET_ERROR(EACCES));
1716 }
1717
1718 if (inoff >= inzp->z_size) {
1719 *lenp = 0;
1720 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1721 return (0);
1722 }
1723 if (len > inzp->z_size - inoff) {
1724 len = inzp->z_size - inoff;
1725 }
1726 if (len == 0) {
1727 *lenp = 0;
1728 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1729 return (0);
1730 }
1731
1732 /*
1733 * Callers might not be able to detect properly that we are read-only,
1734 * so check it explicitly here.
1735 */
1736 if (zfs_is_readonly(outzfsvfs)) {
1737 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1738 return (SET_ERROR(EROFS));
1739 }
1740
1741 /*
1742 * If immutable or not appending then return EPERM.
1743 * Intentionally allow ZFS_READONLY through here.
1744 * See zfs_zaccess_common()
1745 */
1746 if ((outzp->z_pflags & ZFS_IMMUTABLE) != 0) {
1747 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1748 return (SET_ERROR(EPERM));
1749 }
1750
1751 /*
1752 * No overlapping if we are cloning within the same file.
1753 */
1754 if (inzp == outzp) {
1755 if (inoff < outoff + len && outoff < inoff + len) {
1756 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
1757 return (SET_ERROR(EINVAL));
1758 }
1759 }
1760
1761 /* Flush any mmap()'d data to disk */
1762 if (zn_has_cached_data(inzp, inoff, inoff + len - 1))
1763 zn_flush_cached_data(inzp, B_TRUE);
1764
1765 /*
1766 * Maintain predictable lock order.
1767 */
1768 if (inzp < outzp || (inzp == outzp && inoff < outoff)) {
1769 inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
1770 RL_READER);
1771 outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
1772 RL_WRITER);
1773 } else {
1774 outlr = zfs_rangelock_enter(&outzp->z_rangelock, outoff, len,
1775 RL_WRITER);
1776 inlr = zfs_rangelock_enter(&inzp->z_rangelock, inoff, len,
1777 RL_READER);
1778 }
1779
1780 inblksz = inzp->z_blksz;
1781
1782 /*
1783 * Cloning between datasets with different special_small_blocks would
1784 * bypass storage tier migration that would occur with a regular copy.
1785 */
1786 if (zfs_bclone_strict_properties && inos != outos &&
1787 !inzfsvfs->z_issnap && spa_has_special(dmu_objset_spa(inos))) {
1788 uint64_t in_smallblk = inos->os_zpl_special_smallblock;
1789 uint64_t out_smallblk = outos->os_zpl_special_smallblock;
1790 if (in_smallblk != out_smallblk) {
1791 uint64_t min_smallblk = MIN(in_smallblk, out_smallblk);
1792 uint64_t max_smallblk = MAX(in_smallblk, out_smallblk);
1793 if (min_smallblk < inblksz &&
1794 (inos->os_compress != ZIO_COMPRESS_OFF ||
1795 max_smallblk >= inblksz)) {
1796 error = SET_ERROR(EXDEV);
1797 goto unlock;
1798 }
1799 }
1800 }
1801
1802 /*
1803 * We cannot clone into a file with different block size if we can't
1804 * grow it (block size is already bigger, has more than one block, or
1805 * not locked for growth). There are other possible reasons for the
1806 * grow to fail, but we cover what we can before opening transaction
1807 * and the rest detect after we try to do it.
1808 */
1809 if (inblksz < outzp->z_blksz) {
1810 error = SET_ERROR(EINVAL);
1811 goto unlock;
1812 }
1813 if (inblksz != outzp->z_blksz && (outzp->z_size > outzp->z_blksz ||
1814 outlr->lr_length != UINT64_MAX)) {
1815 error = SET_ERROR(EINVAL);
1816 goto unlock;
1817 }
1818
1819 /*
1820 * Block size must be power-of-2 if destination offset != 0.
1821 * There can be no multiple blocks of non-power-of-2 size.
1822 */
1823 if (outoff != 0 && !ISP2(inblksz)) {
1824 error = SET_ERROR(EINVAL);
1825 goto unlock;
1826 }
1827
1828 /*
1829 * Offsets and len must be at block boundries.
1830 */
1831 if ((inoff % inblksz) != 0 || (outoff % inblksz) != 0) {
1832 error = SET_ERROR(EINVAL);
1833 goto unlock;
1834 }
1835 /*
1836 * Length must be multipe of blksz, except for the end of the file.
1837 */
1838 if ((len % inblksz) != 0 &&
1839 (len < inzp->z_size - inoff || len < outzp->z_size - outoff)) {
1840 error = SET_ERROR(EINVAL);
1841 goto unlock;
1842 }
1843
1844 /*
1845 * If we are copying only one block and it is smaller than recordsize
1846 * property, do not allow destination to grow beyond one block if it
1847 * is not there yet. Otherwise the destination will get stuck with
1848 * that block size forever, that can be as small as 512 bytes, no
1849 * matter how big the destination grow later.
1850 */
1851 if (len <= inblksz && inblksz < outzfsvfs->z_max_blksz &&
1852 outzp->z_size <= inblksz && outoff + len > inblksz) {
1853 error = SET_ERROR(EINVAL);
1854 goto unlock;
1855 }
1856
1857 error = zn_rlimit_fsize(outoff + len);
1858 if (error != 0) {
1859 goto unlock;
1860 }
1861
1862 if (inoff >= MAXOFFSET_T || outoff >= MAXOFFSET_T) {
1863 error = SET_ERROR(EFBIG);
1864 goto unlock;
1865 }
1866
1867 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(outzfsvfs), NULL,
1868 &mtime, 16);
1869 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(outzfsvfs), NULL,
1870 &ctime, 16);
1871 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(outzfsvfs), NULL,
1872 &outzp->z_size, 8);
1873
1874 zilog = outzfsvfs->z_log;
1875 maxblocks = zil_max_log_data(zilog, sizeof (lr_clone_range_t)) /
1876 sizeof (bps[0]);
1877
1878 uid = KUID_TO_SUID(ZTOUID(outzp));
1879 gid = KGID_TO_SGID(ZTOGID(outzp));
1880 projid = outzp->z_projid;
1881
1882 bps = vmem_alloc(sizeof (bps[0]) * maxblocks, KM_SLEEP);
1883
1884 /*
1885 * Clone the file in reasonable size chunks. Each chunk is cloned
1886 * in a separate transaction; this keeps the intent log records small
1887 * and allows us to do more fine-grained space accounting.
1888 */
1889 while (len > 0) {
1890 size = MIN(inblksz * maxblocks, len);
1891
1892 if (zfs_id_overblockquota(outzfsvfs, DMU_USERUSED_OBJECT,
1893 uid) ||
1894 zfs_id_overblockquota(outzfsvfs, DMU_GROUPUSED_OBJECT,
1895 gid) ||
1896 (projid != ZFS_DEFAULT_PROJID &&
1897 zfs_id_overblockquota(outzfsvfs, DMU_PROJECTUSED_OBJECT,
1898 projid))) {
1899 error = SET_ERROR(EDQUOT);
1900 break;
1901 }
1902
1903 nbps = maxblocks;
1904 last_synced_txg = spa_last_synced_txg(dmu_objset_spa(inos));
1905 error = dmu_read_l0_bps(inos, inzp->z_id, inoff, size, bps,
1906 &nbps);
1907 if (error != 0) {
1908 /*
1909 * If we are trying to clone a block that was created
1910 * in the current transaction group, the error will be
1911 * EAGAIN here. Based on zfs_bclone_wait_dirty either
1912 * return a shortened range to the caller so it can
1913 * fallback, or wait for the next TXG and check again.
1914 */
1915 if (error == EAGAIN && zfs_bclone_wait_dirty) {
1916 txg_wait_flag_t wait_flags =
1917 spa_get_failmode(dmu_objset_spa(inos)) ==
1918 ZIO_FAILURE_MODE_CONTINUE ?
1919 TXG_WAIT_SUSPEND : 0;
1920 error = txg_wait_synced_flags(
1921 dmu_objset_pool(inos), last_synced_txg + 1,
1922 wait_flags);
1923 if (error == 0)
1924 continue;
1925 ASSERT3U(error, ==, ESHUTDOWN);
1926 error = SET_ERROR(EIO);
1927 }
1928
1929 break;
1930 }
1931
1932 /*
1933 * Start a transaction.
1934 */
1935 tx = dmu_tx_create(outos);
1936 dmu_tx_hold_sa(tx, outzp->z_sa_hdl, B_FALSE);
1937 db = (dmu_buf_impl_t *)sa_get_db(outzp->z_sa_hdl);
1938 DB_DNODE_ENTER(db);
1939 dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), outoff, size,
1940 inblksz);
1941 DB_DNODE_EXIT(db);
1942 zfs_sa_upgrade_txholds(tx, outzp);
1943 error = dmu_tx_assign(tx, DMU_TX_WAIT);
1944 if (error != 0) {
1945 dmu_tx_abort(tx);
1946 break;
1947 }
1948
1949 /*
1950 * Copy source znode's block size. This is done only if the
1951 * whole znode is locked (see zfs_rangelock_cb()) and only
1952 * on the first iteration since zfs_rangelock_reduce() will
1953 * shrink down lr_length to the appropriate size.
1954 */
1955 if (outlr->lr_length == UINT64_MAX) {
1956 zfs_grow_blocksize(outzp, inblksz, tx);
1957
1958 /*
1959 * Block growth may fail for many reasons we can not
1960 * predict here. If it happen the cloning is doomed.
1961 */
1962 if (inblksz != outzp->z_blksz) {
1963 error = SET_ERROR(EINVAL);
1964 dmu_tx_commit(tx);
1965 break;
1966 }
1967
1968 /*
1969 * Round range lock up to the block boundary, so we
1970 * prevent appends until we are done.
1971 */
1972 zfs_rangelock_reduce(outlr, outoff,
1973 ((len - 1) / inblksz + 1) * inblksz);
1974 }
1975
1976 error = dmu_brt_clone(outos, outzp->z_id, outoff, size, tx,
1977 bps, nbps);
1978 if (error != 0) {
1979 dmu_tx_commit(tx);
1980 break;
1981 }
1982
1983 if (zn_has_cached_data(outzp, outoff, outoff + size - 1)) {
1984 update_pages(outzp, outoff, size, outos);
1985 }
1986
1987 zfs_clear_setid_bits_if_necessary(outzfsvfs, outzp, cr,
1988 &clear_setid_bits_txg, tx);
1989
1990 zfs_tstamp_update_setup(outzp, CONTENT_MODIFIED, mtime, ctime);
1991
1992 /*
1993 * Update the file size (zp_size) if it has changed;
1994 * account for possible concurrent updates.
1995 */
1996 while ((outsize = outzp->z_size) < outoff + size) {
1997 (void) atomic_cas_64(&outzp->z_size, outsize,
1998 outoff + size);
1999 }
2000
2001 error = sa_bulk_update(outzp->z_sa_hdl, bulk, count, tx);
2002
2003 zfs_log_clone_range(zilog, tx, TX_CLONE_RANGE, outzp, outoff,
2004 size, inblksz, bps, nbps);
2005
2006 dmu_tx_commit(tx);
2007
2008 if (error != 0)
2009 break;
2010
2011 inoff += size;
2012 outoff += size;
2013 len -= size;
2014 done += size;
2015
2016 if (issig()) {
2017 error = SET_ERROR(EINTR);
2018 break;
2019 }
2020 }
2021
2022 vmem_free(bps, sizeof (bps[0]) * maxblocks);
2023 zfs_znode_update_vfs(outzp);
2024
2025 unlock:
2026 zfs_rangelock_exit(outlr);
2027 zfs_rangelock_exit(inlr);
2028
2029 if (done > 0) {
2030 /*
2031 * If we have made at least partial progress, reset the error.
2032 */
2033 error = 0;
2034
2035 ZFS_ACCESSTIME_STAMP(inzfsvfs, inzp);
2036
2037 if (outos->os_sync == ZFS_SYNC_ALWAYS) {
2038 error = zil_commit(zilog, outzp->z_id);
2039 }
2040
2041 *inoffp += done;
2042 *outoffp += done;
2043 *lenp = done;
2044 } else {
2045 /*
2046 * If we made no progress, there must be a good reason.
2047 * EOF is handled explicitly above, before the loop.
2048 */
2049 ASSERT3S(error, !=, 0);
2050 }
2051
2052 zfs_exit_two(inzfsvfs, outzfsvfs, FTAG);
2053
2054 return (error);
2055 }
2056
2057 /*
2058 * Usual pattern would be to call zfs_clone_range() from zfs_replay_clone(),
2059 * but we cannot do that, because when replaying we don't have source znode
2060 * available. This is why we need a dedicated replay function.
2061 */
2062 int
zfs_clone_range_replay(znode_t * zp,uint64_t off,uint64_t len,uint64_t blksz,const blkptr_t * bps,size_t nbps)2063 zfs_clone_range_replay(znode_t *zp, uint64_t off, uint64_t len, uint64_t blksz,
2064 const blkptr_t *bps, size_t nbps)
2065 {
2066 zfsvfs_t *zfsvfs;
2067 dmu_buf_impl_t *db;
2068 dmu_tx_t *tx;
2069 int error;
2070 int count = 0;
2071 sa_bulk_attr_t bulk[3];
2072 uint64_t mtime[2], ctime[2];
2073
2074 ASSERT3U(off, <, MAXOFFSET_T);
2075 ASSERT3U(len, >, 0);
2076 ASSERT3U(nbps, >, 0);
2077
2078 zfsvfs = ZTOZSB(zp);
2079
2080 ASSERT(spa_feature_is_enabled(dmu_objset_spa(zfsvfs->z_os),
2081 SPA_FEATURE_BLOCK_CLONING));
2082
2083 if ((error = zfs_enter_verify_zp(zfsvfs, zp, FTAG)) != 0)
2084 return (error);
2085
2086 ASSERT(zfsvfs->z_replay);
2087 ASSERT(!zfs_is_readonly(zfsvfs));
2088
2089 if ((off % blksz) != 0) {
2090 zfs_exit(zfsvfs, FTAG);
2091 return (SET_ERROR(EINVAL));
2092 }
2093
2094 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_MTIME(zfsvfs), NULL, &mtime, 16);
2095 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_CTIME(zfsvfs), NULL, &ctime, 16);
2096 SA_ADD_BULK_ATTR(bulk, count, SA_ZPL_SIZE(zfsvfs), NULL,
2097 &zp->z_size, 8);
2098
2099 /*
2100 * Start a transaction.
2101 */
2102 tx = dmu_tx_create(zfsvfs->z_os);
2103
2104 dmu_tx_hold_sa(tx, zp->z_sa_hdl, B_FALSE);
2105 db = (dmu_buf_impl_t *)sa_get_db(zp->z_sa_hdl);
2106 DB_DNODE_ENTER(db);
2107 dmu_tx_hold_clone_by_dnode(tx, DB_DNODE(db), off, len, blksz);
2108 DB_DNODE_EXIT(db);
2109 zfs_sa_upgrade_txholds(tx, zp);
2110 error = dmu_tx_assign(tx, DMU_TX_WAIT);
2111 if (error != 0) {
2112 dmu_tx_abort(tx);
2113 zfs_exit(zfsvfs, FTAG);
2114 return (error);
2115 }
2116
2117 if (zp->z_blksz < blksz)
2118 zfs_grow_blocksize(zp, blksz, tx);
2119
2120 dmu_brt_clone(zfsvfs->z_os, zp->z_id, off, len, tx, bps, nbps);
2121
2122 zfs_tstamp_update_setup(zp, CONTENT_MODIFIED, mtime, ctime);
2123
2124 if (zp->z_size < off + len)
2125 zp->z_size = off + len;
2126
2127 error = sa_bulk_update(zp->z_sa_hdl, bulk, count, tx);
2128
2129 /*
2130 * zil_replaying() not only check if we are replaying ZIL, but also
2131 * updates the ZIL header to record replay progress.
2132 */
2133 VERIFY(zil_replaying(zfsvfs->z_log, tx));
2134
2135 dmu_tx_commit(tx);
2136
2137 zfs_znode_update_vfs(zp);
2138
2139 zfs_exit(zfsvfs, FTAG);
2140
2141 return (error);
2142 }
2143
2144 EXPORT_SYMBOL(zfs_access);
2145 EXPORT_SYMBOL(zfs_fsync);
2146 EXPORT_SYMBOL(zfs_holey);
2147 EXPORT_SYMBOL(zfs_read);
2148 EXPORT_SYMBOL(zfs_write);
2149 EXPORT_SYMBOL(zfs_getsecattr);
2150 EXPORT_SYMBOL(zfs_setsecattr);
2151 EXPORT_SYMBOL(zfs_clone_range);
2152 EXPORT_SYMBOL(zfs_clone_range_replay);
2153
2154 ZFS_MODULE_PARAM(zfs_vnops, zfs_vnops_, read_chunk_size, U64, ZMOD_RW,
2155 "Bytes to read per chunk");
2156
2157 ZFS_MODULE_PARAM(zfs, zfs_, bclone_enabled, INT, ZMOD_RW,
2158 "Enable block cloning");
2159
2160 ZFS_MODULE_PARAM(zfs, zfs_, bclone_strict_properties, INT, ZMOD_RW,
2161 "Restrict cross-dataset cloning with different properties");
2162
2163 ZFS_MODULE_PARAM(zfs, zfs_, bclone_wait_dirty, INT, ZMOD_RW,
2164 "Wait for dirty blocks when cloning");
2165
2166 ZFS_MODULE_PARAM(zfs, zfs_, dio_enabled, INT, ZMOD_RW,
2167 "Enable Direct I/O");
2168
2169 ZFS_MODULE_PARAM(zfs, zfs_, dio_strict, INT, ZMOD_RW,
2170 "Return errors on misaligned Direct I/O");
2171