1 /*-
2 * SPDX-License-Identifier: BSD-3-Clause
3 *
4 * Copyright (c) 2007-2009 Google Inc.
5 * All rights reserved.
6 *
7 * Redistribution and use in source and binary forms, with or without
8 * modification, are permitted provided that the following conditions are
9 * met:
10 *
11 * * Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * * Redistributions in binary form must reproduce the above
14 * copyright notice, this list of conditions and the following disclaimer
15 * in the documentation and/or other materials provided with the
16 * distribution.
17 * * Neither the name of Google Inc. nor the names of its
18 * contributors may be used to endorse or promote products derived from
19 * this software without specific prior written permission.
20 *
21 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
22 * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
23 * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
24 * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
25 * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
26 * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
27 * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
28 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
29 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
30 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
31 * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
32 *
33 * Copyright (C) 2005 Csaba Henk.
34 * All rights reserved.
35 *
36 * Copyright (c) 2019 The FreeBSD Foundation
37 *
38 * Portions of this software were developed by BFF Storage Systems, LLC under
39 * sponsorship from the FreeBSD Foundation.
40 *
41 * Redistribution and use in source and binary forms, with or without
42 * modification, are permitted provided that the following conditions
43 * are met:
44 * 1. Redistributions of source code must retain the above copyright
45 * notice, this list of conditions and the following disclaimer.
46 * 2. Redistributions in binary form must reproduce the above copyright
47 * notice, this list of conditions and the following disclaimer in the
48 * documentation and/or other materials provided with the distribution.
49 *
50 * THIS SOFTWARE IS PROVIDED BY AUTHOR AND CONTRIBUTORS ``AS IS'' AND
51 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
52 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
53 * ARE DISCLAIMED. IN NO EVENT SHALL AUTHOR OR CONTRIBUTORS BE LIABLE
54 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
55 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
56 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
57 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
58 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
59 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
60 * SUCH DAMAGE.
61 */
62
63 #include <sys/types.h>
64 #include <sys/param.h>
65 #include <sys/module.h>
66 #include <sys/systm.h>
67 #include <sys/errno.h>
68 #include <sys/param.h>
69 #include <sys/kernel.h>
70 #include <sys/conf.h>
71 #include <sys/uio.h>
72 #include <sys/malloc.h>
73 #include <sys/queue.h>
74 #include <sys/lock.h>
75 #include <sys/sx.h>
76 #include <sys/mutex.h>
77 #include <sys/rwlock.h>
78 #include <sys/priv.h>
79 #include <sys/proc.h>
80 #include <sys/mount.h>
81 #include <sys/vnode.h>
82 #include <sys/stat.h>
83 #include <sys/unistd.h>
84 #include <sys/filedesc.h>
85 #include <sys/file.h>
86 #include <sys/fcntl.h>
87 #include <sys/bio.h>
88 #include <sys/buf.h>
89 #include <sys/sysctl.h>
90 #include <sys/vmmeter.h>
91
92 #include <vm/vm.h>
93 #include <vm/vm_extern.h>
94 #include <vm/pmap.h>
95 #include <vm/vm_map.h>
96 #include <vm/vm_page.h>
97 #include <vm/vm_object.h>
98 #include <vm/vnode_pager.h>
99
100 #include "fuse.h"
101 #include "fuse_file.h"
102 #include "fuse_node.h"
103 #include "fuse_internal.h"
104 #include "fuse_ipc.h"
105 #include "fuse_io.h"
106
107 /*
108 * Set in a struct buf to indicate that the write came from the buffer cache
109 * and the originating cred and pid are no longer known.
110 */
111 #define B_FUSEFS_WRITE_CACHE B_FS_FLAG1
112
113 SDT_PROVIDER_DECLARE(fusefs);
114 /*
115 * Fuse trace probe:
116 * arg0: verbosity. Higher numbers give more verbose messages
117 * arg1: Textual message
118 */
119 SDT_PROBE_DEFINE2(fusefs, , io, trace, "int", "char*");
120
121 SDT_PROBE_DEFINE4(fusefs, , io, read_bio_backend_start, "int", "int", "int", "int");
122 SDT_PROBE_DEFINE2(fusefs, , io, read_bio_backend_feed, "int", "struct buf*");
123 SDT_PROBE_DEFINE4(fusefs, , io, read_bio_backend_end, "int", "ssize_t", "int",
124 "struct buf*");
125 int
fuse_read_biobackend(struct vnode * vp,struct uio * uio,int ioflag,struct ucred * cred,struct fuse_filehandle * fufh,pid_t pid)126 fuse_read_biobackend(struct vnode *vp, struct uio *uio, int ioflag,
127 struct ucred *cred, struct fuse_filehandle *fufh, pid_t pid)
128 {
129 struct buf *bp;
130 struct mount *mp;
131 struct fuse_data *data;
132 daddr_t lbn, nextlbn;
133 int bcount, nextsize;
134 int err, n = 0, on = 0, seqcount;
135 off_t filesize;
136
137 const int biosize = fuse_iosize(vp);
138 mp = vnode_mount(vp);
139 data = fuse_get_mpdata(mp);
140
141 if (uio->uio_offset < 0)
142 return (EINVAL);
143
144 seqcount = ioflag >> IO_SEQSHIFT;
145
146 err = fuse_vnode_size(vp, &filesize, cred, curthread);
147 if (err)
148 return err;
149
150 for (err = 0, bp = NULL; uio->uio_resid > 0; bp = NULL) {
151 if (fuse_isdeadfs(vp)) {
152 err = ENXIO;
153 break;
154 }
155 if (filesize - uio->uio_offset <= 0)
156 break;
157 lbn = uio->uio_offset / biosize;
158 on = uio->uio_offset & (biosize - 1);
159
160 if ((off_t)lbn * biosize >= filesize) {
161 bcount = 0;
162 } else if ((off_t)(lbn + 1) * biosize > filesize) {
163 bcount = filesize - (off_t)lbn *biosize;
164 } else {
165 bcount = biosize;
166 }
167 nextlbn = lbn + 1;
168 nextsize = MIN(biosize, filesize - nextlbn * biosize);
169
170 SDT_PROBE4(fusefs, , io, read_bio_backend_start,
171 biosize, (int)lbn, on, bcount);
172
173 if (bcount < biosize) {
174 /* If near EOF, don't do readahead */
175 err = bread(vp, lbn, bcount, NOCRED, &bp);
176 } else if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERR) == 0) {
177 /* Try clustered read */
178 long totread = uio->uio_resid + on;
179 seqcount = MIN(seqcount,
180 data->max_readahead_blocks + 1);
181 err = cluster_read(vp, filesize, lbn, bcount, NOCRED,
182 totread, seqcount, 0, &bp);
183 } else if (seqcount > 1 && data->max_readahead_blocks >= 1) {
184 /* Try non-clustered readahead */
185 err = breadn(vp, lbn, bcount, &nextlbn, &nextsize, 1,
186 NOCRED, &bp);
187 } else {
188 /* Just read what was requested */
189 err = bread(vp, lbn, bcount, NOCRED, &bp);
190 }
191
192 if (err) {
193 brelse(bp);
194 bp = NULL;
195 break;
196 }
197
198 /*
199 * on is the offset into the current bp. Figure out how many
200 * bytes we can copy out of the bp. Note that bcount is
201 * NOT DEV_BSIZE aligned.
202 *
203 * Then figure out how many bytes we can copy into the uio.
204 */
205
206 n = 0;
207 if (on < bcount - bp->b_resid)
208 n = MIN((unsigned)(bcount - bp->b_resid - on),
209 uio->uio_resid);
210 if (n > 0) {
211 SDT_PROBE2(fusefs, , io, read_bio_backend_feed, n, bp);
212 err = uiomove(bp->b_data + on, n, uio);
213 }
214 vfs_bio_brelse(bp, ioflag);
215 SDT_PROBE4(fusefs, , io, read_bio_backend_end, err,
216 uio->uio_resid, n, bp);
217 if (bp->b_resid > 0) {
218 /* Short read indicates EOF */
219 break;
220 }
221 }
222
223 return (err);
224 }
225
226 SDT_PROBE_DEFINE1(fusefs, , io, read_directbackend_start,
227 "struct fuse_read_in*");
228 SDT_PROBE_DEFINE3(fusefs, , io, read_directbackend_complete,
229 "struct fuse_dispatcher*", "struct fuse_read_in*", "struct uio*");
230
231 int
fuse_read_directbackend(struct vnode * vp,struct uio * uio,struct ucred * cred,struct fuse_filehandle * fufh)232 fuse_read_directbackend(struct vnode *vp, struct uio *uio,
233 struct ucred *cred, struct fuse_filehandle *fufh)
234 {
235 struct fuse_data *data;
236 struct fuse_dispatcher fdi;
237 struct fuse_read_in *fri;
238 int err = 0;
239
240 data = fuse_get_mpdata(vp->v_mount);
241
242 if (uio->uio_resid == 0)
243 return (0);
244
245 fdisp_init(&fdi, 0);
246
247 /*
248 * XXX In "normal" case we use an intermediate kernel buffer for
249 * transmitting data from daemon's context to ours. Eventually, we should
250 * get rid of this. Anyway, if the target uio lives in sysspace (we are
251 * called from pageops), and the input data doesn't need kernel-side
252 * processing (we are not called from readdir) we can already invoke
253 * an optimized, "peer-to-peer" I/O routine.
254 */
255 while (uio->uio_resid > 0) {
256 fdi.iosize = sizeof(*fri);
257 fdisp_make_vp(&fdi, FUSE_READ, vp, uio->uio_td, cred);
258 fri = fdi.indata;
259 fri->fh = fufh->fh_id;
260 fri->offset = uio->uio_offset;
261 fri->size = MIN(uio->uio_resid,
262 fuse_get_mpdata(vp->v_mount)->max_read);
263 if (fuse_libabi_geq(data, 7, 9)) {
264 /* See comment regarding FUSE_WRITE_LOCKOWNER */
265 fri->read_flags = 0;
266 fri->flags = fufh_type_2_fflags(fufh->fufh_type);
267 }
268
269 SDT_PROBE1(fusefs, , io, read_directbackend_start, fri);
270
271 if ((err = fdisp_wait_answ(&fdi)))
272 goto out;
273
274 SDT_PROBE3(fusefs, , io, read_directbackend_complete,
275 &fdi, fri, uio);
276
277 if ((err = uiomove(fdi.answ, MIN(fri->size, fdi.iosize), uio)))
278 break;
279 if (fdi.iosize < fri->size) {
280 /*
281 * Short read. Should only happen at EOF or with
282 * direct io.
283 */
284 break;
285 }
286 }
287
288 out:
289 fdisp_destroy(&fdi);
290 return (err);
291 }
292
293 int
fuse_write_directbackend(struct vnode * vp,struct uio * uio,struct ucred * cred,struct fuse_filehandle * fufh,off_t filesize,int ioflag,bool pages)294 fuse_write_directbackend(struct vnode *vp, struct uio *uio,
295 struct ucred *cred, struct fuse_filehandle *fufh, off_t filesize,
296 int ioflag, bool pages)
297 {
298 struct fuse_vnode_data *fvdat = VTOFUD(vp);
299 struct fuse_data *data;
300 struct fuse_write_in *fwi;
301 struct fuse_write_out *fwo;
302 struct fuse_dispatcher fdi;
303 size_t chunksize;
304 ssize_t r;
305 void *fwi_data;
306 off_t as_written_offset;
307 int diff;
308 int err = 0;
309 bool direct_io = fufh->fuse_open_flags & FOPEN_DIRECT_IO;
310 bool wrote_anything = false;
311 uint32_t write_flags;
312
313 data = fuse_get_mpdata(vp->v_mount);
314
315 /*
316 * Don't set FUSE_WRITE_LOCKOWNER in write_flags. It can't be set
317 * accurately when using POSIX AIO, libfuse doesn't use it, and I'm not
318 * aware of any file systems that do. It was an attempt to add
319 * Linux-style mandatory locking to the FUSE protocol, but mandatory
320 * locking is deprecated even on Linux. See Linux commit
321 * f33321141b273d60cbb3a8f56a5489baad82ba5e .
322 */
323 /*
324 * Set FUSE_WRITE_CACHE whenever we don't know the uid, gid, and/or pid
325 * that originated a write. For example when writing from the
326 * writeback cache. I don't know of a single file system that cares,
327 * but the protocol says we're supposed to do this.
328 */
329 write_flags = !pages && (
330 (ioflag & IO_DIRECT) ||
331 !fsess_opt_datacache(vnode_mount(vp)) ||
332 !fsess_opt_writeback(vnode_mount(vp))) ? 0 : FUSE_WRITE_CACHE;
333
334 if (uio->uio_resid == 0)
335 return (0);
336
337 if (ioflag & IO_APPEND)
338 uio_setoffset(uio, filesize);
339
340 err = vn_rlimit_fsizex(vp, uio, 0, &r, uio->uio_td);
341 if (err != 0) {
342 vn_rlimit_fsizex_res(uio, r);
343 return (err);
344 }
345
346 fdisp_init(&fdi, 0);
347
348 while (uio->uio_resid > 0) {
349 size_t sizeof_fwi;
350
351 if (fuse_libabi_geq(data, 7, 9)) {
352 sizeof_fwi = sizeof(*fwi);
353 } else {
354 sizeof_fwi = FUSE_COMPAT_WRITE_IN_SIZE;
355 }
356
357 chunksize = MIN(uio->uio_resid, data->max_write);
358
359 fdi.iosize = sizeof_fwi + chunksize;
360 fdisp_make_vp(&fdi, FUSE_WRITE, vp, uio->uio_td, cred);
361
362 fwi = fdi.indata;
363 fwi->fh = fufh->fh_id;
364 fwi->offset = uio->uio_offset;
365 fwi->size = chunksize;
366 fwi->write_flags = write_flags;
367 if (fuse_libabi_geq(data, 7, 9)) {
368 fwi->flags = fufh_type_2_fflags(fufh->fufh_type);
369 }
370 fwi_data = (char *)fdi.indata + sizeof_fwi;
371
372 if ((err = uiomove(fwi_data, chunksize, uio)))
373 break;
374
375 retry:
376 err = fdisp_wait_answ(&fdi);
377 if (err == ERESTART || err == EINTR || err == EWOULDBLOCK) {
378 /*
379 * Rewind the uio so dofilewrite will know it's
380 * incomplete
381 */
382 uio->uio_resid += fwi->size;
383 uio->uio_offset -= fwi->size;
384 /*
385 * Change ERESTART into EINTR because we can't rewind
386 * uio->uio_iov. Basically, once uiomove(9) has been
387 * called, it's impossible to restart a syscall.
388 */
389 if (err == ERESTART)
390 err = EINTR;
391 break;
392 } else if (err) {
393 break;
394 } else {
395 wrote_anything = true;
396 }
397
398 fwo = ((struct fuse_write_out *)fdi.answ);
399
400 if (fwo->size > fwi->size) {
401 fuse_warn(data, FSESS_WARN_WROTE_LONG,
402 "wrote more data than we provided it.");
403 /* This is bonkers. Clear attr cache. */
404 ASSERT_CACHED_ATTRS_LOCKED(vp);
405 fvdat->flag &= ~FN_SIZECHANGE;
406 fuse_vnode_clear_attr_cache(vp);
407 err = EINVAL;
408 break;
409 }
410
411 /* Adjust the uio in the case of short writes */
412 diff = fwi->size - fwo->size;
413
414 as_written_offset = uio->uio_offset - diff;
415
416 if (as_written_offset - diff > filesize) {
417 fuse_vnode_setsize(vp, as_written_offset, false);
418 getnanouptime(&fvdat->last_local_modify);
419 }
420 if (as_written_offset - diff >= filesize) {
421 ASSERT_CACHED_ATTRS_LOCKED(vp);
422 fvdat->flag &= ~FN_SIZECHANGE;
423 }
424
425 if (diff > 0) {
426 /* Short write */
427 if (!direct_io) {
428 fuse_warn(data, FSESS_WARN_SHORT_WRITE,
429 "short writes are only allowed with "
430 "direct_io.");
431 }
432 if (ioflag & IO_DIRECT) {
433 /* Return early */
434 uio->uio_resid += diff;
435 uio->uio_offset -= diff;
436 break;
437 } else {
438 /* Resend the unwritten portion of data */
439 fdi.iosize = sizeof_fwi + diff;
440 /* Refresh fdi without clearing data buffer */
441 fdisp_refresh_vp(&fdi, FUSE_WRITE, vp,
442 uio->uio_td, cred);
443 fwi = fdi.indata;
444 MPASS2(fwi == fdi.indata, "FUSE dispatcher "
445 "reallocated despite no increase in "
446 "size?");
447 void *src = (char*)fwi_data + fwo->size;
448 memmove(fwi_data, src, diff);
449 fwi->fh = fufh->fh_id;
450 fwi->offset = as_written_offset;
451 fwi->size = diff;
452 fwi->write_flags = write_flags;
453 goto retry;
454 }
455 }
456 }
457
458 fdisp_destroy(&fdi);
459
460 if (wrote_anything) {
461 CACHED_ATTR_LOCK(vp);
462 fuse_vnode_undirty_cached_timestamps(vp, false);
463 CACHED_ATTR_UNLOCK(vp);
464 }
465
466 vn_rlimit_fsizex_res(uio, r);
467 return (err);
468 }
469
470 SDT_PROBE_DEFINE6(fusefs, , io, write_biobackend_start, "int64_t", "int", "int",
471 "struct uio*", "int", "bool");
472 SDT_PROBE_DEFINE2(fusefs, , io, write_biobackend_append_race, "long", "int");
473 SDT_PROBE_DEFINE2(fusefs, , io, write_biobackend_issue, "int", "struct buf*");
474
475 int
fuse_write_biobackend(struct vnode * vp,struct uio * uio,struct ucred * cred,struct fuse_filehandle * fufh,int ioflag,pid_t pid)476 fuse_write_biobackend(struct vnode *vp, struct uio *uio,
477 struct ucred *cred, struct fuse_filehandle *fufh, int ioflag, pid_t pid)
478 {
479 struct fuse_vnode_data *fvdat = VTOFUD(vp);
480 struct buf *bp;
481 daddr_t lbn;
482 off_t filesize;
483 ssize_t r;
484 int bcount;
485 int n, on, seqcount, err = 0;
486
487 const int biosize = fuse_iosize(vp);
488
489 seqcount = ioflag >> IO_SEQSHIFT;
490
491 KASSERT(uio->uio_rw == UIO_WRITE, ("fuse_write_biobackend mode"));
492 if (vp->v_type != VREG)
493 return (EIO);
494 if (uio->uio_offset < 0)
495 return (EINVAL);
496 if (uio->uio_resid == 0)
497 return (0);
498
499 err = fuse_vnode_size(vp, &filesize, cred, curthread);
500 if (err)
501 return err;
502
503 if (ioflag & IO_APPEND)
504 uio_setoffset(uio, filesize);
505
506 err = vn_rlimit_fsizex(vp, uio, 0, &r, uio->uio_td);
507 if (err != 0) {
508 vn_rlimit_fsizex_res(uio, r);
509 return (err);
510 }
511
512 do {
513 bool direct_append, extending;
514
515 if (fuse_isdeadfs(vp)) {
516 err = ENXIO;
517 break;
518 }
519 lbn = uio->uio_offset / biosize;
520 on = uio->uio_offset & (biosize - 1);
521 n = MIN((unsigned)(biosize - on), uio->uio_resid);
522
523 again:
524 /* Get or create a buffer for the write */
525 direct_append = uio->uio_offset == filesize && n;
526 if (uio->uio_offset + n < filesize) {
527 extending = false;
528 if ((off_t)(lbn + 1) * biosize < filesize) {
529 /* Not the file's last block */
530 bcount = biosize;
531 } else {
532 /* The file's last block */
533 bcount = filesize - (off_t)lbn * biosize;
534 }
535 } else {
536 extending = true;
537 bcount = on + n;
538 }
539 if (direct_append) {
540 /*
541 * Take care to preserve the buffer's B_CACHE state so
542 * as not to cause an unnecessary read.
543 */
544 bp = getblk(vp, lbn, on, PCATCH, 0, 0);
545 if (bp != NULL) {
546 uint32_t save = bp->b_flags & B_CACHE;
547 allocbuf(bp, bcount);
548 bp->b_flags |= save;
549 }
550 } else {
551 bp = getblk(vp, lbn, bcount, PCATCH, 0, 0);
552 }
553 if (!bp) {
554 err = EINTR;
555 break;
556 }
557 if (extending) {
558 /*
559 * Extend file _after_ locking buffer so we won't race
560 * with other readers
561 */
562 err = fuse_vnode_setsize(vp, uio->uio_offset + n, false);
563 filesize = uio->uio_offset + n;
564 getnanouptime(&fvdat->last_local_modify);
565 ASSERT_CACHED_ATTRS_LOCKED(vp);
566 fvdat->flag |= FN_SIZECHANGE;
567 if (err) {
568 brelse(bp);
569 break;
570 }
571 }
572
573 SDT_PROBE6(fusefs, , io, write_biobackend_start,
574 lbn, on, n, uio, bcount, direct_append);
575 /*
576 * Issue a READ if B_CACHE is not set. In special-append
577 * mode, B_CACHE is based on the buffer prior to the write
578 * op and is typically set, avoiding the read. If a read
579 * is required in special append mode, the server will
580 * probably send us a short-read since we extended the file
581 * on our end, resulting in b_resid == 0 and, thusly,
582 * B_CACHE getting set.
583 *
584 * We can also avoid issuing the read if the write covers
585 * the entire buffer. We have to make sure the buffer state
586 * is reasonable in this case since we will not be initiating
587 * I/O. See the comments in kern/vfs_bio.c's getblk() for
588 * more information.
589 *
590 * B_CACHE may also be set due to the buffer being cached
591 * normally.
592 */
593
594 if (on == 0 && n == bcount) {
595 bp->b_flags |= B_CACHE;
596 bp->b_flags &= ~B_INVAL;
597 bp->b_ioflags &= ~BIO_ERROR;
598 }
599 if ((bp->b_flags & B_CACHE) == 0) {
600 bp->b_iocmd = BIO_READ;
601 vfs_busy_pages(bp, 0);
602 fuse_io_strategy(vp, bp);
603 if ((err = bp->b_error)) {
604 brelse(bp);
605 break;
606 }
607 if (bp->b_resid > 0) {
608 /*
609 * Short read indicates EOF. Update file size
610 * from the server and try again.
611 */
612 SDT_PROBE2(fusefs, , io, trace, 1,
613 "Short read during a RMW");
614 brelse(bp);
615 err = fuse_vnode_size(vp, &filesize, cred,
616 curthread);
617 if (err)
618 break;
619 else
620 goto again;
621 }
622 }
623 if (bp->b_wcred == NOCRED)
624 bp->b_wcred = crhold(cred);
625
626 /*
627 * If dirtyend exceeds file size, chop it down. This should
628 * not normally occur but there is an append race where it
629 * might occur XXX, so we log it.
630 *
631 * If the chopping creates a reverse-indexed or degenerate
632 * situation with dirtyoff/end, we 0 both of them.
633 */
634 if (bp->b_dirtyend > bcount) {
635 SDT_PROBE2(fusefs, , io, write_biobackend_append_race,
636 (long)bp->b_blkno * biosize,
637 bp->b_dirtyend - bcount);
638 bp->b_dirtyend = bcount;
639 }
640 if (bp->b_dirtyoff >= bp->b_dirtyend)
641 bp->b_dirtyoff = bp->b_dirtyend = 0;
642
643 /*
644 * If the new write will leave a contiguous dirty
645 * area, just update the b_dirtyoff and b_dirtyend,
646 * otherwise force a write rpc of the old dirty area.
647 *
648 * While it is possible to merge discontiguous writes due to
649 * our having a B_CACHE buffer ( and thus valid read data
650 * for the hole), we don't because it could lead to
651 * significant cache coherency problems with multiple clients,
652 * especially if locking is implemented later on.
653 *
654 * as an optimization we could theoretically maintain
655 * a linked list of discontinuous areas, but we would still
656 * have to commit them separately so there isn't much
657 * advantage to it except perhaps a bit of asynchronization.
658 */
659
660 if (bp->b_dirtyend > 0 &&
661 (on > bp->b_dirtyend || (on + n) < bp->b_dirtyoff)) {
662 /*
663 * Yes, we mean it. Write out everything to "storage"
664 * immediately, without hesitation. (Apart from other
665 * reasons: the only way to know if a write is valid
666 * if its actually written out.)
667 */
668 SDT_PROBE2(fusefs, , io, write_biobackend_issue, 0, bp);
669 bwrite(bp);
670 if (bp->b_error == EINTR) {
671 err = EINTR;
672 break;
673 }
674 goto again;
675 }
676 err = uiomove((char *)bp->b_data + on, n, uio);
677
678 if (err) {
679 bp->b_ioflags |= BIO_ERROR;
680 bp->b_error = err;
681 brelse(bp);
682 break;
683 /* TODO: vfs_bio_clrbuf like ffs_write does? */
684 }
685 /*
686 * Only update dirtyoff/dirtyend if not a degenerate
687 * condition.
688 */
689 if (n) {
690 if (bp->b_dirtyend > 0) {
691 bp->b_dirtyoff = MIN(on, bp->b_dirtyoff);
692 bp->b_dirtyend = MAX((on + n), bp->b_dirtyend);
693 } else {
694 bp->b_dirtyoff = on;
695 bp->b_dirtyend = on + n;
696 }
697 vfs_bio_set_valid(bp, on, n);
698 }
699
700 vfs_bio_set_flags(bp, ioflag);
701
702 bp->b_flags |= B_FUSEFS_WRITE_CACHE;
703 if (ioflag & IO_SYNC) {
704 SDT_PROBE2(fusefs, , io, write_biobackend_issue, 2, bp);
705 if (!(ioflag & IO_VMIO))
706 bp->b_flags &= ~B_FUSEFS_WRITE_CACHE;
707 err = bwrite(bp);
708 } else if (vm_page_count_severe() ||
709 buf_dirty_count_severe() ||
710 (ioflag & IO_ASYNC)) {
711 bp->b_flags |= B_CLUSTEROK;
712 SDT_PROBE2(fusefs, , io, write_biobackend_issue, 3, bp);
713 bawrite(bp);
714 } else if (on == 0 && n == bcount) {
715 if ((vp->v_mount->mnt_flag & MNT_NOCLUSTERW) == 0) {
716 bp->b_flags |= B_CLUSTEROK;
717 SDT_PROBE2(fusefs, , io, write_biobackend_issue,
718 4, bp);
719 cluster_write(vp, &fvdat->clusterw, bp,
720 filesize, seqcount, 0);
721 } else {
722 SDT_PROBE2(fusefs, , io, write_biobackend_issue,
723 5, bp);
724 bawrite(bp);
725 }
726 } else if (ioflag & IO_DIRECT) {
727 bp->b_flags |= B_CLUSTEROK;
728 SDT_PROBE2(fusefs, , io, write_biobackend_issue, 6, bp);
729 bawrite(bp);
730 } else {
731 bp->b_flags &= ~B_CLUSTEROK;
732 SDT_PROBE2(fusefs, , io, write_biobackend_issue, 7, bp);
733 bdwrite(bp);
734 }
735 if (err)
736 break;
737 } while (uio->uio_resid > 0 && n > 0);
738
739 vn_rlimit_fsizex_res(uio, r);
740 return (err);
741 }
742
743 int
fuse_io_strategy(struct vnode * vp,struct buf * bp)744 fuse_io_strategy(struct vnode *vp, struct buf *bp)
745 {
746 struct fuse_vnode_data *fvdat = VTOFUD(vp);
747 struct fuse_filehandle *fufh;
748 struct ucred *cred;
749 struct uio *uiop;
750 struct uio uio;
751 struct iovec io;
752 off_t filesize;
753 int error = 0;
754 int fflag;
755 /* We don't know the true pid when we're dealing with the cache */
756 pid_t pid = 0;
757
758 const int biosize = fuse_iosize(vp);
759
760 MPASS(vp->v_type == VREG || vp->v_type == VDIR);
761 MPASS(bp->b_iocmd == BIO_READ || bp->b_iocmd == BIO_WRITE);
762
763 fflag = bp->b_iocmd == BIO_READ ? FREAD : FWRITE;
764 cred = bp->b_iocmd == BIO_READ ? bp->b_rcred : bp->b_wcred;
765 error = fuse_filehandle_getrw(vp, fflag, &fufh, cred, pid);
766 if (bp->b_iocmd == BIO_READ && error == EBADF) {
767 /*
768 * This may be a read-modify-write operation on a cached file
769 * opened O_WRONLY. The FUSE protocol allows this.
770 */
771 error = fuse_filehandle_get(vp, FWRITE, &fufh, cred, pid);
772 }
773 if (error) {
774 printf("FUSE: strategy: filehandles are closed\n");
775 bp->b_ioflags |= BIO_ERROR;
776 bp->b_error = error;
777 bufdone(bp);
778 return (error);
779 }
780
781 uiop = &uio;
782 uiop->uio_iov = &io;
783 uiop->uio_iovcnt = 1;
784 uiop->uio_segflg = UIO_SYSSPACE;
785 uiop->uio_td = curthread;
786
787 /*
788 * clear BIO_ERROR and B_INVAL state prior to initiating the I/O. We
789 * do this here so we do not have to do it in all the code that
790 * calls us.
791 */
792 bp->b_flags &= ~B_INVAL;
793 bp->b_ioflags &= ~BIO_ERROR;
794
795 KASSERT(!(bp->b_flags & B_DONE),
796 ("fuse_io_strategy: bp %p already marked done", bp));
797 if (bp->b_iocmd == BIO_READ) {
798 ssize_t left;
799
800 io.iov_len = uiop->uio_resid = bp->b_bcount;
801 io.iov_base = bp->b_data;
802 uiop->uio_rw = UIO_READ;
803
804 uiop->uio_offset = ((off_t)bp->b_lblkno) * biosize;
805 error = fuse_read_directbackend(vp, uiop, cred, fufh);
806 /*
807 * Store the amount we failed to read in the buffer's private
808 * field, so callers can truncate the file if necessary'
809 */
810
811 if (!error && uiop->uio_resid) {
812 int nread = bp->b_bcount - uiop->uio_resid;
813 left = uiop->uio_resid;
814 bzero((char *)bp->b_data + nread, left);
815
816 CACHED_ATTR_LOCK(vp);
817 if ((fvdat->flag & FN_SIZECHANGE) == 0) {
818 /*
819 * A short read with no error, when not using
820 * direct io, and when no writes are cached,
821 * indicates EOF caused by a server-side
822 * truncation. Clear the attr cache so we'll
823 * pick up the new file size and timestamps.
824 *
825 * We must still bzero the remaining buffer so
826 * uninitialized data doesn't get exposed by a
827 * future truncate that extends the file.
828 *
829 * To prevent lock order problems, we must
830 * truncate the file upstack, not here.
831 */
832 SDT_PROBE2(fusefs, , io, trace, 1,
833 "Short read of a clean file");
834 fuse_vnode_clear_attr_cache(vp);
835 } else {
836 /*
837 * If dirty writes _are_ cached beyond EOF,
838 * that indicates a newly created hole that the
839 * server doesn't know about. Those don't pose
840 * any problem.
841 * XXX: we don't currently track whether dirty
842 * writes are cached beyond EOF, before EOF, or
843 * both.
844 */
845 SDT_PROBE2(fusefs, , io, trace, 1,
846 "Short read of a dirty file");
847 uiop->uio_resid = 0;
848 }
849 CACHED_ATTR_UNLOCK(vp);
850 }
851 if (error) {
852 bp->b_ioflags |= BIO_ERROR;
853 bp->b_error = error;
854 }
855 } else {
856 /*
857 * Setup for actual write
858 */
859 /*
860 * If the file's size is cached, use that value, even if the
861 * cache is expired. At this point we're already committed to
862 * writing something. If the FUSE server has changed the
863 * file's size behind our back, it's too late for us to do
864 * anything about it. In particular, we can't invalidate any
865 * part of the file's buffers because VOP_STRATEGY is called
866 * with them already locked.
867 *
868 * Normally the vnode should be exclusively locked at this
869 * point. However, if clustered reads are in use, then in a
870 * mixed read-write workload getblkx may need to flush a
871 * partially written buffer to disk during a read. In such a
872 * case, the vnode may only have a shared lock at this point.
873 */
874 CACHED_ATTR_LOCK(vp);
875 filesize = fvdat->cached_attrs.va_size;
876 /* filesize must've been cached by fuse_vnop_open. */
877 KASSERT(filesize != VNOVAL, ("filesize should've been cached"));
878 CACHED_ATTR_UNLOCK(vp);
879
880 if ((off_t)bp->b_lblkno * biosize + bp->b_dirtyend > filesize)
881 bp->b_dirtyend = filesize -
882 (off_t)bp->b_lblkno * biosize;
883
884 if (bp->b_dirtyend > bp->b_dirtyoff) {
885 io.iov_len = uiop->uio_resid = bp->b_dirtyend
886 - bp->b_dirtyoff;
887 uiop->uio_offset = (off_t)bp->b_lblkno * biosize
888 + bp->b_dirtyoff;
889 io.iov_base = (char *)bp->b_data + bp->b_dirtyoff;
890 uiop->uio_rw = UIO_WRITE;
891
892 bool pages = bp->b_flags & B_FUSEFS_WRITE_CACHE;
893 error = fuse_write_directbackend(vp, uiop, cred, fufh,
894 filesize, 0, pages);
895
896 if (error == EINTR || error == ETIMEDOUT) {
897 bp->b_flags &= ~(B_INVAL | B_NOCACHE);
898 if ((bp->b_flags & B_PAGING) == 0) {
899 bdirty(bp);
900 bp->b_flags &= ~B_DONE;
901 }
902 if ((error == EINTR || error == ETIMEDOUT) &&
903 (bp->b_flags & B_ASYNC) == 0)
904 bp->b_flags |= B_EINTR;
905 } else {
906 if (error) {
907 bp->b_ioflags |= BIO_ERROR;
908 bp->b_flags |= B_INVAL;
909 bp->b_error = error;
910 }
911 bp->b_dirtyoff = bp->b_dirtyend = 0;
912 }
913 } else {
914 bp->b_resid = 0;
915 bufdone(bp);
916 return (0);
917 }
918 }
919 bp->b_resid = uiop->uio_resid;
920 bufdone(bp);
921 return (error);
922 }
923
924 int
fuse_io_flushbuf(struct vnode * vp,int waitfor,struct thread * td)925 fuse_io_flushbuf(struct vnode *vp, int waitfor, struct thread *td)
926 {
927
928 return (vn_fsync_buf(vp, waitfor));
929 }
930
931 /*
932 * Flush and invalidate all dirty buffers. If another process is already
933 * doing the flush, just wait for completion.
934 */
935 int
fuse_io_invalbuf(struct vnode * vp,struct thread * td)936 fuse_io_invalbuf(struct vnode *vp, struct thread *td)
937 {
938 struct fuse_vnode_data *fvdat = VTOFUD(vp);
939 int error = 0;
940
941 if (VN_IS_DOOMED(vp))
942 return 0;
943
944 ASSERT_VOP_ELOCKED(vp, "fuse_io_invalbuf");
945
946 while (fvdat->flag & FN_FLUSHINPROG) {
947 struct proc *p = td->td_proc;
948
949 if (vp->v_mount->mnt_kern_flag & MNTK_UNMOUNTF)
950 return EIO;
951 fvdat->flag |= FN_FLUSHWANT;
952 tsleep(&fvdat->flag, PRIBIO, "fusevinv", 2 * hz);
953 error = 0;
954 if (p != NULL) {
955 PROC_LOCK(p);
956 if (SIGNOTEMPTY(p->p_siglist) ||
957 SIGNOTEMPTY(td->td_siglist))
958 error = EINTR;
959 PROC_UNLOCK(p);
960 }
961 if (error == EINTR)
962 return EINTR;
963 }
964 fvdat->flag |= FN_FLUSHINPROG;
965
966 vnode_pager_clean_sync(vp);
967 error = vinvalbuf(vp, V_SAVE, PCATCH, 0);
968 while (error) {
969 if (error == ERESTART || error == EINTR) {
970 fvdat->flag &= ~FN_FLUSHINPROG;
971 if (fvdat->flag & FN_FLUSHWANT) {
972 fvdat->flag &= ~FN_FLUSHWANT;
973 wakeup(&fvdat->flag);
974 }
975 return EINTR;
976 }
977 error = vinvalbuf(vp, V_SAVE, PCATCH, 0);
978 }
979 fvdat->flag &= ~FN_FLUSHINPROG;
980 if (fvdat->flag & FN_FLUSHWANT) {
981 fvdat->flag &= ~FN_FLUSHWANT;
982 wakeup(&fvdat->flag);
983 }
984 return (error);
985 }
986