Lines Matching full:cs
819 void fuse_copy_init(struct fuse_copy_state *cs, int write, in fuse_copy_init() argument
822 memset(cs, 0, sizeof(*cs)); in fuse_copy_init()
823 cs->write = write; in fuse_copy_init()
824 cs->iter = iter; in fuse_copy_init()
828 static void fuse_copy_finish(struct fuse_copy_state *cs) in fuse_copy_finish() argument
830 if (cs->currbuf) { in fuse_copy_finish()
831 struct pipe_buffer *buf = cs->currbuf; in fuse_copy_finish()
833 if (cs->write) in fuse_copy_finish()
834 buf->len = PAGE_SIZE - cs->len; in fuse_copy_finish()
835 cs->currbuf = NULL; in fuse_copy_finish()
836 } else if (cs->pg) { in fuse_copy_finish()
837 if (cs->write) { in fuse_copy_finish()
838 flush_dcache_page(cs->pg); in fuse_copy_finish()
839 set_page_dirty_lock(cs->pg); in fuse_copy_finish()
841 put_page(cs->pg); in fuse_copy_finish()
843 cs->pg = NULL; in fuse_copy_finish()
850 static int fuse_copy_fill(struct fuse_copy_state *cs) in fuse_copy_fill() argument
855 err = unlock_request(cs->req); in fuse_copy_fill()
859 fuse_copy_finish(cs); in fuse_copy_fill()
860 if (cs->pipebufs) { in fuse_copy_fill()
861 struct pipe_buffer *buf = cs->pipebufs; in fuse_copy_fill()
863 if (!cs->write) { in fuse_copy_fill()
864 err = pipe_buf_confirm(cs->pipe, buf); in fuse_copy_fill()
868 BUG_ON(!cs->nr_segs); in fuse_copy_fill()
869 cs->currbuf = buf; in fuse_copy_fill()
870 cs->pg = buf->page; in fuse_copy_fill()
871 cs->offset = buf->offset; in fuse_copy_fill()
872 cs->len = buf->len; in fuse_copy_fill()
873 cs->pipebufs++; in fuse_copy_fill()
874 cs->nr_segs--; in fuse_copy_fill()
876 if (cs->nr_segs >= cs->pipe->max_usage) in fuse_copy_fill()
887 cs->currbuf = buf; in fuse_copy_fill()
888 cs->pg = page; in fuse_copy_fill()
889 cs->offset = 0; in fuse_copy_fill()
890 cs->len = PAGE_SIZE; in fuse_copy_fill()
891 cs->pipebufs++; in fuse_copy_fill()
892 cs->nr_segs++; in fuse_copy_fill()
896 err = iov_iter_get_pages2(cs->iter, &page, PAGE_SIZE, 1, &off); in fuse_copy_fill()
900 cs->len = err; in fuse_copy_fill()
901 cs->offset = off; in fuse_copy_fill()
902 cs->pg = page; in fuse_copy_fill()
905 return lock_request(cs->req); in fuse_copy_fill()
909 static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size) in fuse_copy_do() argument
911 unsigned ncpy = min(*size, cs->len); in fuse_copy_do()
913 void *pgaddr = kmap_local_page(cs->pg); in fuse_copy_do()
914 void *buf = pgaddr + cs->offset; in fuse_copy_do()
916 if (cs->write) in fuse_copy_do()
925 cs->len -= ncpy; in fuse_copy_do()
926 cs->offset += ncpy; in fuse_copy_do()
927 if (cs->is_uring) in fuse_copy_do()
928 cs->ring.copied_sz += ncpy; in fuse_copy_do()
958 static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep) in fuse_try_move_page() argument
963 struct pipe_buffer *buf = cs->pipebufs; in fuse_try_move_page()
966 err = unlock_request(cs->req); in fuse_try_move_page()
970 fuse_copy_finish(cs); in fuse_try_move_page()
972 err = pipe_buf_confirm(cs->pipe, buf); in fuse_try_move_page()
976 BUG_ON(!cs->nr_segs); in fuse_try_move_page()
977 cs->currbuf = buf; in fuse_try_move_page()
978 cs->len = buf->len; in fuse_try_move_page()
979 cs->pipebufs++; in fuse_try_move_page()
980 cs->nr_segs--; in fuse_try_move_page()
982 if (cs->len != PAGE_SIZE) in fuse_try_move_page()
985 if (!pipe_buf_try_steal(cs->pipe, buf)) in fuse_try_move_page()
1021 pipe_buf_release(cs->pipe, buf); in fuse_try_move_page()
1024 spin_lock(&cs->req->waitq.lock); in fuse_try_move_page()
1025 if (test_bit(FR_ABORTED, &cs->req->flags)) in fuse_try_move_page()
1029 spin_unlock(&cs->req->waitq.lock); in fuse_try_move_page()
1040 cs->len = 0; in fuse_try_move_page()
1051 cs->pg = buf->page; in fuse_try_move_page()
1052 cs->offset = buf->offset; in fuse_try_move_page()
1054 err = lock_request(cs->req); in fuse_try_move_page()
1061 static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page, in fuse_ref_page() argument
1067 if (cs->nr_segs >= cs->pipe->max_usage) in fuse_ref_page()
1071 err = unlock_request(cs->req); in fuse_ref_page()
1077 fuse_copy_finish(cs); in fuse_ref_page()
1079 buf = cs->pipebufs; in fuse_ref_page()
1084 cs->pipebufs++; in fuse_ref_page()
1085 cs->nr_segs++; in fuse_ref_page()
1086 cs->len = 0; in fuse_ref_page()
1095 static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep, in fuse_copy_page() argument
1105 if (cs->write && cs->pipebufs && page) { in fuse_copy_page()
1110 if (cs->req->args->user_pages) { in fuse_copy_page()
1111 err = fuse_copy_fill(cs); in fuse_copy_page()
1115 return fuse_ref_page(cs, page, offset, count); in fuse_copy_page()
1117 } else if (!cs->len) { in fuse_copy_page()
1118 if (cs->move_pages && page && in fuse_copy_page()
1120 err = fuse_try_move_page(cs, pagep); in fuse_copy_page()
1124 err = fuse_copy_fill(cs); in fuse_copy_page()
1132 offset += fuse_copy_do(cs, &buf, &count); in fuse_copy_page()
1135 offset += fuse_copy_do(cs, NULL, &count); in fuse_copy_page()
1137 if (page && !cs->write) in fuse_copy_page()
1143 static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes, in fuse_copy_pages() argument
1147 struct fuse_req *req = cs->req; in fuse_copy_pages()
1158 err = fuse_copy_page(cs, &pagep, offset, count, zeroing); in fuse_copy_pages()
1176 static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size) in fuse_copy_one() argument
1179 if (!cs->len) { in fuse_copy_one()
1180 int err = fuse_copy_fill(cs); in fuse_copy_one()
1184 fuse_copy_do(cs, &val, &size); in fuse_copy_one()
1190 int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs, in fuse_copy_args() argument
1200 err = fuse_copy_pages(cs, arg->size, zeroing); in fuse_copy_args()
1202 err = fuse_copy_one(cs, arg->value, arg->size); in fuse_copy_args()
1227 struct fuse_copy_state *cs, in fuse_read_interrupt() argument
1248 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_interrupt()
1250 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_interrupt()
1251 fuse_copy_finish(cs); in fuse_read_interrupt()
1279 struct fuse_copy_state *cs, in fuse_read_single_forget() argument
1300 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_single_forget()
1302 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_single_forget()
1303 fuse_copy_finish(cs); in fuse_read_single_forget()
1312 struct fuse_copy_state *cs, size_t nbytes) in fuse_read_batch_forget() argument
1337 err = fuse_copy_one(cs, &ih, sizeof(ih)); in fuse_read_batch_forget()
1339 err = fuse_copy_one(cs, &arg, sizeof(arg)); in fuse_read_batch_forget()
1345 err = fuse_copy_one(cs, &forget->forget_one, in fuse_read_batch_forget()
1352 fuse_copy_finish(cs); in fuse_read_batch_forget()
1361 struct fuse_copy_state *cs, in fuse_read_forget() argument
1366 return fuse_read_single_forget(fiq, cs, nbytes); in fuse_read_forget()
1368 return fuse_read_batch_forget(fiq, cs, nbytes); in fuse_read_forget()
1381 struct fuse_copy_state *cs, size_t nbytes) in fuse_dev_do_read() argument
1433 return fuse_read_interrupt(fiq, cs, nbytes, req); in fuse_dev_do_read()
1438 return fuse_read_forget(fc, fiq, cs, nbytes); in fuse_dev_do_read()
1473 cs->req = req; in fuse_dev_do_read()
1474 err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h)); in fuse_dev_do_read()
1476 err = fuse_copy_args(cs, args->in_numargs, args->in_pages, in fuse_dev_do_read()
1478 fuse_copy_finish(cs); in fuse_dev_do_read()
1531 struct fuse_copy_state cs; in fuse_dev_read() local
1541 fuse_copy_init(&cs, 1, to); in fuse_dev_read()
1543 return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to)); in fuse_dev_read()
1553 struct fuse_copy_state cs; in fuse_dev_splice_read() local
1564 fuse_copy_init(&cs, 1, NULL); in fuse_dev_splice_read()
1565 cs.pipebufs = bufs; in fuse_dev_splice_read()
1566 cs.pipe = pipe; in fuse_dev_splice_read()
1567 ret = fuse_dev_do_read(fud, in, &cs, len); in fuse_dev_splice_read()
1571 if (pipe_buf_usage(pipe) + cs.nr_segs > pipe->max_usage) { in fuse_dev_splice_read()
1576 for (ret = total = 0; page_nr < cs.nr_segs; total += ret) { in fuse_dev_splice_read()
1590 for (; page_nr < cs.nr_segs; page_nr++) in fuse_dev_splice_read()
1598 struct fuse_copy_state *cs) in fuse_notify_poll() argument
1606 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_poll()
1610 fuse_copy_finish(cs); in fuse_notify_poll()
1614 fuse_copy_finish(cs); in fuse_notify_poll()
1619 struct fuse_copy_state *cs) in fuse_notify_inval_inode() argument
1627 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_inval_inode()
1630 fuse_copy_finish(cs); in fuse_notify_inval_inode()
1639 fuse_copy_finish(cs); in fuse_notify_inval_inode()
1644 struct fuse_copy_state *cs) in fuse_notify_inval_entry() argument
1655 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_inval_entry()
1674 err = fuse_copy_one(cs, buf, outarg.namelen + 1); in fuse_notify_inval_entry()
1677 fuse_copy_finish(cs); in fuse_notify_inval_entry()
1688 fuse_copy_finish(cs); in fuse_notify_inval_entry()
1693 struct fuse_copy_state *cs) in fuse_notify_delete() argument
1704 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_delete()
1723 err = fuse_copy_one(cs, buf, outarg.namelen + 1); in fuse_notify_delete()
1726 fuse_copy_finish(cs); in fuse_notify_delete()
1737 fuse_copy_finish(cs); in fuse_notify_delete()
1742 struct fuse_copy_state *cs) in fuse_notify_store() argument
1759 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_store()
1799 err = fuse_copy_page(cs, &page, offset, this_num, 0); in fuse_notify_store()
1823 fuse_copy_finish(cs); in fuse_notify_store()
1925 struct fuse_copy_state *cs) in fuse_notify_retrieve() argument
1937 err = fuse_copy_one(cs, &outarg, sizeof(outarg)); in fuse_notify_retrieve()
1941 fuse_copy_finish(cs); in fuse_notify_retrieve()
1957 fuse_copy_finish(cs); in fuse_notify_retrieve()
2025 unsigned int size, struct fuse_copy_state *cs) in fuse_notify() argument
2028 cs->move_pages = 0; in fuse_notify()
2032 return fuse_notify_poll(fc, size, cs); in fuse_notify()
2035 return fuse_notify_inval_inode(fc, size, cs); in fuse_notify()
2038 return fuse_notify_inval_entry(fc, size, cs); in fuse_notify()
2041 return fuse_notify_store(fc, size, cs); in fuse_notify()
2044 return fuse_notify_retrieve(fc, size, cs); in fuse_notify()
2047 return fuse_notify_delete(fc, size, cs); in fuse_notify()
2053 fuse_copy_finish(cs); in fuse_notify()
2071 int fuse_copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args, in fuse_copy_out_args() argument
2080 if (!cs->is_uring) in fuse_copy_out_args()
2095 return fuse_copy_args(cs, args->out_numargs, args->out_pages, in fuse_copy_out_args()
2107 struct fuse_copy_state *cs, size_t nbytes) in fuse_dev_do_write() argument
2119 err = fuse_copy_one(cs, &oh, sizeof(oh)); in fuse_dev_do_write()
2132 err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs); in fuse_dev_do_write()
2174 cs->req = req; in fuse_dev_do_write()
2176 cs->move_pages = 0; in fuse_dev_do_write()
2181 err = fuse_copy_out_args(cs, req->args, nbytes); in fuse_dev_do_write()
2182 fuse_copy_finish(cs); in fuse_dev_do_write()
2199 fuse_copy_finish(cs); in fuse_dev_do_write()
2205 struct fuse_copy_state cs; in fuse_dev_write() local
2214 fuse_copy_init(&cs, 0, from); in fuse_dev_write()
2216 return fuse_dev_do_write(fud, &cs, iov_iter_count(from)); in fuse_dev_write()
2227 struct fuse_copy_state cs; in fuse_dev_splice_write() local
2288 fuse_copy_init(&cs, 0, NULL); in fuse_dev_splice_write()
2289 cs.pipebufs = bufs; in fuse_dev_splice_write()
2290 cs.nr_segs = nbuf; in fuse_dev_splice_write()
2291 cs.pipe = pipe; in fuse_dev_splice_write()
2294 cs.move_pages = 1; in fuse_dev_splice_write()
2296 ret = fuse_dev_do_write(fud, &cs, len); in fuse_dev_splice_write()