Lines Matching +full:data +full:- +full:mirror
1 // SPDX-License-Identifier: GPL-2.0-only
37 if (desc->pg_ops->pg_get_mirror) in nfs_pgio_get_mirror()
38 return desc->pg_ops->pg_get_mirror(desc, idx); in nfs_pgio_get_mirror()
39 return &desc->pg_mirrors[0]; in nfs_pgio_get_mirror()
45 return nfs_pgio_get_mirror(desc, desc->pg_mirror_idx); in nfs_pgio_current_mirror()
52 if (desc->pg_ops->pg_set_mirror) in nfs_pgio_set_current_mirror()
53 return desc->pg_ops->pg_set_mirror(desc, idx); in nfs_pgio_set_current_mirror()
54 return desc->pg_mirror_idx; in nfs_pgio_set_current_mirror()
61 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_pgheader_init() local
64 hdr->req = nfs_list_entry(mirror->pg_list.next); in nfs_pgheader_init()
65 hdr->inode = desc->pg_inode; in nfs_pgheader_init()
66 hdr->cred = nfs_req_openctx(hdr->req)->cred; in nfs_pgheader_init()
67 hdr->io_start = req_offset(hdr->req); in nfs_pgheader_init()
68 hdr->good_bytes = mirror->pg_count; in nfs_pgheader_init()
69 hdr->io_completion = desc->pg_io_completion; in nfs_pgheader_init()
70 hdr->dreq = desc->pg_dreq; in nfs_pgheader_init()
71 hdr->release = release; in nfs_pgheader_init()
72 hdr->completion_ops = desc->pg_completion_ops; in nfs_pgheader_init()
73 if (hdr->completion_ops->init_hdr) in nfs_pgheader_init()
74 hdr->completion_ops->init_hdr(hdr); in nfs_pgheader_init()
76 hdr->pgio_mirror_idx = desc->pg_mirror_idx; in nfs_pgheader_init()
82 unsigned int new = pos - hdr->io_start; in nfs_set_pgio_error()
85 if (hdr->good_bytes > new) { in nfs_set_pgio_error()
86 hdr->good_bytes = new; in nfs_set_pgio_error()
87 clear_bit(NFS_IOHDR_EOF, &hdr->flags); in nfs_set_pgio_error()
88 if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags)) in nfs_set_pgio_error()
89 hdr->error = error; in nfs_set_pgio_error()
98 INIT_LIST_HEAD(&p->wb_list); in nfs_page_alloc()
109 * nfs_iocounter_wait - wait for i/o to complete
112 * returns -ERESTARTSYS if interrupted by a fatal signal.
118 return wait_var_event_killable(&l_ctx->io_count, in nfs_iocounter_wait()
119 !atomic_read(&l_ctx->io_count)); in nfs_iocounter_wait()
123 * nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O
134 struct inode *inode = d_inode(l_ctx->open_context->dentry); in nfs_async_iocounter_wait()
137 if (atomic_read(&l_ctx->io_count) > 0) { in nfs_async_iocounter_wait()
138 rpc_sleep_on(&NFS_SERVER(inode)->uoc_rpcwaitq, task, NULL); in nfs_async_iocounter_wait()
142 if (atomic_read(&l_ctx->io_count) == 0) { in nfs_async_iocounter_wait()
143 rpc_wake_up_queued_task(&NFS_SERVER(inode)->uoc_rpcwaitq, task); in nfs_async_iocounter_wait()
152 * nfs_page_lock_head_request - page lock the head of the page group
158 struct nfs_page *head = req->wb_head; in nfs_page_group_lock_head()
166 kref_get(&head->wb_kref); in nfs_page_group_lock_head()
171 * nfs_unroll_locks - unlock all newly locked reqs and wait on @req
184 for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) { in nfs_unroll_locks()
185 if (!kref_read(&tmp->wb_kref)) in nfs_unroll_locks()
192 * nfs_page_group_lock_subreq - try to lock a subrequest
205 if (!kref_get_unless_zero(&subreq->wb_kref)) in nfs_page_group_lock_subreq()
222 * nfs_page_group_lock_subrequests - try to lock the subrequests
237 for (subreq = head->wb_this_page; subreq != head; in nfs_page_group_lock_subrequests()
238 subreq = subreq->wb_this_page) { in nfs_page_group_lock_subrequests()
248 * nfs_page_set_headlock - set the request PG_HEADLOCK
251 * this lock must be held when modifying req->wb_head
258 if (!test_and_set_bit(PG_HEADLOCK, &req->wb_flags)) in nfs_page_set_headlock()
261 set_bit(PG_CONTENDED1, &req->wb_flags); in nfs_page_set_headlock()
263 return wait_on_bit_lock(&req->wb_flags, PG_HEADLOCK, in nfs_page_set_headlock()
268 * nfs_page_clear_headlock - clear the request PG_HEADLOCK
275 clear_bit(PG_HEADLOCK, &req->wb_flags); in nfs_page_clear_headlock()
277 if (!test_bit(PG_CONTENDED1, &req->wb_flags)) in nfs_page_clear_headlock()
279 wake_up_bit(&req->wb_flags, PG_HEADLOCK); in nfs_page_clear_headlock()
283 * nfs_page_group_lock - lock the head of the page group
297 if (ret || req->wb_head == req) in nfs_page_group_lock()
299 return nfs_page_set_headlock(req->wb_head); in nfs_page_group_lock()
303 * nfs_page_group_unlock - unlock the head of the page group
309 if (req != req->wb_head) in nfs_page_group_unlock()
310 nfs_page_clear_headlock(req->wb_head); in nfs_page_group_unlock()
322 struct nfs_page *head = req->wb_head; in nfs_page_group_sync_on_bit_locked()
325 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags)); in nfs_page_group_sync_on_bit_locked()
326 WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags)); in nfs_page_group_sync_on_bit_locked()
328 tmp = req->wb_this_page; in nfs_page_group_sync_on_bit_locked()
330 if (!test_bit(bit, &tmp->wb_flags)) in nfs_page_group_sync_on_bit_locked()
332 tmp = tmp->wb_this_page; in nfs_page_group_sync_on_bit_locked()
338 clear_bit(bit, &tmp->wb_flags); in nfs_page_group_sync_on_bit_locked()
339 tmp = tmp->wb_this_page; in nfs_page_group_sync_on_bit_locked()
346 * nfs_page_group_sync_on_bit - set bit on current request, but only
348 * @req - request in page group
349 * @bit - PG_* bit that is used to sync page group
363 * nfs_page_group_init - Initialize the page group linkage for @req
364 * @req - a new nfs request
365 * @prev - the previous request in page group, or NULL if @req is the first
376 req->wb_head = req; in nfs_page_group_init()
377 req->wb_this_page = req; in nfs_page_group_init()
380 WARN_ON_ONCE(prev->wb_this_page != prev->wb_head); in nfs_page_group_init()
381 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags)); in nfs_page_group_init()
382 req->wb_head = prev->wb_head; in nfs_page_group_init()
383 req->wb_this_page = prev->wb_this_page; in nfs_page_group_init()
384 prev->wb_this_page = req; in nfs_page_group_init()
388 kref_get(&req->wb_head->wb_kref); in nfs_page_group_init()
393 if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) { in nfs_page_group_init()
394 inode = page_file_mapping(req->wb_page)->host; in nfs_page_group_init()
395 set_bit(PG_INODE_REF, &req->wb_flags); in nfs_page_group_init()
396 kref_get(&req->wb_kref); in nfs_page_group_init()
397 atomic_long_inc(&NFS_I(inode)->nrequests); in nfs_page_group_init()
403 * nfs_page_group_destroy - sync the destruction of page groups
404 * @req - request that no longer needs the page group
413 struct nfs_page *head = req->wb_head; in nfs_page_group_destroy()
421 next = tmp->wb_this_page; in nfs_page_group_destroy()
423 tmp->wb_this_page = tmp; in nfs_page_group_destroy()
424 tmp->wb_head = tmp; in nfs_page_group_destroy()
440 struct nfs_open_context *ctx = l_ctx->open_context; in __nfs_create_request()
442 if (test_bit(NFS_CONTEXT_BAD, &ctx->flags)) in __nfs_create_request()
443 return ERR_PTR(-EBADF); in __nfs_create_request()
447 return ERR_PTR(-ENOMEM); in __nfs_create_request()
449 req->wb_lock_context = l_ctx; in __nfs_create_request()
450 refcount_inc(&l_ctx->count); in __nfs_create_request()
451 atomic_inc(&l_ctx->io_count); in __nfs_create_request()
454 * long write-back delay. This will be adjusted in in __nfs_create_request()
456 req->wb_page = page; in __nfs_create_request()
458 req->wb_index = page_index(page); in __nfs_create_request()
461 req->wb_offset = offset; in __nfs_create_request()
462 req->wb_pgbase = pgbase; in __nfs_create_request()
463 req->wb_bytes = count; in __nfs_create_request()
464 kref_init(&req->wb_kref); in __nfs_create_request()
465 req->wb_nio = 0; in __nfs_create_request()
470 * nfs_create_request - Create an NFS read/write request.
505 ret = __nfs_create_request(req->wb_lock_context, req->wb_page, in nfs_create_subreq()
509 for (last = req->wb_head; in nfs_create_subreq()
510 last->wb_this_page != req->wb_head; in nfs_create_subreq()
511 last = last->wb_this_page) in nfs_create_subreq()
515 ret->wb_index = req->wb_index; in nfs_create_subreq()
517 ret->wb_nio = req->wb_nio; in nfs_create_subreq()
523 * nfs_unlock_request - Unlock request and wake up sleepers.
533 clear_bit(PG_BUSY, &req->wb_flags); in nfs_unlock_request()
535 if (!test_bit(PG_CONTENDED2, &req->wb_flags)) in nfs_unlock_request()
537 wake_up_bit(&req->wb_flags, PG_BUSY); in nfs_unlock_request()
541 * nfs_unlock_and_release_request - Unlock request and release the nfs_page
551 * nfs_clear_request - Free up all resources allocated to the request
559 struct page *page = req->wb_page; in nfs_clear_request()
560 struct nfs_lock_context *l_ctx = req->wb_lock_context; in nfs_clear_request()
565 req->wb_page = NULL; in nfs_clear_request()
568 if (atomic_dec_and_test(&l_ctx->io_count)) { in nfs_clear_request()
569 wake_up_var(&l_ctx->io_count); in nfs_clear_request()
570 ctx = l_ctx->open_context; in nfs_clear_request()
571 if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags)) in nfs_clear_request()
572 rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq); in nfs_clear_request()
575 req->wb_lock_context = NULL; in nfs_clear_request()
580 * nfs_release_request - Release the count on an NFS read/write request
587 WARN_ON_ONCE(req->wb_this_page != req); in nfs_free_request()
590 WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags)); in nfs_free_request()
591 WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags)); in nfs_free_request()
592 WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags)); in nfs_free_request()
593 WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags)); in nfs_free_request()
594 WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags)); in nfs_free_request()
603 kref_put(&req->wb_kref, nfs_page_group_destroy); in nfs_release_request()
608 * nfs_wait_on_request - Wait for a request to complete.
617 if (!test_bit(PG_BUSY, &req->wb_flags)) in nfs_wait_on_request()
619 set_bit(PG_CONTENDED2, &req->wb_flags); in nfs_wait_on_request()
621 return wait_on_bit_io(&req->wb_flags, PG_BUSY, in nfs_wait_on_request()
627 * nfs_generic_pg_test - determine if requests can be coalesced
638 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_generic_pg_test() local
641 if (mirror->pg_count > mirror->pg_bsize) { in nfs_generic_pg_test()
651 if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) * in nfs_generic_pg_test()
655 return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes); in nfs_generic_pg_test()
661 struct nfs_pgio_header *hdr = ops->rw_alloc_header(); in nfs_pgio_header_alloc()
664 INIT_LIST_HEAD(&hdr->pages); in nfs_pgio_header_alloc()
665 hdr->rw_ops = ops; in nfs_pgio_header_alloc()
672 * nfs_pgio_data_destroy - make @hdr suitable for reuse
681 if (hdr->args.context) in nfs_pgio_data_destroy()
682 put_nfs_open_context(hdr->args.context); in nfs_pgio_data_destroy()
683 if (hdr->page_array.pagevec != hdr->page_array.page_array) in nfs_pgio_data_destroy()
684 kfree(hdr->page_array.pagevec); in nfs_pgio_data_destroy()
688 * nfs_pgio_header_free - Free a read or write header
694 hdr->rw_ops->rw_free_header(hdr); in nfs_pgio_header_free()
699 * nfs_pgio_rpcsetup - Set up arguments for a pageio call
702 * @how: How to commit data (writes only)
709 struct nfs_page *req = hdr->req; in nfs_pgio_rpcsetup()
712 * NB: take care not to mess about with hdr->commit et al. */ in nfs_pgio_rpcsetup()
714 hdr->args.fh = NFS_FH(hdr->inode); in nfs_pgio_rpcsetup()
715 hdr->args.offset = req_offset(req); in nfs_pgio_rpcsetup()
717 hdr->mds_offset = hdr->args.offset; in nfs_pgio_rpcsetup()
718 hdr->args.pgbase = req->wb_pgbase; in nfs_pgio_rpcsetup()
719 hdr->args.pages = hdr->page_array.pagevec; in nfs_pgio_rpcsetup()
720 hdr->args.count = count; in nfs_pgio_rpcsetup()
721 hdr->args.context = get_nfs_open_context(nfs_req_openctx(req)); in nfs_pgio_rpcsetup()
722 hdr->args.lock_context = req->wb_lock_context; in nfs_pgio_rpcsetup()
723 hdr->args.stable = NFS_UNSTABLE; in nfs_pgio_rpcsetup()
732 hdr->args.stable = NFS_FILE_SYNC; in nfs_pgio_rpcsetup()
735 hdr->res.fattr = &hdr->fattr; in nfs_pgio_rpcsetup()
736 hdr->res.count = 0; in nfs_pgio_rpcsetup()
737 hdr->res.eof = 0; in nfs_pgio_rpcsetup()
738 hdr->res.verf = &hdr->verf; in nfs_pgio_rpcsetup()
739 nfs_fattr_init(&hdr->fattr); in nfs_pgio_rpcsetup()
743 * nfs_pgio_prepare - Prepare pageio hdr to go over the wire
751 err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr); in nfs_pgio_prepare()
762 .rpc_argp = &hdr->args, in nfs_initiate_pgio()
763 .rpc_resp = &hdr->res, in nfs_initiate_pgio()
768 .task = &hdr->task, in nfs_initiate_pgio()
776 hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how); in nfs_initiate_pgio()
780 hdr->inode->i_sb->s_id, in nfs_initiate_pgio()
781 (unsigned long long)NFS_FILEID(hdr->inode), in nfs_initiate_pgio()
782 hdr->args.count, in nfs_initiate_pgio()
783 (unsigned long long)hdr->args.offset); in nfs_initiate_pgio()
794 * nfs_pgio_error - Clean up from a pageio error
799 set_bit(NFS_IOHDR_REDO, &hdr->flags); in nfs_pgio_error()
800 hdr->completion_ops->completion(hdr); in nfs_pgio_error()
804 * nfs_pgio_release - Release pageio data
810 hdr->completion_ops->completion(hdr); in nfs_pgio_release()
813 static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror, in nfs_pageio_mirror_init() argument
816 INIT_LIST_HEAD(&mirror->pg_list); in nfs_pageio_mirror_init()
817 mirror->pg_bytes_written = 0; in nfs_pageio_mirror_init()
818 mirror->pg_count = 0; in nfs_pageio_mirror_init()
819 mirror->pg_bsize = bsize; in nfs_pageio_mirror_init()
820 mirror->pg_base = 0; in nfs_pageio_mirror_init()
821 mirror->pg_recoalesce = 0; in nfs_pageio_mirror_init()
825 * nfs_pageio_init - initialise a page io descriptor
842 desc->pg_moreio = 0; in nfs_pageio_init()
843 desc->pg_inode = inode; in nfs_pageio_init()
844 desc->pg_ops = pg_ops; in nfs_pageio_init()
845 desc->pg_completion_ops = compl_ops; in nfs_pageio_init()
846 desc->pg_rw_ops = rw_ops; in nfs_pageio_init()
847 desc->pg_ioflags = io_flags; in nfs_pageio_init()
848 desc->pg_error = 0; in nfs_pageio_init()
849 desc->pg_lseg = NULL; in nfs_pageio_init()
850 desc->pg_io_completion = NULL; in nfs_pageio_init()
851 desc->pg_dreq = NULL; in nfs_pageio_init()
852 desc->pg_bsize = bsize; in nfs_pageio_init()
854 desc->pg_mirror_count = 1; in nfs_pageio_init()
855 desc->pg_mirror_idx = 0; in nfs_pageio_init()
857 desc->pg_mirrors_dynamic = NULL; in nfs_pageio_init()
858 desc->pg_mirrors = desc->pg_mirrors_static; in nfs_pageio_init()
859 nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize); in nfs_pageio_init()
860 desc->pg_maxretrans = 0; in nfs_pageio_init()
864 * nfs_pgio_result - Basic pageio error handling
871 struct inode *inode = hdr->inode; in nfs_pgio_result()
874 task->tk_pid, task->tk_status); in nfs_pgio_result()
876 if (hdr->rw_ops->rw_done(task, hdr, inode) != 0) in nfs_pgio_result()
878 if (task->tk_status < 0) in nfs_pgio_result()
879 nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset); in nfs_pgio_result()
881 hdr->rw_ops->rw_result(task, hdr); in nfs_pgio_result()
895 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_generic_pgio() local
900 struct list_head *head = &mirror->pg_list; in nfs_generic_pgio()
902 struct nfs_page_array *pg_array = &hdr->page_array; in nfs_generic_pgio()
906 pagecount = nfs_page_array_len(mirror->pg_base, mirror->pg_count); in nfs_generic_pgio()
907 pg_array->npages = pagecount; in nfs_generic_pgio()
909 if (pagecount <= ARRAY_SIZE(pg_array->page_array)) in nfs_generic_pgio()
910 pg_array->pagevec = pg_array->page_array; in nfs_generic_pgio()
912 pg_array->pagevec = kcalloc(pagecount, sizeof(struct page *), gfp_flags); in nfs_generic_pgio()
913 if (!pg_array->pagevec) { in nfs_generic_pgio()
914 pg_array->npages = 0; in nfs_generic_pgio()
916 desc->pg_error = -ENOMEM; in nfs_generic_pgio()
917 return desc->pg_error; in nfs_generic_pgio()
921 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); in nfs_generic_pgio()
922 pages = hdr->page_array.pagevec; in nfs_generic_pgio()
926 req = nfs_list_entry(head->next); in nfs_generic_pgio()
927 nfs_list_move_request(req, &hdr->pages); in nfs_generic_pgio()
929 if (!last_page || last_page != req->wb_page) { in nfs_generic_pgio()
933 *pages++ = last_page = req->wb_page; in nfs_generic_pgio()
938 desc->pg_error = -EINVAL; in nfs_generic_pgio()
939 return desc->pg_error; in nfs_generic_pgio()
942 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && in nfs_generic_pgio()
943 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo))) in nfs_generic_pgio()
944 desc->pg_ioflags &= ~FLUSH_COND_STABLE; in nfs_generic_pgio()
947 nfs_pgio_rpcsetup(hdr, mirror->pg_count, desc->pg_ioflags, &cinfo); in nfs_generic_pgio()
948 desc->pg_rpc_callops = &nfs_pgio_common_ops; in nfs_generic_pgio()
958 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); in nfs_generic_pg_pgios()
960 desc->pg_error = -ENOMEM; in nfs_generic_pg_pgios()
961 return desc->pg_error; in nfs_generic_pg_pgios()
966 ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode), in nfs_generic_pg_pgios()
968 hdr->cred, in nfs_generic_pg_pgios()
969 NFS_PROTO(hdr->inode), in nfs_generic_pg_pgios()
970 desc->pg_rpc_callops, in nfs_generic_pg_pgios()
971 desc->pg_ioflags, in nfs_generic_pg_pgios()
983 kfree(desc->pg_mirrors_dynamic); in nfs_pageio_alloc_mirrors()
984 desc->pg_mirrors_dynamic = NULL; in nfs_pageio_alloc_mirrors()
986 return desc->pg_mirrors_static; in nfs_pageio_alloc_mirrors()
990 nfs_pageio_mirror_init(&ret[i], desc->pg_bsize); in nfs_pageio_alloc_mirrors()
991 desc->pg_mirrors_dynamic = ret; in nfs_pageio_alloc_mirrors()
997 * nfs_pageio_setup_mirroring - determine if mirroring is to be used
1005 if (pgio->pg_ops->pg_get_mirror_count) in nfs_pageio_setup_mirroring()
1006 mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req); in nfs_pageio_setup_mirroring()
1007 if (mirror_count == pgio->pg_mirror_count || pgio->pg_error < 0) in nfs_pageio_setup_mirroring()
1011 pgio->pg_error = -EINVAL; in nfs_pageio_setup_mirroring()
1015 pgio->pg_mirrors = nfs_pageio_alloc_mirrors(pgio, mirror_count); in nfs_pageio_setup_mirroring()
1016 if (pgio->pg_mirrors == NULL) { in nfs_pageio_setup_mirroring()
1017 pgio->pg_error = -ENOMEM; in nfs_pageio_setup_mirroring()
1018 pgio->pg_mirrors = pgio->pg_mirrors_static; in nfs_pageio_setup_mirroring()
1021 pgio->pg_mirror_count = mirror_count; in nfs_pageio_setup_mirroring()
1026 pgio->pg_mirror_count = 1; in nfs_pageio_cleanup_mirroring()
1027 pgio->pg_mirror_idx = 0; in nfs_pageio_cleanup_mirroring()
1028 pgio->pg_mirrors = pgio->pg_mirrors_static; in nfs_pageio_cleanup_mirroring()
1029 kfree(pgio->pg_mirrors_dynamic); in nfs_pageio_cleanup_mirroring()
1030 pgio->pg_mirrors_dynamic = NULL; in nfs_pageio_cleanup_mirroring()
1036 return l1->lockowner == l2->lockowner; in nfs_match_lock_context()
1040 * nfs_coalesce_size - test two requests for compatibility
1046 * page data area they describe is contiguous, and that their RPC
1060 flctx = d_inode(nfs_req_openctx(req)->dentry)->i_flctx; in nfs_coalesce_size()
1062 !(list_empty_careful(&flctx->flc_posix) && in nfs_coalesce_size()
1063 list_empty_careful(&flctx->flc_flock)) && in nfs_coalesce_size()
1064 !nfs_match_lock_context(req->wb_lock_context, in nfs_coalesce_size()
1065 prev->wb_lock_context)) in nfs_coalesce_size()
1067 if (req_offset(req) != req_offset(prev) + prev->wb_bytes) in nfs_coalesce_size()
1069 if (req->wb_page == prev->wb_page) { in nfs_coalesce_size()
1070 if (req->wb_pgbase != prev->wb_pgbase + prev->wb_bytes) in nfs_coalesce_size()
1073 if (req->wb_pgbase != 0 || in nfs_coalesce_size()
1074 prev->wb_pgbase + prev->wb_bytes != PAGE_SIZE) in nfs_coalesce_size()
1078 return pgio->pg_ops->pg_test(pgio, prev, req); in nfs_coalesce_size()
1082 * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
1093 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_pageio_do_add_request() local
1097 if (mirror->pg_count != 0) { in nfs_pageio_do_add_request()
1098 prev = nfs_list_entry(mirror->pg_list.prev); in nfs_pageio_do_add_request()
1100 if (desc->pg_ops->pg_init) in nfs_pageio_do_add_request()
1101 desc->pg_ops->pg_init(desc, req); in nfs_pageio_do_add_request()
1102 if (desc->pg_error < 0) in nfs_pageio_do_add_request()
1104 mirror->pg_base = req->wb_pgbase; in nfs_pageio_do_add_request()
1107 if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) { in nfs_pageio_do_add_request()
1108 if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR) in nfs_pageio_do_add_request()
1109 desc->pg_error = -ETIMEDOUT; in nfs_pageio_do_add_request()
1111 desc->pg_error = -EIO; in nfs_pageio_do_add_request()
1116 if (size < req->wb_bytes) in nfs_pageio_do_add_request()
1118 nfs_list_move_request(req, &mirror->pg_list); in nfs_pageio_do_add_request()
1119 mirror->pg_count += req->wb_bytes; in nfs_pageio_do_add_request()
1120 return req->wb_bytes; in nfs_pageio_do_add_request()
1128 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_pageio_doio() local
1131 if (!list_empty(&mirror->pg_list)) { in nfs_pageio_doio()
1132 int error = desc->pg_ops->pg_doio(desc); in nfs_pageio_doio()
1134 desc->pg_error = error; in nfs_pageio_doio()
1136 mirror->pg_bytes_written += mirror->pg_count; in nfs_pageio_doio()
1138 if (list_empty(&mirror->pg_list)) { in nfs_pageio_doio()
1139 mirror->pg_count = 0; in nfs_pageio_doio()
1140 mirror->pg_base = 0; in nfs_pageio_doio()
1151 desc->pg_completion_ops->error_cleanup(&head, desc->pg_error); in nfs_pageio_cleanup_request()
1155 * nfs_pageio_add_request - Attempt to coalesce a request into a page list.
1169 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in __nfs_pageio_add_request() local
1176 subreq_size = subreq->wb_bytes; in __nfs_pageio_add_request()
1183 req->wb_pgbase += size; in __nfs_pageio_add_request()
1184 req->wb_bytes -= size; in __nfs_pageio_add_request()
1185 req->wb_offset += size; in __nfs_pageio_add_request()
1186 subreq_size = req->wb_bytes; in __nfs_pageio_add_request()
1194 subreq_size = req->wb_bytes; in __nfs_pageio_add_request()
1200 desc->pg_moreio = 1; in __nfs_pageio_add_request()
1202 if (desc->pg_error < 0 || mirror->pg_recoalesce) in __nfs_pageio_add_request()
1208 subreq = nfs_create_subreq(req, req->wb_pgbase, in __nfs_pageio_add_request()
1209 req->wb_offset, size); in __nfs_pageio_add_request()
1218 desc->pg_error = PTR_ERR(subreq); in __nfs_pageio_add_request()
1225 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_do_recoalesce() local
1229 list_splice_init(&mirror->pg_list, &head); in nfs_do_recoalesce()
1230 mirror->pg_bytes_written -= mirror->pg_count; in nfs_do_recoalesce()
1231 mirror->pg_count = 0; in nfs_do_recoalesce()
1232 mirror->pg_base = 0; in nfs_do_recoalesce()
1233 mirror->pg_recoalesce = 0; in nfs_do_recoalesce()
1241 if (desc->pg_error < 0) { in nfs_do_recoalesce()
1242 list_splice_tail(&head, &mirror->pg_list); in nfs_do_recoalesce()
1243 mirror->pg_recoalesce = 1; in nfs_do_recoalesce()
1248 } while (mirror->pg_recoalesce); in nfs_do_recoalesce()
1261 if (desc->pg_error < 0) in nfs_pageio_add_request_mirror()
1272 struct nfs_pgio_mirror *mirror; in nfs_pageio_error_cleanup() local
1274 if (!desc->pg_error) in nfs_pageio_error_cleanup()
1277 for (midx = 0; midx < desc->pg_mirror_count; midx++) { in nfs_pageio_error_cleanup()
1278 mirror = nfs_pgio_get_mirror(desc, midx); in nfs_pageio_error_cleanup()
1279 desc->pg_completion_ops->error_cleanup(&mirror->pg_list, in nfs_pageio_error_cleanup()
1280 desc->pg_error); in nfs_pageio_error_cleanup()
1291 pgbase = req->wb_pgbase; in nfs_pageio_add_request()
1292 offset = req->wb_offset; in nfs_pageio_add_request()
1293 bytes = req->wb_bytes; in nfs_pageio_add_request()
1296 if (desc->pg_error < 0) in nfs_pageio_add_request()
1299 /* Create the mirror instances first, and fire them off */ in nfs_pageio_add_request()
1300 for (midx = 1; midx < desc->pg_mirror_count; midx++) { in nfs_pageio_add_request()
1308 desc->pg_error = PTR_ERR(dupreq); in nfs_pageio_add_request()
1331 * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
1334 * @mirror_idx: pointer to mirror index
1339 struct nfs_pgio_mirror *mirror; in nfs_pageio_complete_mirror() local
1343 mirror = nfs_pgio_current_mirror(desc); in nfs_pageio_complete_mirror()
1347 if (desc->pg_error < 0 || !mirror->pg_recoalesce) in nfs_pageio_complete_mirror()
1356 * nfs_pageio_resend - Transfer requests to new descriptor and resend
1357 * @hdr - the pgio header to move request from
1358 * @desc - the pageio descriptor to add requests to
1370 desc->pg_io_completion = hdr->io_completion; in nfs_pageio_resend()
1371 desc->pg_dreq = hdr->dreq; in nfs_pageio_resend()
1372 list_splice_init(&hdr->pages, &pages); in nfs_pageio_resend()
1381 int err = desc->pg_error < 0 ? desc->pg_error : -EIO; in nfs_pageio_resend()
1382 hdr->completion_ops->error_cleanup(&pages, err); in nfs_pageio_resend()
1383 nfs_set_pgio_error(hdr, err, hdr->io_start); in nfs_pageio_resend()
1391 * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
1398 for (midx = 0; midx < desc->pg_mirror_count; midx++) in nfs_pageio_complete()
1401 if (desc->pg_error < 0) in nfs_pageio_complete()
1403 if (desc->pg_ops->pg_cleanup) in nfs_pageio_complete()
1404 desc->pg_ops->pg_cleanup(desc); in nfs_pageio_complete()
1409 * nfs_pageio_cond_complete - Conditional I/O completion
1414 * on non-contiguous ranges of pages as that might deadlock. This
1421 struct nfs_pgio_mirror *mirror; in nfs_pageio_cond_complete() local
1425 for (midx = 0; midx < desc->pg_mirror_count; midx++) { in nfs_pageio_cond_complete()
1426 mirror = nfs_pgio_get_mirror(desc, midx); in nfs_pageio_cond_complete()
1427 if (!list_empty(&mirror->pg_list)) { in nfs_pageio_cond_complete()
1428 prev = nfs_list_entry(mirror->pg_list.prev); in nfs_pageio_cond_complete()
1429 if (index != prev->wb_index + 1) { in nfs_pageio_cond_complete()
1438 * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
1452 return -ENOMEM; in nfs_init_nfspagecache()