Lines Matching +full:data +full:- +full:mirror
1 // SPDX-License-Identifier: GPL-2.0-only
44 i->req = req; in nfs_page_iter_page_init()
45 i->count = 0; in nfs_page_iter_page_init()
50 const struct nfs_page *req = i->req; in nfs_page_iter_page_advance()
51 size_t tmp = i->count + sz; in nfs_page_iter_page_advance()
53 i->count = (tmp < req->wb_bytes) ? tmp : req->wb_bytes; in nfs_page_iter_page_advance()
58 const struct nfs_page *req = i->req; in nfs_page_iter_page_get()
61 if (i->count != req->wb_bytes) { in nfs_page_iter_page_get()
62 size_t base = i->count + req->wb_pgbase; in nfs_page_iter_page_get()
63 size_t len = PAGE_SIZE - offset_in_page(base); in nfs_page_iter_page_get()
75 if (desc->pg_ops->pg_get_mirror) in nfs_pgio_get_mirror()
76 return desc->pg_ops->pg_get_mirror(desc, idx); in nfs_pgio_get_mirror()
77 return &desc->pg_mirrors[0]; in nfs_pgio_get_mirror()
83 return nfs_pgio_get_mirror(desc, desc->pg_mirror_idx); in nfs_pgio_current_mirror()
90 if (desc->pg_ops->pg_set_mirror) in nfs_pgio_set_current_mirror()
91 return desc->pg_ops->pg_set_mirror(desc, idx); in nfs_pgio_set_current_mirror()
92 return desc->pg_mirror_idx; in nfs_pgio_set_current_mirror()
99 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_pgheader_init() local
102 hdr->req = nfs_list_entry(mirror->pg_list.next); in nfs_pgheader_init()
103 hdr->inode = desc->pg_inode; in nfs_pgheader_init()
104 hdr->cred = nfs_req_openctx(hdr->req)->cred; in nfs_pgheader_init()
105 hdr->io_start = req_offset(hdr->req); in nfs_pgheader_init()
106 hdr->good_bytes = mirror->pg_count; in nfs_pgheader_init()
107 hdr->io_completion = desc->pg_io_completion; in nfs_pgheader_init()
108 hdr->dreq = desc->pg_dreq; in nfs_pgheader_init()
110 hdr->release = release; in nfs_pgheader_init()
111 hdr->completion_ops = desc->pg_completion_ops; in nfs_pgheader_init()
112 if (hdr->completion_ops->init_hdr) in nfs_pgheader_init()
113 hdr->completion_ops->init_hdr(hdr); in nfs_pgheader_init()
115 hdr->pgio_mirror_idx = desc->pg_mirror_idx; in nfs_pgheader_init()
121 unsigned int new = pos - hdr->io_start; in nfs_set_pgio_error()
124 if (hdr->good_bytes > new) { in nfs_set_pgio_error()
125 hdr->good_bytes = new; in nfs_set_pgio_error()
126 clear_bit(NFS_IOHDR_EOF, &hdr->flags); in nfs_set_pgio_error()
127 if (!test_and_set_bit(NFS_IOHDR_ERROR, &hdr->flags)) in nfs_set_pgio_error()
128 hdr->error = error; in nfs_set_pgio_error()
137 INIT_LIST_HEAD(&p->wb_list); in nfs_page_alloc()
148 * nfs_iocounter_wait - wait for i/o to complete
151 * returns -ERESTARTSYS if interrupted by a fatal signal.
157 return wait_var_event_killable(&l_ctx->io_count, in nfs_iocounter_wait()
158 !atomic_read(&l_ctx->io_count)); in nfs_iocounter_wait()
162 * nfs_async_iocounter_wait - wait on a rpc_waitqueue for I/O
173 struct inode *inode = d_inode(l_ctx->open_context->dentry); in nfs_async_iocounter_wait()
176 if (atomic_read(&l_ctx->io_count) > 0) { in nfs_async_iocounter_wait()
177 rpc_sleep_on(&NFS_SERVER(inode)->uoc_rpcwaitq, task, NULL); in nfs_async_iocounter_wait()
181 if (atomic_read(&l_ctx->io_count) == 0) { in nfs_async_iocounter_wait()
182 rpc_wake_up_queued_task(&NFS_SERVER(inode)->uoc_rpcwaitq, task); in nfs_async_iocounter_wait()
191 * nfs_page_lock_head_request - page lock the head of the page group
197 struct nfs_page *head = req->wb_head; in nfs_page_group_lock_head()
205 kref_get(&head->wb_kref); in nfs_page_group_lock_head()
210 * nfs_unroll_locks - unlock all newly locked reqs and wait on @req
223 for (tmp = head->wb_this_page ; tmp != req; tmp = tmp->wb_this_page) { in nfs_unroll_locks()
224 if (!kref_read(&tmp->wb_kref)) in nfs_unroll_locks()
231 * nfs_page_group_lock_subreq - try to lock a subrequest
244 if (!kref_get_unless_zero(&subreq->wb_kref)) in nfs_page_group_lock_subreq()
261 * nfs_page_group_lock_subrequests - try to lock the subrequests
276 for (subreq = head->wb_this_page; subreq != head; in nfs_page_group_lock_subrequests()
277 subreq = subreq->wb_this_page) { in nfs_page_group_lock_subrequests()
287 * nfs_page_set_headlock - set the request PG_HEADLOCK
290 * this lock must be held when modifying req->wb_head
297 if (!test_and_set_bit(PG_HEADLOCK, &req->wb_flags)) in nfs_page_set_headlock()
300 set_bit(PG_CONTENDED1, &req->wb_flags); in nfs_page_set_headlock()
302 return wait_on_bit_lock(&req->wb_flags, PG_HEADLOCK, in nfs_page_set_headlock()
307 * nfs_page_clear_headlock - clear the request PG_HEADLOCK
313 clear_bit_unlock(PG_HEADLOCK, &req->wb_flags); in nfs_page_clear_headlock()
315 if (!test_bit(PG_CONTENDED1, &req->wb_flags)) in nfs_page_clear_headlock()
317 wake_up_bit(&req->wb_flags, PG_HEADLOCK); in nfs_page_clear_headlock()
321 * nfs_page_group_lock - lock the head of the page group
335 if (ret || req->wb_head == req) in nfs_page_group_lock()
337 return nfs_page_set_headlock(req->wb_head); in nfs_page_group_lock()
341 * nfs_page_group_unlock - unlock the head of the page group
347 if (req != req->wb_head) in nfs_page_group_unlock()
348 nfs_page_clear_headlock(req->wb_head); in nfs_page_group_unlock()
360 struct nfs_page *head = req->wb_head; in nfs_page_group_sync_on_bit_locked()
363 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &head->wb_flags)); in nfs_page_group_sync_on_bit_locked()
364 WARN_ON_ONCE(test_and_set_bit(bit, &req->wb_flags)); in nfs_page_group_sync_on_bit_locked()
366 tmp = req->wb_this_page; in nfs_page_group_sync_on_bit_locked()
368 if (!test_bit(bit, &tmp->wb_flags)) in nfs_page_group_sync_on_bit_locked()
370 tmp = tmp->wb_this_page; in nfs_page_group_sync_on_bit_locked()
376 clear_bit(bit, &tmp->wb_flags); in nfs_page_group_sync_on_bit_locked()
377 tmp = tmp->wb_this_page; in nfs_page_group_sync_on_bit_locked()
384 * nfs_page_group_sync_on_bit - set bit on current request, but only
386 * @req - request in page group
387 * @bit - PG_* bit that is used to sync page group
401 * nfs_page_group_init - Initialize the page group linkage for @req
402 * @req - a new nfs request
403 * @prev - the previous request in page group, or NULL if @req is the first
414 req->wb_head = req; in nfs_page_group_init()
415 req->wb_this_page = req; in nfs_page_group_init()
418 WARN_ON_ONCE(prev->wb_this_page != prev->wb_head); in nfs_page_group_init()
419 WARN_ON_ONCE(!test_bit(PG_HEADLOCK, &prev->wb_head->wb_flags)); in nfs_page_group_init()
420 req->wb_head = prev->wb_head; in nfs_page_group_init()
421 req->wb_this_page = prev->wb_this_page; in nfs_page_group_init()
422 prev->wb_this_page = req; in nfs_page_group_init()
426 kref_get(&req->wb_head->wb_kref); in nfs_page_group_init()
431 if (test_bit(PG_INODE_REF, &prev->wb_head->wb_flags)) { in nfs_page_group_init()
433 set_bit(PG_INODE_REF, &req->wb_flags); in nfs_page_group_init()
434 kref_get(&req->wb_kref); in nfs_page_group_init()
435 atomic_long_inc(&NFS_I(inode)->nrequests); in nfs_page_group_init()
441 * nfs_page_group_destroy - sync the destruction of page groups
442 * @req - request that no longer needs the page group
451 struct nfs_page *head = req->wb_head; in nfs_page_group_destroy()
459 next = tmp->wb_this_page; in nfs_page_group_destroy()
461 tmp->wb_this_page = tmp; in nfs_page_group_destroy()
462 tmp->wb_head = tmp; in nfs_page_group_destroy()
477 struct nfs_open_context *ctx = l_ctx->open_context; in nfs_page_create()
479 if (test_bit(NFS_CONTEXT_BAD, &ctx->flags)) in nfs_page_create()
480 return ERR_PTR(-EBADF); in nfs_page_create()
484 return ERR_PTR(-ENOMEM); in nfs_page_create()
486 req->wb_lock_context = l_ctx; in nfs_page_create()
487 refcount_inc(&l_ctx->count); in nfs_page_create()
488 atomic_inc(&l_ctx->io_count); in nfs_page_create()
491 * long write-back delay. This will be adjusted in in nfs_page_create()
493 req->wb_pgbase = pgbase; in nfs_page_create()
494 req->wb_index = index; in nfs_page_create()
495 req->wb_offset = offset; in nfs_page_create()
496 req->wb_bytes = count; in nfs_page_create()
497 kref_init(&req->wb_kref); in nfs_page_create()
498 req->wb_nio = 0; in nfs_page_create()
505 req->wb_folio = folio; in nfs_page_assign_folio()
507 set_bit(PG_FOLIO, &req->wb_flags); in nfs_page_assign_folio()
514 req->wb_page = page; in nfs_page_assign_page()
520 * nfs_page_create_from_page - Create an NFS read/write request.
552 * nfs_page_create_from_folio - Create an NFS read/write request.
592 ret = nfs_page_create(req->wb_lock_context, pgbase, req->wb_index, in nfs_create_subreq()
600 for (last = req->wb_head; in nfs_create_subreq()
601 last->wb_this_page != req->wb_head; in nfs_create_subreq()
602 last = last->wb_this_page) in nfs_create_subreq()
607 ret->wb_nio = req->wb_nio; in nfs_create_subreq()
613 * nfs_unlock_request - Unlock request and wake up sleepers.
618 clear_bit_unlock(PG_BUSY, &req->wb_flags); in nfs_unlock_request()
620 if (!test_bit(PG_CONTENDED2, &req->wb_flags)) in nfs_unlock_request()
622 wake_up_bit(&req->wb_flags, PG_BUSY); in nfs_unlock_request()
626 * nfs_unlock_and_release_request - Unlock request and release the nfs_page
636 * nfs_clear_request - Free up all resources allocated to the request
645 struct page *page = req->wb_page; in nfs_clear_request()
646 struct nfs_lock_context *l_ctx = req->wb_lock_context; in nfs_clear_request()
651 req->wb_folio = NULL; in nfs_clear_request()
652 clear_bit(PG_FOLIO, &req->wb_flags); in nfs_clear_request()
655 req->wb_page = NULL; in nfs_clear_request()
658 if (atomic_dec_and_test(&l_ctx->io_count)) { in nfs_clear_request()
659 wake_up_var(&l_ctx->io_count); in nfs_clear_request()
660 ctx = l_ctx->open_context; in nfs_clear_request()
661 if (test_bit(NFS_CONTEXT_UNLOCK, &ctx->flags)) in nfs_clear_request()
662 rpc_wake_up(&NFS_SERVER(d_inode(ctx->dentry))->uoc_rpcwaitq); in nfs_clear_request()
665 req->wb_lock_context = NULL; in nfs_clear_request()
670 * nfs_free_request - Release the count on an NFS read/write request
677 WARN_ON_ONCE(req->wb_this_page != req); in nfs_free_request()
680 WARN_ON_ONCE(test_bit(PG_TEARDOWN, &req->wb_flags)); in nfs_free_request()
681 WARN_ON_ONCE(test_bit(PG_UNLOCKPAGE, &req->wb_flags)); in nfs_free_request()
682 WARN_ON_ONCE(test_bit(PG_UPTODATE, &req->wb_flags)); in nfs_free_request()
683 WARN_ON_ONCE(test_bit(PG_WB_END, &req->wb_flags)); in nfs_free_request()
684 WARN_ON_ONCE(test_bit(PG_REMOVE, &req->wb_flags)); in nfs_free_request()
693 kref_put(&req->wb_kref, nfs_page_group_destroy); in nfs_release_request()
698 * nfs_wait_on_request - Wait for a request to complete.
707 if (!test_bit(PG_BUSY, &req->wb_flags)) in nfs_wait_on_request()
709 set_bit(PG_CONTENDED2, &req->wb_flags); in nfs_wait_on_request()
711 return wait_on_bit_io(&req->wb_flags, PG_BUSY, in nfs_wait_on_request()
717 * nfs_generic_pg_test - determine if requests can be coalesced
728 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_generic_pg_test() local
731 if (mirror->pg_count > mirror->pg_bsize) { in nfs_generic_pg_test()
741 if (((mirror->pg_count + req->wb_bytes) >> PAGE_SHIFT) * in nfs_generic_pg_test()
745 return min(mirror->pg_bsize - mirror->pg_count, (size_t)req->wb_bytes); in nfs_generic_pg_test()
751 struct nfs_pgio_header *hdr = ops->rw_alloc_header(); in nfs_pgio_header_alloc()
754 INIT_LIST_HEAD(&hdr->pages); in nfs_pgio_header_alloc()
755 hdr->rw_ops = ops; in nfs_pgio_header_alloc()
762 * nfs_pgio_data_destroy - make @hdr suitable for reuse
771 if (hdr->args.context) in nfs_pgio_data_destroy()
772 put_nfs_open_context(hdr->args.context); in nfs_pgio_data_destroy()
773 if (hdr->page_array.pagevec != hdr->page_array.page_array) in nfs_pgio_data_destroy()
774 kfree(hdr->page_array.pagevec); in nfs_pgio_data_destroy()
778 * nfs_pgio_header_free - Free a read or write header
784 hdr->rw_ops->rw_free_header(hdr); in nfs_pgio_header_free()
789 * nfs_pgio_rpcsetup - Set up arguments for a pageio call
793 * @how: How to commit data (writes only)
800 struct nfs_page *req = hdr->req; in nfs_pgio_rpcsetup()
803 * NB: take care not to mess about with hdr->commit et al. */ in nfs_pgio_rpcsetup()
805 hdr->args.fh = NFS_FH(hdr->inode); in nfs_pgio_rpcsetup()
806 hdr->args.offset = req_offset(req); in nfs_pgio_rpcsetup()
808 hdr->mds_offset = hdr->args.offset; in nfs_pgio_rpcsetup()
809 hdr->args.pgbase = pgbase; in nfs_pgio_rpcsetup()
810 hdr->args.pages = hdr->page_array.pagevec; in nfs_pgio_rpcsetup()
811 hdr->args.count = count; in nfs_pgio_rpcsetup()
812 hdr->args.context = get_nfs_open_context(nfs_req_openctx(req)); in nfs_pgio_rpcsetup()
813 hdr->args.lock_context = req->wb_lock_context; in nfs_pgio_rpcsetup()
814 hdr->args.stable = NFS_UNSTABLE; in nfs_pgio_rpcsetup()
823 hdr->args.stable = NFS_FILE_SYNC; in nfs_pgio_rpcsetup()
826 hdr->res.fattr = &hdr->fattr; in nfs_pgio_rpcsetup()
827 hdr->res.count = 0; in nfs_pgio_rpcsetup()
828 hdr->res.eof = 0; in nfs_pgio_rpcsetup()
829 hdr->res.verf = &hdr->verf; in nfs_pgio_rpcsetup()
830 nfs_fattr_init(&hdr->fattr); in nfs_pgio_rpcsetup()
834 * nfs_pgio_prepare - Prepare pageio hdr to go over the wire
842 err = NFS_PROTO(hdr->inode)->pgio_rpc_prepare(task, hdr); in nfs_pgio_prepare()
853 .rpc_argp = &hdr->args, in nfs_initiate_pgio()
854 .rpc_resp = &hdr->res, in nfs_initiate_pgio()
859 .task = &hdr->task, in nfs_initiate_pgio()
867 if (nfs_server_capable(hdr->inode, NFS_CAP_MOVEABLE)) in nfs_initiate_pgio()
870 hdr->rw_ops->rw_initiate(hdr, &msg, rpc_ops, &task_setup_data, how); in nfs_initiate_pgio()
874 hdr->inode->i_sb->s_id, in nfs_initiate_pgio()
875 (unsigned long long)NFS_FILEID(hdr->inode), in nfs_initiate_pgio()
876 hdr->args.count, in nfs_initiate_pgio()
877 (unsigned long long)hdr->args.offset); in nfs_initiate_pgio()
888 * nfs_pgio_error - Clean up from a pageio error
893 set_bit(NFS_IOHDR_REDO, &hdr->flags); in nfs_pgio_error()
894 hdr->completion_ops->completion(hdr); in nfs_pgio_error()
898 * nfs_pgio_release - Release pageio data
904 hdr->completion_ops->completion(hdr); in nfs_pgio_release()
907 static void nfs_pageio_mirror_init(struct nfs_pgio_mirror *mirror, in nfs_pageio_mirror_init() argument
910 INIT_LIST_HEAD(&mirror->pg_list); in nfs_pageio_mirror_init()
911 mirror->pg_bytes_written = 0; in nfs_pageio_mirror_init()
912 mirror->pg_count = 0; in nfs_pageio_mirror_init()
913 mirror->pg_bsize = bsize; in nfs_pageio_mirror_init()
914 mirror->pg_base = 0; in nfs_pageio_mirror_init()
915 mirror->pg_recoalesce = 0; in nfs_pageio_mirror_init()
919 * nfs_pageio_init - initialise a page io descriptor
936 desc->pg_moreio = 0; in nfs_pageio_init()
937 desc->pg_inode = inode; in nfs_pageio_init()
938 desc->pg_ops = pg_ops; in nfs_pageio_init()
939 desc->pg_completion_ops = compl_ops; in nfs_pageio_init()
940 desc->pg_rw_ops = rw_ops; in nfs_pageio_init()
941 desc->pg_ioflags = io_flags; in nfs_pageio_init()
942 desc->pg_error = 0; in nfs_pageio_init()
943 desc->pg_lseg = NULL; in nfs_pageio_init()
944 desc->pg_io_completion = NULL; in nfs_pageio_init()
945 desc->pg_dreq = NULL; in nfs_pageio_init()
947 desc->pg_bsize = bsize; in nfs_pageio_init()
949 desc->pg_mirror_count = 1; in nfs_pageio_init()
950 desc->pg_mirror_idx = 0; in nfs_pageio_init()
952 desc->pg_mirrors_dynamic = NULL; in nfs_pageio_init()
953 desc->pg_mirrors = desc->pg_mirrors_static; in nfs_pageio_init()
954 nfs_pageio_mirror_init(&desc->pg_mirrors[0], bsize); in nfs_pageio_init()
955 desc->pg_maxretrans = 0; in nfs_pageio_init()
959 * nfs_pgio_result - Basic pageio error handling
966 struct inode *inode = hdr->inode; in nfs_pgio_result()
968 if (hdr->rw_ops->rw_done(task, hdr, inode) != 0) in nfs_pgio_result()
970 if (task->tk_status < 0) in nfs_pgio_result()
971 nfs_set_pgio_error(hdr, task->tk_status, hdr->args.offset); in nfs_pgio_result()
973 hdr->rw_ops->rw_result(task, hdr); in nfs_pgio_result()
987 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_generic_pgio() local
992 struct list_head *head = &mirror->pg_list; in nfs_generic_pgio()
994 struct nfs_page_array *pg_array = &hdr->page_array; in nfs_generic_pgio()
996 unsigned int pg_base = offset_in_page(mirror->pg_base); in nfs_generic_pgio()
999 pagecount = nfs_page_array_len(pg_base, mirror->pg_count); in nfs_generic_pgio()
1000 pg_array->npages = pagecount; in nfs_generic_pgio()
1002 if (pagecount <= ARRAY_SIZE(pg_array->page_array)) in nfs_generic_pgio()
1003 pg_array->pagevec = pg_array->page_array; in nfs_generic_pgio()
1005 pg_array->pagevec = kcalloc(pagecount, sizeof(struct page *), gfp_flags); in nfs_generic_pgio()
1006 if (!pg_array->pagevec) { in nfs_generic_pgio()
1007 pg_array->npages = 0; in nfs_generic_pgio()
1009 desc->pg_error = -ENOMEM; in nfs_generic_pgio()
1010 return desc->pg_error; in nfs_generic_pgio()
1014 nfs_init_cinfo(&cinfo, desc->pg_inode, desc->pg_dreq); in nfs_generic_pgio()
1015 pages = hdr->page_array.pagevec; in nfs_generic_pgio()
1022 req = nfs_list_entry(head->next); in nfs_generic_pgio()
1023 nfs_list_move_request(req, &hdr->pages); in nfs_generic_pgio()
1025 if (req->wb_pgbase == 0) in nfs_generic_pgio()
1041 desc->pg_error = -EINVAL; in nfs_generic_pgio()
1042 return desc->pg_error; in nfs_generic_pgio()
1045 if ((desc->pg_ioflags & FLUSH_COND_STABLE) && in nfs_generic_pgio()
1046 (desc->pg_moreio || nfs_reqs_to_commit(&cinfo))) in nfs_generic_pgio()
1047 desc->pg_ioflags &= ~FLUSH_COND_STABLE; in nfs_generic_pgio()
1050 nfs_pgio_rpcsetup(hdr, pg_base, mirror->pg_count, desc->pg_ioflags, in nfs_generic_pgio()
1052 desc->pg_rpc_callops = &nfs_pgio_common_ops; in nfs_generic_pgio()
1063 hdr = nfs_pgio_header_alloc(desc->pg_rw_ops); in nfs_generic_pg_pgios()
1065 desc->pg_error = -ENOMEM; in nfs_generic_pg_pgios()
1066 return desc->pg_error; in nfs_generic_pg_pgios()
1071 if (NFS_SERVER(hdr->inode)->nfs_client->cl_minorversion) in nfs_generic_pg_pgios()
1073 ret = nfs_initiate_pgio(NFS_CLIENT(hdr->inode), in nfs_generic_pg_pgios()
1075 hdr->cred, in nfs_generic_pg_pgios()
1076 NFS_PROTO(hdr->inode), in nfs_generic_pg_pgios()
1077 desc->pg_rpc_callops, in nfs_generic_pg_pgios()
1078 desc->pg_ioflags, in nfs_generic_pg_pgios()
1091 kfree(desc->pg_mirrors_dynamic); in nfs_pageio_alloc_mirrors()
1092 desc->pg_mirrors_dynamic = NULL; in nfs_pageio_alloc_mirrors()
1094 return desc->pg_mirrors_static; in nfs_pageio_alloc_mirrors()
1098 nfs_pageio_mirror_init(&ret[i], desc->pg_bsize); in nfs_pageio_alloc_mirrors()
1099 desc->pg_mirrors_dynamic = ret; in nfs_pageio_alloc_mirrors()
1105 * nfs_pageio_setup_mirroring - determine if mirroring is to be used
1113 if (pgio->pg_ops->pg_get_mirror_count) in nfs_pageio_setup_mirroring()
1114 mirror_count = pgio->pg_ops->pg_get_mirror_count(pgio, req); in nfs_pageio_setup_mirroring()
1115 if (mirror_count == pgio->pg_mirror_count || pgio->pg_error < 0) in nfs_pageio_setup_mirroring()
1119 pgio->pg_error = -EINVAL; in nfs_pageio_setup_mirroring()
1123 pgio->pg_mirrors = nfs_pageio_alloc_mirrors(pgio, mirror_count); in nfs_pageio_setup_mirroring()
1124 if (pgio->pg_mirrors == NULL) { in nfs_pageio_setup_mirroring()
1125 pgio->pg_error = -ENOMEM; in nfs_pageio_setup_mirroring()
1126 pgio->pg_mirrors = pgio->pg_mirrors_static; in nfs_pageio_setup_mirroring()
1129 pgio->pg_mirror_count = mirror_count; in nfs_pageio_setup_mirroring()
1134 pgio->pg_mirror_count = 1; in nfs_pageio_cleanup_mirroring()
1135 pgio->pg_mirror_idx = 0; in nfs_pageio_cleanup_mirroring()
1136 pgio->pg_mirrors = pgio->pg_mirrors_static; in nfs_pageio_cleanup_mirroring()
1137 kfree(pgio->pg_mirrors_dynamic); in nfs_pageio_cleanup_mirroring()
1138 pgio->pg_mirrors_dynamic = NULL; in nfs_pageio_cleanup_mirroring()
1144 return l1->lockowner == l2->lockowner; in nfs_match_lock_context()
1150 size_t prev_end = prev->wb_pgbase + prev->wb_bytes; in nfs_page_is_contiguous()
1152 if (req_offset(req) != req_offset(prev) + prev->wb_bytes) in nfs_page_is_contiguous()
1154 if (req->wb_pgbase == 0) in nfs_page_is_contiguous()
1156 if (req->wb_pgbase == prev_end) { in nfs_page_is_contiguous()
1160 return req->wb_page == prev->wb_page; in nfs_page_is_contiguous()
1166 * nfs_coalesce_size - test two requests for compatibility
1172 * page data area they describe is contiguous, and that their RPC
1186 flctx = locks_inode_context(d_inode(nfs_req_openctx(req)->dentry)); in nfs_coalesce_size()
1188 !(list_empty_careful(&flctx->flc_posix) && in nfs_coalesce_size()
1189 list_empty_careful(&flctx->flc_flock)) && in nfs_coalesce_size()
1190 !nfs_match_lock_context(req->wb_lock_context, in nfs_coalesce_size()
1191 prev->wb_lock_context)) in nfs_coalesce_size()
1196 return pgio->pg_ops->pg_test(pgio, prev, req); in nfs_coalesce_size()
1200 * nfs_pageio_do_add_request - Attempt to coalesce a request into a page list.
1211 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_pageio_do_add_request() local
1215 if (list_empty(&mirror->pg_list)) { in nfs_pageio_do_add_request()
1216 if (desc->pg_ops->pg_init) in nfs_pageio_do_add_request()
1217 desc->pg_ops->pg_init(desc, req); in nfs_pageio_do_add_request()
1218 if (desc->pg_error < 0) in nfs_pageio_do_add_request()
1220 mirror->pg_base = req->wb_pgbase; in nfs_pageio_do_add_request()
1221 mirror->pg_count = 0; in nfs_pageio_do_add_request()
1222 mirror->pg_recoalesce = 0; in nfs_pageio_do_add_request()
1224 prev = nfs_list_entry(mirror->pg_list.prev); in nfs_pageio_do_add_request()
1226 if (desc->pg_maxretrans && req->wb_nio > desc->pg_maxretrans) { in nfs_pageio_do_add_request()
1227 if (NFS_SERVER(desc->pg_inode)->flags & NFS_MOUNT_SOFTERR) in nfs_pageio_do_add_request()
1228 desc->pg_error = -ETIMEDOUT; in nfs_pageio_do_add_request()
1230 desc->pg_error = -EIO; in nfs_pageio_do_add_request()
1235 if (size < req->wb_bytes) in nfs_pageio_do_add_request()
1237 nfs_list_move_request(req, &mirror->pg_list); in nfs_pageio_do_add_request()
1238 mirror->pg_count += req->wb_bytes; in nfs_pageio_do_add_request()
1239 return req->wb_bytes; in nfs_pageio_do_add_request()
1247 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_pageio_doio() local
1249 if (!list_empty(&mirror->pg_list)) { in nfs_pageio_doio()
1250 int error = desc->pg_ops->pg_doio(desc); in nfs_pageio_doio()
1252 desc->pg_error = error; in nfs_pageio_doio()
1253 if (list_empty(&mirror->pg_list)) in nfs_pageio_doio()
1254 mirror->pg_bytes_written += mirror->pg_count; in nfs_pageio_doio()
1265 desc->pg_completion_ops->error_cleanup(&head, desc->pg_error); in nfs_pageio_cleanup_request()
1269 * __nfs_pageio_add_request - Attempt to coalesce a request into a page list.
1283 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in __nfs_pageio_add_request() local
1290 subreq_size = subreq->wb_bytes; in __nfs_pageio_add_request()
1297 req->wb_pgbase += size; in __nfs_pageio_add_request()
1298 req->wb_bytes -= size; in __nfs_pageio_add_request()
1299 req->wb_offset += size; in __nfs_pageio_add_request()
1300 subreq_size = req->wb_bytes; in __nfs_pageio_add_request()
1308 subreq_size = req->wb_bytes; in __nfs_pageio_add_request()
1314 desc->pg_moreio = 1; in __nfs_pageio_add_request()
1316 if (desc->pg_error < 0 || mirror->pg_recoalesce) in __nfs_pageio_add_request()
1322 subreq = nfs_create_subreq(req, req->wb_pgbase, in __nfs_pageio_add_request()
1323 req->wb_offset, size); in __nfs_pageio_add_request()
1332 desc->pg_error = PTR_ERR(subreq); in __nfs_pageio_add_request()
1339 struct nfs_pgio_mirror *mirror = nfs_pgio_current_mirror(desc); in nfs_do_recoalesce() local
1343 list_splice_init(&mirror->pg_list, &head); in nfs_do_recoalesce()
1344 mirror->pg_recoalesce = 0; in nfs_do_recoalesce()
1352 if (desc->pg_error < 0) { in nfs_do_recoalesce()
1353 list_splice_tail(&head, &mirror->pg_list); in nfs_do_recoalesce()
1354 mirror->pg_recoalesce = 1; in nfs_do_recoalesce()
1359 } while (mirror->pg_recoalesce); in nfs_do_recoalesce()
1372 if (desc->pg_error < 0) in nfs_pageio_add_request_mirror()
1383 struct nfs_pgio_mirror *mirror; in nfs_pageio_error_cleanup() local
1385 if (!desc->pg_error) in nfs_pageio_error_cleanup()
1388 for (midx = 0; midx < desc->pg_mirror_count; midx++) { in nfs_pageio_error_cleanup()
1389 mirror = nfs_pgio_get_mirror(desc, midx); in nfs_pageio_error_cleanup()
1390 desc->pg_completion_ops->error_cleanup(&mirror->pg_list, in nfs_pageio_error_cleanup()
1391 desc->pg_error); in nfs_pageio_error_cleanup()
1402 pgbase = req->wb_pgbase; in nfs_pageio_add_request()
1403 offset = req->wb_offset; in nfs_pageio_add_request()
1404 bytes = req->wb_bytes; in nfs_pageio_add_request()
1407 if (desc->pg_error < 0) in nfs_pageio_add_request()
1410 /* Create the mirror instances first, and fire them off */ in nfs_pageio_add_request()
1411 for (midx = 1; midx < desc->pg_mirror_count; midx++) { in nfs_pageio_add_request()
1419 desc->pg_error = PTR_ERR(dupreq); in nfs_pageio_add_request()
1442 * nfs_pageio_complete_mirror - Complete I/O on the current mirror of an
1445 * @mirror_idx: pointer to mirror index
1450 struct nfs_pgio_mirror *mirror; in nfs_pageio_complete_mirror() local
1454 mirror = nfs_pgio_current_mirror(desc); in nfs_pageio_complete_mirror()
1458 if (desc->pg_error < 0 || !mirror->pg_recoalesce) in nfs_pageio_complete_mirror()
1467 * nfs_pageio_resend - Transfer requests to new descriptor and resend
1468 * @hdr - the pgio header to move request from
1469 * @desc - the pageio descriptor to add requests to
1481 desc->pg_io_completion = hdr->io_completion; in nfs_pageio_resend()
1482 desc->pg_dreq = hdr->dreq; in nfs_pageio_resend()
1484 list_splice_init(&hdr->pages, &pages); in nfs_pageio_resend()
1493 int err = desc->pg_error < 0 ? desc->pg_error : -EIO; in nfs_pageio_resend()
1494 hdr->completion_ops->error_cleanup(&pages, err); in nfs_pageio_resend()
1495 nfs_set_pgio_error(hdr, err, hdr->io_start); in nfs_pageio_resend()
1503 * nfs_pageio_complete - Complete I/O then cleanup an nfs_pageio_descriptor
1510 for (midx = 0; midx < desc->pg_mirror_count; midx++) in nfs_pageio_complete()
1513 if (desc->pg_error < 0) in nfs_pageio_complete()
1515 if (desc->pg_ops->pg_cleanup) in nfs_pageio_complete()
1516 desc->pg_ops->pg_cleanup(desc); in nfs_pageio_complete()
1521 * nfs_pageio_cond_complete - Conditional I/O completion
1526 * on non-contiguous ranges of pages as that might deadlock. This
1533 struct nfs_pgio_mirror *mirror; in nfs_pageio_cond_complete() local
1538 for (midx = 0; midx < desc->pg_mirror_count; midx++) { in nfs_pageio_cond_complete()
1539 mirror = nfs_pgio_get_mirror(desc, midx); in nfs_pageio_cond_complete()
1540 if (!list_empty(&mirror->pg_list)) { in nfs_pageio_cond_complete()
1541 prev = nfs_list_entry(mirror->pg_list.prev); in nfs_pageio_cond_complete()
1546 } else if (index == prev->wb_index + 1) in nfs_pageio_cond_complete()
1555 * nfs_pageio_stop_mirroring - stop using mirroring (set mirror count to 1)
1569 return -ENOMEM; in nfs_init_nfspagecache()