58 		 "Enable inter server to server copy offload. Default: false");
408 	 * field isn't set for some reason, throw warning and just copy  in set_change_info()
966 	 * If we do a zero copy read, then a client will see read data  in nfsd4_read()
970 	 * To ensure proper ordering, we therefore turn off zero copy if  in nfsd4_read()
1284 static void nfs4_put_copy(struct nfsd4_copy *copy)  in nfs4_put_copy()  argument
1286 	if (!refcount_dec_and_test(&copy->refcount))  in nfs4_put_copy()
1288 	kfree(copy->cp_src);  in nfs4_put_copy()
1289 	kfree(copy);  in nfs4_put_copy()
1292 static void nfsd4_stop_copy(struct nfsd4_copy *copy)  in nfsd4_stop_copy()  argument
1294 	if (!test_and_set_bit(NFSD4_COPY_F_STOPPED, &copy->cp_flags))  in nfsd4_stop_copy()
1295 		kthread_stop(copy->copy_task);  in nfsd4_stop_copy()
1296 	nfs4_put_copy(copy);  in nfsd4_stop_copy()
1301 	struct nfsd4_copy *copy = NULL;  in nfsd4_get_copy()  local
1305 		copy = list_first_entry(&clp->async_copies, struct nfsd4_copy,  in nfsd4_get_copy()
1307 		refcount_inc(&copy->refcount);  in nfsd4_get_copy()
1310 	return copy;  in nfsd4_get_copy()
1315 	struct nfsd4_copy *copy;  in nfsd4_shutdown_copy()  local
1317 	while ((copy = nfsd4_get_copy(clp)) != NULL)  in nfsd4_shutdown_copy()
1318 		nfsd4_stop_copy(copy);  in nfsd4_shutdown_copy()
1409  * Support one copy source server for now.
1503  * Verify COPY destination stateid.
1507  * Called with COPY cstate:
1514 		      struct nfsd4_copy *copy)  in nfsd4_setup_inter_ssc()  argument
1517 	stateid_t *s_stid = &copy->cp_src_stateid;  in nfsd4_setup_inter_ssc()
1522 					    &copy->cp_dst_stateid,  in nfsd4_setup_inter_ssc()
1523 					    WR_STATE, &copy->nf_dst, NULL);  in nfsd4_setup_inter_ssc()
1527 	status = nfsd4_interssc_connect(copy->cp_src, rqstp, &copy->ss_nsui);  in nfsd4_setup_inter_ssc()
1533 	copy->c_fh.size = s_fh->fh_handle.fh_size;  in nfsd4_setup_inter_ssc()
1534 	memcpy(copy->c_fh.data, &s_fh->fh_handle.fh_raw, copy->c_fh.size);  in nfsd4_setup_inter_ssc()
1535 	copy->stateid.seqid = cpu_to_be32(s_stid->si_generation);  in nfsd4_setup_inter_ssc()
1536 	memcpy(copy->stateid.other, (void *)&s_stid->si_opaque,  in nfsd4_setup_inter_ssc()
1572 		      struct nfsd4_copy *copy)  in nfsd4_setup_inter_ssc()  argument
1594 		      struct nfsd4_copy *copy)  in nfsd4_setup_intra_ssc()  argument
1596 	return nfsd4_verify_copy(rqstp, cstate, &copy->cp_src_stateid,  in nfsd4_setup_intra_ssc()
1597 				 &copy->nf_src, &copy->cp_dst_stateid,  in nfsd4_setup_intra_ssc()
1598 				 &copy->nf_dst);  in nfsd4_setup_intra_ssc()
1624 static void nfsd4_init_copy_res(struct nfsd4_copy *copy, bool sync)  in nfsd4_init_copy_res()  argument
1626 	copy->cp_res.wr_stable_how =  in nfsd4_init_copy_res()
1627 		test_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags) ?  in nfsd4_init_copy_res()
1629 	nfsd4_copy_set_sync(copy, sync);  in nfsd4_init_copy_res()
1630 	gen_boot_verifier(&copy->cp_res.wr_verifier, copy->cp_clp->net);  in nfsd4_init_copy_res()
1633 static ssize_t _nfsd_copy_file_range(struct nfsd4_copy *copy,  in _nfsd_copy_file_range()  argument
1639 	u64 bytes_total = copy->cp_count;  in _nfsd_copy_file_range()
1640 	u64 src_pos = copy->cp_src_pos;  in _nfsd_copy_file_range()
1641 	u64 dst_pos = copy->cp_dst_pos;  in _nfsd_copy_file_range()
1657 		copy->cp_res.wr_bytes_written += bytes_copied;  in _nfsd_copy_file_range()
1660 	} while (bytes_total > 0 && nfsd4_copy_is_async(copy));  in _nfsd_copy_file_range()
1661 	/* for a non-zero asynchronous copy do a commit of data */  in _nfsd_copy_file_range()
1662 	if (nfsd4_copy_is_async(copy) && copy->cp_res.wr_bytes_written > 0) {  in _nfsd_copy_file_range()
1664 		end = copy->cp_dst_pos + copy->cp_res.wr_bytes_written - 1;  in _nfsd_copy_file_range()
1665 		status = vfs_fsync_range(dst, copy->cp_dst_pos, end, 0);  in _nfsd_copy_file_range()
1669 			set_bit(NFSD4_COPY_F_COMMITTED, &copy->cp_flags);  in _nfsd_copy_file_range()
1674 static __be32 nfsd4_do_copy(struct nfsd4_copy *copy,  in nfsd4_do_copy()  argument
1681 	bytes = _nfsd_copy_file_range(copy, dst, src);  in nfsd4_do_copy()
1683 	/* for async copy, we ignore the error, client can always retry  in nfsd4_do_copy()
1686 	if (bytes < 0 && !copy->cp_res.wr_bytes_written)  in nfsd4_do_copy()
1689 		nfsd4_init_copy_res(copy, sync);  in nfsd4_do_copy()
1716 static void release_copy_files(struct nfsd4_copy *copy)  in release_copy_files()  argument
1718 	if (copy->nf_src)  in release_copy_files()
1719 		nfsd_file_put(copy->nf_src);  in release_copy_files()
1720 	if (copy->nf_dst)  in release_copy_files()
1721 		nfsd_file_put(copy->nf_dst);  in release_copy_files()
1724 static void cleanup_async_copy(struct nfsd4_copy *copy)  in cleanup_async_copy()  argument
1726 	nfs4_free_copy_state(copy);  in cleanup_async_copy()
1727 	release_copy_files(copy);  in cleanup_async_copy()
1728 	if (copy->cp_clp) {  in cleanup_async_copy()
1729 		spin_lock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1730 		if (!list_empty(&copy->copies))  in cleanup_async_copy()
1731 			list_del_init(&copy->copies);  in cleanup_async_copy()
1732 		spin_unlock(&copy->cp_clp->async_lock);  in cleanup_async_copy()
1734 	nfs4_put_copy(copy);  in cleanup_async_copy()
1737 static void nfsd4_send_cb_offload(struct nfsd4_copy *copy, __be32 nfserr)  in nfsd4_send_cb_offload()  argument
1745 	memcpy(&cbo->co_res, &copy->cp_res, sizeof(copy->cp_res));  in nfsd4_send_cb_offload()
1746 	memcpy(&cbo->co_fh, &copy->fh, sizeof(copy->fh));  in nfsd4_send_cb_offload()
1749 	nfsd4_init_cb(&cbo->co_cb, copy->cp_clp, &nfsd4_cb_offload_ops,  in nfsd4_send_cb_offload()
1751 	trace_nfsd_cb_offload(copy->cp_clp, &cbo->co_res.cb_stateid,  in nfsd4_send_cb_offload()
1752 			      &cbo->co_fh, copy->cp_count, nfserr);  in nfsd4_send_cb_offload()
1757  * nfsd4_do_async_copy - kthread function for background server-side COPY
1758  * @data: arguments for COPY operation
1761  *   %0: Copy operation is done.
1765 	struct nfsd4_copy *copy = (struct nfsd4_copy *)data;  in nfsd4_do_async_copy()  local
1768 	trace_nfsd_copy_do_async(copy);  in nfsd4_do_async_copy()
1769 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_do_async_copy()
1772 		filp = nfs42_ssc_open(copy->ss_nsui->nsui_vfsmount,  in nfsd4_do_async_copy()
1773 				      &copy->c_fh, &copy->stateid);  in nfsd4_do_async_copy()
1785 		nfserr = nfsd4_do_copy(copy, filp, copy->nf_dst->nf_file,  in nfsd4_do_async_copy()
1787 		nfsd4_cleanup_inter_ssc(copy->ss_nsui, filp, copy->nf_dst);  in nfsd4_do_async_copy()
1789 		nfserr = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_do_async_copy()
1790 				       copy->nf_dst->nf_file, false);  in nfsd4_do_async_copy()
1794 	nfsd4_send_cb_offload(copy, nfserr);  in nfsd4_do_async_copy()
1795 	cleanup_async_copy(copy);  in nfsd4_do_async_copy()
1803 	struct nfsd4_copy *copy = &u->copy;  in nfsd4_copy()  local
1807 	copy->cp_clp = cstate->clp;  in nfsd4_copy()
1808 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_copy()
1809 		trace_nfsd_copy_inter(copy);  in nfsd4_copy()
1810 		if (!inter_copy_offload_enable || nfsd4_copy_is_sync(copy)) {  in nfsd4_copy()
1814 		status = nfsd4_setup_inter_ssc(rqstp, cstate, copy);  in nfsd4_copy()
1816 			trace_nfsd_copy_done(copy, status);  in nfsd4_copy()
1820 		trace_nfsd_copy_intra(copy);  in nfsd4_copy()
1821 		status = nfsd4_setup_intra_ssc(rqstp, cstate, copy);  in nfsd4_copy()
1823 			trace_nfsd_copy_done(copy, status);  in nfsd4_copy()
1828 	memcpy(&copy->fh, &cstate->current_fh.fh_handle,  in nfsd4_copy()
1830 	if (nfsd4_copy_is_async(copy)) {  in nfsd4_copy()
1842 		if (!nfs4_init_copy_state(nn, copy))  in nfsd4_copy()
1844 		memcpy(&copy->cp_res.cb_stateid, &copy->cp_stateid.cs_stid,  in nfsd4_copy()
1845 			sizeof(copy->cp_res.cb_stateid));  in nfsd4_copy()
1846 		dup_copy_fields(copy, async_copy);  in nfsd4_copy()
1848 				async_copy, "%s", "copy thread");  in nfsd4_copy()
1858 		status = nfsd4_do_copy(copy, copy->nf_src->nf_file,  in nfsd4_copy()
1859 				       copy->nf_dst->nf_file, true);  in nfsd4_copy()
1862 	trace_nfsd_copy_done(copy, status);  in nfsd4_copy()
1863 	release_copy_files(copy);  in nfsd4_copy()
1866 	if (nfsd4_ssc_is_inter(copy)) {  in nfsd4_copy()
1868 		 * Source's vfsmount of inter-copy will be unmounted  in nfsd4_copy()
1869 		 * by the laundromat. Use copy instead of async_copy  in nfsd4_copy()
1872 		refcount_dec(&copy->ss_nsui->nsui_refcnt);  in nfsd4_copy()
1883 	struct nfsd4_copy *copy;  in find_async_copy_locked()  local
1887 	list_for_each_entry(copy, &clp->async_copies, copies) {  in find_async_copy_locked()
1888 		if (memcmp(&copy->cp_stateid.cs_stid, stateid, NFS4_STATEID_SIZE))  in find_async_copy_locked()
1890 		return copy;  in find_async_copy_locked()
1898 	struct nfsd4_copy *copy;  in find_async_copy()  local
1901 	copy = find_async_copy_locked(clp, stateid);  in find_async_copy()
1902 	if (copy)  in find_async_copy()
1903 		refcount_inc(&copy->refcount);  in find_async_copy()
1905 	return copy;  in find_async_copy()
1914 	struct nfsd4_copy *copy;  in nfsd4_offload_cancel()  local
1917 	copy = find_async_copy(clp, &os->stateid);  in nfsd4_offload_cancel()
1918 	if (!copy) {  in nfsd4_offload_cancel()
1923 		nfsd4_stop_copy(copy);  in nfsd4_offload_cancel()
2000 	struct nfsd4_copy *copy;  in nfsd4_offload_status()  local
2004 	copy = find_async_copy_locked(clp, &os->stateid);  in nfsd4_offload_status()
2005 	if (copy)  in nfsd4_offload_status()
2006 		os->count = copy->cp_res.wr_bytes_written;  in nfsd4_offload_status()
2458 	 * Get the entire list, then copy out only the user attributes  in nfsd4_listxattrs()
2591 	struct nfsd4_copy *copy;  in check_if_stalefh_allowed()  local
2595 	/* traverse all operation and if it's a COPY compound, mark the  in check_if_stalefh_allowed()
2607 			copy = (struct nfsd4_copy *)&op->u;  in check_if_stalefh_allowed()
2613 			if (nfsd4_ssc_is_inter(copy))  in check_if_stalefh_allowed()