1 // SPDX-License-Identifier: GPL-2.0-only
2 /*
3  * Common NFS I/O  operations for the pnfs file based
4  * layout drivers.
5  *
6  * Copyright (c) 2014, Primary Data, Inc. All rights reserved.
7  *
8  * Tom Haynes <loghyr@primarydata.com>
9  */
10 
11 #include <linux/nfs_fs.h>
12 #include <linux/nfs_page.h>
13 #include <linux/sunrpc/addr.h>
14 #include <linux/module.h>
15 
16 #include "nfs4session.h"
17 #include "internal.h"
18 #include "pnfs.h"
19 #include "netns.h"
20 
21 #define NFSDBG_FACILITY		NFSDBG_PNFS
22 
pnfs_generic_rw_release(void * data)23 void pnfs_generic_rw_release(void *data)
24 {
25 	struct nfs_pgio_header *hdr = data;
26 
27 	nfs_put_client(hdr->ds_clp);
28 	hdr->mds_ops->rpc_release(data);
29 }
30 EXPORT_SYMBOL_GPL(pnfs_generic_rw_release);
31 
32 /* Fake up some data that will cause nfs_commit_release to retry the writes. */
pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data * data)33 void pnfs_generic_prepare_to_resend_writes(struct nfs_commit_data *data)
34 {
35 	struct nfs_writeverf *verf = data->res.verf;
36 
37 	data->task.tk_status = 0;
38 	memset(&verf->verifier, 0, sizeof(verf->verifier));
39 	verf->committed = NFS_UNSTABLE;
40 }
41 EXPORT_SYMBOL_GPL(pnfs_generic_prepare_to_resend_writes);
42 
pnfs_generic_write_commit_done(struct rpc_task * task,void * data)43 void pnfs_generic_write_commit_done(struct rpc_task *task, void *data)
44 {
45 	struct nfs_commit_data *wdata = data;
46 
47 	/* Note this may cause RPC to be resent */
48 	wdata->mds_ops->rpc_call_done(task, data);
49 }
50 EXPORT_SYMBOL_GPL(pnfs_generic_write_commit_done);
51 
pnfs_generic_commit_release(void * calldata)52 void pnfs_generic_commit_release(void *calldata)
53 {
54 	struct nfs_commit_data *data = calldata;
55 
56 	data->completion_ops->completion(data);
57 	pnfs_put_lseg(data->lseg);
58 	nfs_put_client(data->ds_clp);
59 	nfs_commitdata_release(data);
60 }
61 EXPORT_SYMBOL_GPL(pnfs_generic_commit_release);
62 
63 static struct pnfs_layout_segment *
pnfs_free_bucket_lseg(struct pnfs_commit_bucket * bucket)64 pnfs_free_bucket_lseg(struct pnfs_commit_bucket *bucket)
65 {
66 	if (list_empty(&bucket->committing) && list_empty(&bucket->written)) {
67 		struct pnfs_layout_segment *freeme = bucket->lseg;
68 		bucket->lseg = NULL;
69 		return freeme;
70 	}
71 	return NULL;
72 }
73 
74 /* The generic layer is about to remove the req from the commit list.
75  * If this will make the bucket empty, it will need to put the lseg reference.
76  * Note this must be called holding nfsi->commit_mutex
77  */
78 void
pnfs_generic_clear_request_commit(struct nfs_page * req,struct nfs_commit_info * cinfo)79 pnfs_generic_clear_request_commit(struct nfs_page *req,
80 				  struct nfs_commit_info *cinfo)
81 {
82 	struct pnfs_commit_bucket *bucket = NULL;
83 
84 	if (!test_and_clear_bit(PG_COMMIT_TO_DS, &req->wb_flags))
85 		goto out;
86 	cinfo->ds->nwritten--;
87 	if (list_is_singular(&req->wb_list))
88 		bucket = list_first_entry(&req->wb_list,
89 					  struct pnfs_commit_bucket, written);
90 out:
91 	nfs_request_remove_commit_list(req, cinfo);
92 	if (bucket)
93 		pnfs_put_lseg(pnfs_free_bucket_lseg(bucket));
94 }
95 EXPORT_SYMBOL_GPL(pnfs_generic_clear_request_commit);
96 
97 struct pnfs_commit_array *
pnfs_alloc_commit_array(size_t n,gfp_t gfp_flags)98 pnfs_alloc_commit_array(size_t n, gfp_t gfp_flags)
99 {
100 	struct pnfs_commit_array *p;
101 	struct pnfs_commit_bucket *b;
102 
103 	p = kmalloc(struct_size(p, buckets, n), gfp_flags);
104 	if (!p)
105 		return NULL;
106 	p->nbuckets = n;
107 	INIT_LIST_HEAD(&p->cinfo_list);
108 	INIT_LIST_HEAD(&p->lseg_list);
109 	p->lseg = NULL;
110 	for (b = &p->buckets[0]; n != 0; b++, n--) {
111 		INIT_LIST_HEAD(&b->written);
112 		INIT_LIST_HEAD(&b->committing);
113 		b->lseg = NULL;
114 		b->direct_verf.committed = NFS_INVALID_STABLE_HOW;
115 	}
116 	return p;
117 }
118 EXPORT_SYMBOL_GPL(pnfs_alloc_commit_array);
119 
120 void
pnfs_free_commit_array(struct pnfs_commit_array * p)121 pnfs_free_commit_array(struct pnfs_commit_array *p)
122 {
123 	kfree_rcu(p, rcu);
124 }
125 EXPORT_SYMBOL_GPL(pnfs_free_commit_array);
126 
127 static struct pnfs_commit_array *
pnfs_find_commit_array_by_lseg(struct pnfs_ds_commit_info * fl_cinfo,struct pnfs_layout_segment * lseg)128 pnfs_find_commit_array_by_lseg(struct pnfs_ds_commit_info *fl_cinfo,
129 		struct pnfs_layout_segment *lseg)
130 {
131 	struct pnfs_commit_array *array;
132 
133 	list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) {
134 		if (array->lseg == lseg)
135 			return array;
136 	}
137 	return NULL;
138 }
139 
140 struct pnfs_commit_array *
pnfs_add_commit_array(struct pnfs_ds_commit_info * fl_cinfo,struct pnfs_commit_array * new,struct pnfs_layout_segment * lseg)141 pnfs_add_commit_array(struct pnfs_ds_commit_info *fl_cinfo,
142 		struct pnfs_commit_array *new,
143 		struct pnfs_layout_segment *lseg)
144 {
145 	struct pnfs_commit_array *array;
146 
147 	array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg);
148 	if (array)
149 		return array;
150 	new->lseg = lseg;
151 	refcount_set(&new->refcount, 1);
152 	list_add_rcu(&new->cinfo_list, &fl_cinfo->commits);
153 	list_add(&new->lseg_list, &lseg->pls_commits);
154 	return new;
155 }
156 EXPORT_SYMBOL_GPL(pnfs_add_commit_array);
157 
158 static struct pnfs_commit_array *
pnfs_lookup_commit_array(struct pnfs_ds_commit_info * fl_cinfo,struct pnfs_layout_segment * lseg)159 pnfs_lookup_commit_array(struct pnfs_ds_commit_info *fl_cinfo,
160 		struct pnfs_layout_segment *lseg)
161 {
162 	struct pnfs_commit_array *array;
163 
164 	rcu_read_lock();
165 	array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg);
166 	if (!array) {
167 		rcu_read_unlock();
168 		fl_cinfo->ops->setup_ds_info(fl_cinfo, lseg);
169 		rcu_read_lock();
170 		array = pnfs_find_commit_array_by_lseg(fl_cinfo, lseg);
171 	}
172 	rcu_read_unlock();
173 	return array;
174 }
175 
176 static void
pnfs_release_commit_array_locked(struct pnfs_commit_array * array)177 pnfs_release_commit_array_locked(struct pnfs_commit_array *array)
178 {
179 	list_del_rcu(&array->cinfo_list);
180 	list_del(&array->lseg_list);
181 	pnfs_free_commit_array(array);
182 }
183 
184 static void
pnfs_put_commit_array_locked(struct pnfs_commit_array * array)185 pnfs_put_commit_array_locked(struct pnfs_commit_array *array)
186 {
187 	if (refcount_dec_and_test(&array->refcount))
188 		pnfs_release_commit_array_locked(array);
189 }
190 
191 static void
pnfs_put_commit_array(struct pnfs_commit_array * array,struct inode * inode)192 pnfs_put_commit_array(struct pnfs_commit_array *array, struct inode *inode)
193 {
194 	if (refcount_dec_and_lock(&array->refcount, &inode->i_lock)) {
195 		pnfs_release_commit_array_locked(array);
196 		spin_unlock(&inode->i_lock);
197 	}
198 }
199 
200 static struct pnfs_commit_array *
pnfs_get_commit_array(struct pnfs_commit_array * array)201 pnfs_get_commit_array(struct pnfs_commit_array *array)
202 {
203 	if (refcount_inc_not_zero(&array->refcount))
204 		return array;
205 	return NULL;
206 }
207 
208 static void
pnfs_remove_and_free_commit_array(struct pnfs_commit_array * array)209 pnfs_remove_and_free_commit_array(struct pnfs_commit_array *array)
210 {
211 	array->lseg = NULL;
212 	list_del_init(&array->lseg_list);
213 	pnfs_put_commit_array_locked(array);
214 }
215 
216 void
pnfs_generic_ds_cinfo_release_lseg(struct pnfs_ds_commit_info * fl_cinfo,struct pnfs_layout_segment * lseg)217 pnfs_generic_ds_cinfo_release_lseg(struct pnfs_ds_commit_info *fl_cinfo,
218 		struct pnfs_layout_segment *lseg)
219 {
220 	struct pnfs_commit_array *array, *tmp;
221 
222 	list_for_each_entry_safe(array, tmp, &lseg->pls_commits, lseg_list)
223 		pnfs_remove_and_free_commit_array(array);
224 }
225 EXPORT_SYMBOL_GPL(pnfs_generic_ds_cinfo_release_lseg);
226 
227 void
pnfs_generic_ds_cinfo_destroy(struct pnfs_ds_commit_info * fl_cinfo)228 pnfs_generic_ds_cinfo_destroy(struct pnfs_ds_commit_info *fl_cinfo)
229 {
230 	struct pnfs_commit_array *array, *tmp;
231 
232 	list_for_each_entry_safe(array, tmp, &fl_cinfo->commits, cinfo_list)
233 		pnfs_remove_and_free_commit_array(array);
234 }
235 EXPORT_SYMBOL_GPL(pnfs_generic_ds_cinfo_destroy);
236 
237 /*
238  * Locks the nfs_page requests for commit and moves them to
239  * @bucket->committing.
240  */
241 static int
pnfs_bucket_scan_ds_commit_list(struct pnfs_commit_bucket * bucket,struct nfs_commit_info * cinfo,int max)242 pnfs_bucket_scan_ds_commit_list(struct pnfs_commit_bucket *bucket,
243 				struct nfs_commit_info *cinfo,
244 				int max)
245 {
246 	struct list_head *src = &bucket->written;
247 	struct list_head *dst = &bucket->committing;
248 	int ret;
249 
250 	lockdep_assert_held(&NFS_I(cinfo->inode)->commit_mutex);
251 	ret = nfs_scan_commit_list(src, dst, cinfo, max);
252 	if (ret) {
253 		cinfo->ds->nwritten -= ret;
254 		cinfo->ds->ncommitting += ret;
255 	}
256 	return ret;
257 }
258 
pnfs_bucket_scan_array(struct nfs_commit_info * cinfo,struct pnfs_commit_bucket * buckets,unsigned int nbuckets,int max)259 static int pnfs_bucket_scan_array(struct nfs_commit_info *cinfo,
260 				  struct pnfs_commit_bucket *buckets,
261 				  unsigned int nbuckets,
262 				  int max)
263 {
264 	unsigned int i;
265 	int rv = 0, cnt;
266 
267 	for (i = 0; i < nbuckets && max != 0; i++) {
268 		cnt = pnfs_bucket_scan_ds_commit_list(&buckets[i], cinfo, max);
269 		rv += cnt;
270 		max -= cnt;
271 	}
272 	return rv;
273 }
274 
275 /* Move reqs from written to committing lists, returning count
276  * of number moved.
277  */
pnfs_generic_scan_commit_lists(struct nfs_commit_info * cinfo,int max)278 int pnfs_generic_scan_commit_lists(struct nfs_commit_info *cinfo, int max)
279 {
280 	struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
281 	struct pnfs_commit_array *array;
282 	int rv = 0, cnt;
283 
284 	rcu_read_lock();
285 	list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) {
286 		if (!array->lseg || !pnfs_get_commit_array(array))
287 			continue;
288 		rcu_read_unlock();
289 		cnt = pnfs_bucket_scan_array(cinfo, array->buckets,
290 				array->nbuckets, max);
291 		rcu_read_lock();
292 		pnfs_put_commit_array(array, cinfo->inode);
293 		rv += cnt;
294 		max -= cnt;
295 		if (!max)
296 			break;
297 	}
298 	rcu_read_unlock();
299 	return rv;
300 }
301 EXPORT_SYMBOL_GPL(pnfs_generic_scan_commit_lists);
302 
303 static unsigned int
pnfs_bucket_recover_commit_reqs(struct list_head * dst,struct pnfs_commit_bucket * buckets,unsigned int nbuckets,struct nfs_commit_info * cinfo)304 pnfs_bucket_recover_commit_reqs(struct list_head *dst,
305 			        struct pnfs_commit_bucket *buckets,
306 				unsigned int nbuckets,
307 				struct nfs_commit_info *cinfo)
308 {
309 	struct pnfs_commit_bucket *b;
310 	struct pnfs_layout_segment *freeme;
311 	unsigned int nwritten, ret = 0;
312 	unsigned int i;
313 
314 restart:
315 	for (i = 0, b = buckets; i < nbuckets; i++, b++) {
316 		nwritten = nfs_scan_commit_list(&b->written, dst, cinfo, 0);
317 		if (!nwritten)
318 			continue;
319 		ret += nwritten;
320 		freeme = pnfs_free_bucket_lseg(b);
321 		if (freeme) {
322 			pnfs_put_lseg(freeme);
323 			goto restart;
324 		}
325 	}
326 	return ret;
327 }
328 
329 /* Pull everything off the committing lists and dump into @dst.  */
pnfs_generic_recover_commit_reqs(struct list_head * dst,struct nfs_commit_info * cinfo)330 void pnfs_generic_recover_commit_reqs(struct list_head *dst,
331 				      struct nfs_commit_info *cinfo)
332 {
333 	struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
334 	struct pnfs_commit_array *array;
335 	unsigned int nwritten;
336 
337 	lockdep_assert_held(&NFS_I(cinfo->inode)->commit_mutex);
338 	rcu_read_lock();
339 	list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) {
340 		if (!array->lseg || !pnfs_get_commit_array(array))
341 			continue;
342 		rcu_read_unlock();
343 		nwritten = pnfs_bucket_recover_commit_reqs(dst,
344 							   array->buckets,
345 							   array->nbuckets,
346 							   cinfo);
347 		rcu_read_lock();
348 		pnfs_put_commit_array(array, cinfo->inode);
349 		fl_cinfo->nwritten -= nwritten;
350 	}
351 	rcu_read_unlock();
352 }
353 EXPORT_SYMBOL_GPL(pnfs_generic_recover_commit_reqs);
354 
355 static struct pnfs_layout_segment *
pnfs_bucket_get_committing(struct list_head * head,struct pnfs_commit_bucket * bucket,struct nfs_commit_info * cinfo)356 pnfs_bucket_get_committing(struct list_head *head,
357 			   struct pnfs_commit_bucket *bucket,
358 			   struct nfs_commit_info *cinfo)
359 {
360 	struct pnfs_layout_segment *lseg;
361 	struct list_head *pos;
362 
363 	list_for_each(pos, &bucket->committing)
364 		cinfo->ds->ncommitting--;
365 	list_splice_init(&bucket->committing, head);
366 	lseg = pnfs_free_bucket_lseg(bucket);
367 	if (!lseg)
368 		lseg = pnfs_get_lseg(bucket->lseg);
369 	return lseg;
370 }
371 
372 static struct nfs_commit_data *
pnfs_bucket_fetch_commitdata(struct pnfs_commit_bucket * bucket,struct nfs_commit_info * cinfo)373 pnfs_bucket_fetch_commitdata(struct pnfs_commit_bucket *bucket,
374 			     struct nfs_commit_info *cinfo)
375 {
376 	struct nfs_commit_data *data = nfs_commitdata_alloc();
377 
378 	if (!data)
379 		return NULL;
380 	data->lseg = pnfs_bucket_get_committing(&data->pages, bucket, cinfo);
381 	return data;
382 }
383 
pnfs_generic_retry_commit(struct pnfs_commit_bucket * buckets,unsigned int nbuckets,struct nfs_commit_info * cinfo,unsigned int idx)384 static void pnfs_generic_retry_commit(struct pnfs_commit_bucket *buckets,
385 				      unsigned int nbuckets,
386 				      struct nfs_commit_info *cinfo,
387 				      unsigned int idx)
388 {
389 	struct pnfs_commit_bucket *bucket;
390 	struct pnfs_layout_segment *freeme;
391 	LIST_HEAD(pages);
392 
393 	for (bucket = buckets; idx < nbuckets; bucket++, idx++) {
394 		if (list_empty(&bucket->committing))
395 			continue;
396 		mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
397 		freeme = pnfs_bucket_get_committing(&pages, bucket, cinfo);
398 		mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
399 		nfs_retry_commit(&pages, freeme, cinfo, idx);
400 		pnfs_put_lseg(freeme);
401 	}
402 }
403 
404 static unsigned int
pnfs_bucket_alloc_ds_commits(struct list_head * list,struct pnfs_commit_bucket * buckets,unsigned int nbuckets,struct nfs_commit_info * cinfo)405 pnfs_bucket_alloc_ds_commits(struct list_head *list,
406 			     struct pnfs_commit_bucket *buckets,
407 			     unsigned int nbuckets,
408 			     struct nfs_commit_info *cinfo)
409 {
410 	struct pnfs_commit_bucket *bucket;
411 	struct nfs_commit_data *data;
412 	unsigned int i;
413 	unsigned int nreq = 0;
414 
415 	for (i = 0, bucket = buckets; i < nbuckets; i++, bucket++) {
416 		if (list_empty(&bucket->committing))
417 			continue;
418 		mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
419 		if (!list_empty(&bucket->committing)) {
420 			data = pnfs_bucket_fetch_commitdata(bucket, cinfo);
421 			if (!data)
422 				goto out_error;
423 			data->ds_commit_index = i;
424 			list_add_tail(&data->list, list);
425 			nreq++;
426 		}
427 		mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
428 	}
429 	return nreq;
430 out_error:
431 	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
432 	/* Clean up on error */
433 	pnfs_generic_retry_commit(buckets, nbuckets, cinfo, i);
434 	return nreq;
435 }
436 
437 static unsigned int
pnfs_alloc_ds_commits_list(struct list_head * list,struct pnfs_ds_commit_info * fl_cinfo,struct nfs_commit_info * cinfo)438 pnfs_alloc_ds_commits_list(struct list_head *list,
439 			   struct pnfs_ds_commit_info *fl_cinfo,
440 			   struct nfs_commit_info *cinfo)
441 {
442 	struct pnfs_commit_array *array;
443 	unsigned int ret = 0;
444 
445 	rcu_read_lock();
446 	list_for_each_entry_rcu(array, &fl_cinfo->commits, cinfo_list) {
447 		if (!array->lseg || !pnfs_get_commit_array(array))
448 			continue;
449 		rcu_read_unlock();
450 		ret += pnfs_bucket_alloc_ds_commits(list, array->buckets,
451 				array->nbuckets, cinfo);
452 		rcu_read_lock();
453 		pnfs_put_commit_array(array, cinfo->inode);
454 	}
455 	rcu_read_unlock();
456 	return ret;
457 }
458 
459 /* This follows nfs_commit_list pretty closely */
460 int
pnfs_generic_commit_pagelist(struct inode * inode,struct list_head * mds_pages,int how,struct nfs_commit_info * cinfo,int (* initiate_commit)(struct nfs_commit_data * data,int how))461 pnfs_generic_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
462 			     int how, struct nfs_commit_info *cinfo,
463 			     int (*initiate_commit)(struct nfs_commit_data *data,
464 						    int how))
465 {
466 	struct pnfs_ds_commit_info *fl_cinfo = cinfo->ds;
467 	struct nfs_commit_data *data, *tmp;
468 	LIST_HEAD(list);
469 	unsigned int nreq = 0;
470 
471 	if (!list_empty(mds_pages)) {
472 		data = nfs_commitdata_alloc();
473 		if (!data) {
474 			nfs_retry_commit(mds_pages, NULL, cinfo, -1);
475 			return -ENOMEM;
476 		}
477 		data->ds_commit_index = -1;
478 		list_splice_init(mds_pages, &data->pages);
479 		list_add_tail(&data->list, &list);
480 		nreq++;
481 	}
482 
483 	nreq += pnfs_alloc_ds_commits_list(&list, fl_cinfo, cinfo);
484 	if (nreq == 0)
485 		goto out;
486 
487 	list_for_each_entry_safe(data, tmp, &list, list) {
488 		list_del(&data->list);
489 		if (data->ds_commit_index < 0) {
490 			nfs_init_commit(data, NULL, NULL, cinfo);
491 			nfs_initiate_commit(NFS_CLIENT(inode), data,
492 					    NFS_PROTO(data->inode),
493 					    data->mds_ops, how,
494 					    RPC_TASK_CRED_NOREF, NULL);
495 		} else {
496 			nfs_init_commit(data, NULL, data->lseg, cinfo);
497 			initiate_commit(data, how);
498 		}
499 	}
500 out:
501 	return PNFS_ATTEMPTED;
502 }
503 EXPORT_SYMBOL_GPL(pnfs_generic_commit_pagelist);
504 
505 /*
506  * Data server cache
507  *
508  * Data servers can be mapped to different device ids, but should
509  * never be shared between net namespaces.
510  *
511  * nfs4_pnfs_ds reference counting:
512  *   - set to 1 on allocation
513  *   - incremented when a device id maps a data server already in the cache.
514  *   - decremented when deviceid is removed from the cache.
515  */
516 
517 /* Debug routines */
518 static void
print_ds(struct nfs4_pnfs_ds * ds)519 print_ds(struct nfs4_pnfs_ds *ds)
520 {
521 	if (ds == NULL) {
522 		printk(KERN_WARNING "%s NULL device\n", __func__);
523 		return;
524 	}
525 	printk(KERN_WARNING "        ds %s\n"
526 		"        ref count %d\n"
527 		"        client %p\n"
528 		"        cl_exchange_flags %x\n",
529 		ds->ds_remotestr,
530 		refcount_read(&ds->ds_count), ds->ds_clp,
531 		ds->ds_clp ? ds->ds_clp->cl_exchange_flags : 0);
532 }
533 
534 static bool
same_sockaddr(struct sockaddr * addr1,struct sockaddr * addr2)535 same_sockaddr(struct sockaddr *addr1, struct sockaddr *addr2)
536 {
537 	struct sockaddr_in *a, *b;
538 	struct sockaddr_in6 *a6, *b6;
539 
540 	if (addr1->sa_family != addr2->sa_family)
541 		return false;
542 
543 	switch (addr1->sa_family) {
544 	case AF_INET:
545 		a = (struct sockaddr_in *)addr1;
546 		b = (struct sockaddr_in *)addr2;
547 
548 		if (a->sin_addr.s_addr == b->sin_addr.s_addr &&
549 		    a->sin_port == b->sin_port)
550 			return true;
551 		break;
552 
553 	case AF_INET6:
554 		a6 = (struct sockaddr_in6 *)addr1;
555 		b6 = (struct sockaddr_in6 *)addr2;
556 
557 		/* LINKLOCAL addresses must have matching scope_id */
558 		if (ipv6_addr_src_scope(&a6->sin6_addr) ==
559 		    IPV6_ADDR_SCOPE_LINKLOCAL &&
560 		    a6->sin6_scope_id != b6->sin6_scope_id)
561 			return false;
562 
563 		if (ipv6_addr_equal(&a6->sin6_addr, &b6->sin6_addr) &&
564 		    a6->sin6_port == b6->sin6_port)
565 			return true;
566 		break;
567 
568 	default:
569 		dprintk("%s: unhandled address family: %u\n",
570 			__func__, addr1->sa_family);
571 		return false;
572 	}
573 
574 	return false;
575 }
576 
577 /*
578  * Checks if 'dsaddrs1' contains a subset of 'dsaddrs2'. If it does,
579  * declare a match.
580  */
581 static bool
_same_data_server_addrs_locked(const struct list_head * dsaddrs1,const struct list_head * dsaddrs2)582 _same_data_server_addrs_locked(const struct list_head *dsaddrs1,
583 			       const struct list_head *dsaddrs2)
584 {
585 	struct nfs4_pnfs_ds_addr *da1, *da2;
586 	struct sockaddr *sa1, *sa2;
587 	bool match = false;
588 
589 	list_for_each_entry(da1, dsaddrs1, da_node) {
590 		sa1 = (struct sockaddr *)&da1->da_addr;
591 		match = false;
592 		list_for_each_entry(da2, dsaddrs2, da_node) {
593 			sa2 = (struct sockaddr *)&da2->da_addr;
594 			match = same_sockaddr(sa1, sa2);
595 			if (match)
596 				break;
597 		}
598 		if (!match)
599 			break;
600 	}
601 	return match;
602 }
603 
604 /*
605  * Lookup DS by addresses.  nfs4_ds_cache_lock is held
606  */
607 static struct nfs4_pnfs_ds *
_data_server_lookup_locked(const struct nfs_net * nn,const struct list_head * dsaddrs)608 _data_server_lookup_locked(const struct nfs_net *nn, const struct list_head *dsaddrs)
609 {
610 	struct nfs4_pnfs_ds *ds;
611 
612 	list_for_each_entry(ds, &nn->nfs4_data_server_cache, ds_node)
613 		if (_same_data_server_addrs_locked(&ds->ds_addrs, dsaddrs))
614 			return ds;
615 	return NULL;
616 }
617 
nfs4_pnfs_ds_addr_alloc(gfp_t gfp_flags)618 static struct nfs4_pnfs_ds_addr *nfs4_pnfs_ds_addr_alloc(gfp_t gfp_flags)
619 {
620 	struct nfs4_pnfs_ds_addr *da = kzalloc(sizeof(*da), gfp_flags);
621 	if (da)
622 		INIT_LIST_HEAD(&da->da_node);
623 	return da;
624 }
625 
nfs4_pnfs_ds_addr_free(struct nfs4_pnfs_ds_addr * da)626 static void nfs4_pnfs_ds_addr_free(struct nfs4_pnfs_ds_addr *da)
627 {
628 	kfree(da->da_remotestr);
629 	kfree(da->da_netid);
630 	kfree(da);
631 }
632 
destroy_ds(struct nfs4_pnfs_ds * ds)633 static void destroy_ds(struct nfs4_pnfs_ds *ds)
634 {
635 	struct nfs4_pnfs_ds_addr *da;
636 
637 	dprintk("--> %s\n", __func__);
638 	ifdebug(FACILITY)
639 		print_ds(ds);
640 
641 	nfs_put_client(ds->ds_clp);
642 
643 	while (!list_empty(&ds->ds_addrs)) {
644 		da = list_first_entry(&ds->ds_addrs,
645 				      struct nfs4_pnfs_ds_addr,
646 				      da_node);
647 		list_del_init(&da->da_node);
648 		nfs4_pnfs_ds_addr_free(da);
649 	}
650 
651 	kfree(ds->ds_remotestr);
652 	kfree(ds);
653 }
654 
nfs4_pnfs_ds_put(struct nfs4_pnfs_ds * ds)655 void nfs4_pnfs_ds_put(struct nfs4_pnfs_ds *ds)
656 {
657 	struct nfs_net *nn = net_generic(ds->ds_net, nfs_net_id);
658 
659 	if (refcount_dec_and_lock(&ds->ds_count, &nn->nfs4_data_server_lock)) {
660 		list_del_init(&ds->ds_node);
661 		spin_unlock(&nn->nfs4_data_server_lock);
662 		destroy_ds(ds);
663 	}
664 }
665 EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_put);
666 
667 /*
668  * Create a string with a human readable address and port to avoid
669  * complicated setup around many dprinks.
670  */
671 static char *
nfs4_pnfs_remotestr(struct list_head * dsaddrs,gfp_t gfp_flags)672 nfs4_pnfs_remotestr(struct list_head *dsaddrs, gfp_t gfp_flags)
673 {
674 	struct nfs4_pnfs_ds_addr *da;
675 	char *remotestr;
676 	size_t len;
677 	char *p;
678 
679 	len = 3;        /* '{', '}' and eol */
680 	list_for_each_entry(da, dsaddrs, da_node) {
681 		len += strlen(da->da_remotestr) + 1;    /* string plus comma */
682 	}
683 
684 	remotestr = kzalloc(len, gfp_flags);
685 	if (!remotestr)
686 		return NULL;
687 
688 	p = remotestr;
689 	*(p++) = '{';
690 	len--;
691 	list_for_each_entry(da, dsaddrs, da_node) {
692 		size_t ll = strlen(da->da_remotestr);
693 
694 		if (ll > len)
695 			goto out_err;
696 
697 		memcpy(p, da->da_remotestr, ll);
698 		p += ll;
699 		len -= ll;
700 
701 		if (len < 1)
702 			goto out_err;
703 		(*p++) = ',';
704 		len--;
705 	}
706 	if (len < 2)
707 		goto out_err;
708 	*(p++) = '}';
709 	*p = '\0';
710 	return remotestr;
711 out_err:
712 	kfree(remotestr);
713 	return NULL;
714 }
715 
716 /*
717  * Given a list of multipath struct nfs4_pnfs_ds_addr, add it to ds cache if
718  * uncached and return cached struct nfs4_pnfs_ds.
719  */
720 struct nfs4_pnfs_ds *
nfs4_pnfs_ds_add(const struct net * net,struct list_head * dsaddrs,gfp_t gfp_flags)721 nfs4_pnfs_ds_add(const struct net *net, struct list_head *dsaddrs, gfp_t gfp_flags)
722 {
723 	struct nfs_net *nn = net_generic(net, nfs_net_id);
724 	struct nfs4_pnfs_ds *tmp_ds, *ds = NULL;
725 	char *remotestr;
726 
727 	if (list_empty(dsaddrs)) {
728 		dprintk("%s: no addresses defined\n", __func__);
729 		goto out;
730 	}
731 
732 	ds = kzalloc(sizeof(*ds), gfp_flags);
733 	if (!ds)
734 		goto out;
735 
736 	/* this is only used for debugging, so it's ok if its NULL */
737 	remotestr = nfs4_pnfs_remotestr(dsaddrs, gfp_flags);
738 
739 	spin_lock(&nn->nfs4_data_server_lock);
740 	tmp_ds = _data_server_lookup_locked(nn, dsaddrs);
741 	if (tmp_ds == NULL) {
742 		INIT_LIST_HEAD(&ds->ds_addrs);
743 		list_splice_init(dsaddrs, &ds->ds_addrs);
744 		ds->ds_remotestr = remotestr;
745 		refcount_set(&ds->ds_count, 1);
746 		INIT_LIST_HEAD(&ds->ds_node);
747 		ds->ds_net = net;
748 		ds->ds_clp = NULL;
749 		list_add(&ds->ds_node, &nn->nfs4_data_server_cache);
750 		dprintk("%s add new data server %s\n", __func__,
751 			ds->ds_remotestr);
752 	} else {
753 		kfree(remotestr);
754 		kfree(ds);
755 		refcount_inc(&tmp_ds->ds_count);
756 		dprintk("%s data server %s found, inc'ed ds_count to %d\n",
757 			__func__, tmp_ds->ds_remotestr,
758 			refcount_read(&tmp_ds->ds_count));
759 		ds = tmp_ds;
760 	}
761 	spin_unlock(&nn->nfs4_data_server_lock);
762 out:
763 	return ds;
764 }
765 EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_add);
766 
nfs4_wait_ds_connect(struct nfs4_pnfs_ds * ds)767 static int nfs4_wait_ds_connect(struct nfs4_pnfs_ds *ds)
768 {
769 	might_sleep();
770 	return wait_on_bit(&ds->ds_state, NFS4DS_CONNECTING, TASK_KILLABLE);
771 }
772 
nfs4_clear_ds_conn_bit(struct nfs4_pnfs_ds * ds)773 static void nfs4_clear_ds_conn_bit(struct nfs4_pnfs_ds *ds)
774 {
775 	smp_mb__before_atomic();
776 	clear_and_wake_up_bit(NFS4DS_CONNECTING, &ds->ds_state);
777 }
778 
779 static struct nfs_client *(*get_v3_ds_connect)(
780 			struct nfs_server *mds_srv,
781 			const struct sockaddr_storage *ds_addr,
782 			int ds_addrlen,
783 			int ds_proto,
784 			unsigned int ds_timeo,
785 			unsigned int ds_retrans);
786 
load_v3_ds_connect(void)787 static bool load_v3_ds_connect(void)
788 {
789 	if (!get_v3_ds_connect) {
790 		get_v3_ds_connect = symbol_request(nfs3_set_ds_client);
791 		WARN_ON_ONCE(!get_v3_ds_connect);
792 	}
793 
794 	return(get_v3_ds_connect != NULL);
795 }
796 
nfs4_pnfs_v3_ds_connect_unload(void)797 void nfs4_pnfs_v3_ds_connect_unload(void)
798 {
799 	if (get_v3_ds_connect) {
800 		symbol_put(nfs3_set_ds_client);
801 		get_v3_ds_connect = NULL;
802 	}
803 }
804 
_nfs4_pnfs_v3_ds_connect(struct nfs_server * mds_srv,struct nfs4_pnfs_ds * ds,unsigned int timeo,unsigned int retrans)805 static int _nfs4_pnfs_v3_ds_connect(struct nfs_server *mds_srv,
806 				 struct nfs4_pnfs_ds *ds,
807 				 unsigned int timeo,
808 				 unsigned int retrans)
809 {
810 	struct nfs_client *clp = ERR_PTR(-EIO);
811 	struct nfs4_pnfs_ds_addr *da;
812 	unsigned long connect_timeout = timeo * (retrans + 1) * HZ / 10;
813 	int status = 0;
814 
815 	dprintk("--> %s DS %s\n", __func__, ds->ds_remotestr);
816 
817 	if (!load_v3_ds_connect())
818 		return -EPROTONOSUPPORT;
819 
820 	list_for_each_entry(da, &ds->ds_addrs, da_node) {
821 		dprintk("%s: DS %s: trying address %s\n",
822 			__func__, ds->ds_remotestr, da->da_remotestr);
823 
824 		if (!IS_ERR(clp)) {
825 			struct xprt_create xprt_args = {
826 				.ident = da->da_transport,
827 				.net = clp->cl_net,
828 				.dstaddr = (struct sockaddr *)&da->da_addr,
829 				.addrlen = da->da_addrlen,
830 				.servername = clp->cl_hostname,
831 				.connect_timeout = connect_timeout,
832 				.reconnect_timeout = connect_timeout,
833 			};
834 
835 			if (da->da_transport != clp->cl_proto)
836 				continue;
837 			if (da->da_addr.ss_family != clp->cl_addr.ss_family)
838 				continue;
839 			/* Add this address as an alias */
840 			rpc_clnt_add_xprt(clp->cl_rpcclient, &xprt_args,
841 					rpc_clnt_test_and_add_xprt, NULL);
842 			continue;
843 		}
844 		clp = get_v3_ds_connect(mds_srv,
845 				&da->da_addr,
846 				da->da_addrlen, da->da_transport,
847 				timeo, retrans);
848 		if (IS_ERR(clp))
849 			continue;
850 		clp->cl_rpcclient->cl_softerr = 0;
851 		clp->cl_rpcclient->cl_softrtry = 0;
852 	}
853 
854 	if (IS_ERR(clp)) {
855 		status = PTR_ERR(clp);
856 		goto out;
857 	}
858 
859 	smp_wmb();
860 	WRITE_ONCE(ds->ds_clp, clp);
861 	dprintk("%s [new] addr: %s\n", __func__, ds->ds_remotestr);
862 out:
863 	return status;
864 }
865 
_nfs4_pnfs_v4_ds_connect(struct nfs_server * mds_srv,struct nfs4_pnfs_ds * ds,unsigned int timeo,unsigned int retrans,u32 minor_version)866 static int _nfs4_pnfs_v4_ds_connect(struct nfs_server *mds_srv,
867 				 struct nfs4_pnfs_ds *ds,
868 				 unsigned int timeo,
869 				 unsigned int retrans,
870 				 u32 minor_version)
871 {
872 	struct nfs_client *clp = ERR_PTR(-EIO);
873 	struct nfs4_pnfs_ds_addr *da;
874 	int status = 0;
875 
876 	dprintk("--> %s DS %s\n", __func__, ds->ds_remotestr);
877 
878 	list_for_each_entry(da, &ds->ds_addrs, da_node) {
879 		char servername[48];
880 
881 		dprintk("%s: DS %s: trying address %s\n",
882 			__func__, ds->ds_remotestr, da->da_remotestr);
883 
884 		if (!IS_ERR(clp) && clp->cl_mvops->session_trunk) {
885 			struct xprt_create xprt_args = {
886 				.ident = da->da_transport,
887 				.net = clp->cl_net,
888 				.dstaddr = (struct sockaddr *)&da->da_addr,
889 				.addrlen = da->da_addrlen,
890 				.servername = clp->cl_hostname,
891 				.xprtsec = clp->cl_xprtsec,
892 			};
893 			struct nfs4_add_xprt_data xprtdata = {
894 				.clp = clp,
895 			};
896 			struct rpc_add_xprt_test rpcdata = {
897 				.add_xprt_test = clp->cl_mvops->session_trunk,
898 				.data = &xprtdata,
899 			};
900 
901 			if (da->da_transport != clp->cl_proto &&
902 					clp->cl_proto != XPRT_TRANSPORT_TCP_TLS)
903 				continue;
904 			if (da->da_transport == XPRT_TRANSPORT_TCP &&
905 				mds_srv->nfs_client->cl_proto ==
906 					XPRT_TRANSPORT_TCP_TLS) {
907 				struct sockaddr *addr =
908 					(struct sockaddr *)&da->da_addr;
909 				struct sockaddr_in *sin =
910 					(struct sockaddr_in *)&da->da_addr;
911 				struct sockaddr_in6 *sin6 =
912 					(struct sockaddr_in6 *)&da->da_addr;
913 
914 				/* for NFS with TLS we need to supply a correct
915 				 * servername of the trunked transport, not the
916 				 * servername of the main transport stored in
917 				 * clp->cl_hostname. And set the protocol to
918 				 * indicate to use TLS
919 				 */
920 				servername[0] = '\0';
921 				switch(addr->sa_family) {
922 				case AF_INET:
923 					snprintf(servername, sizeof(servername),
924 						"%pI4", &sin->sin_addr.s_addr);
925 					break;
926 				case AF_INET6:
927 					snprintf(servername, sizeof(servername),
928 						"%pI6", &sin6->sin6_addr);
929 					break;
930 				default:
931 					/* do not consider this address */
932 					continue;
933 				}
934 				xprt_args.ident = XPRT_TRANSPORT_TCP_TLS;
935 				xprt_args.servername = servername;
936 			}
937 			if (da->da_addr.ss_family != clp->cl_addr.ss_family)
938 				continue;
939 
940 			/**
941 			* Test this address for session trunking and
942 			* add as an alias
943 			*/
944 			xprtdata.cred = nfs4_get_clid_cred(clp);
945 			rpc_clnt_add_xprt(clp->cl_rpcclient, &xprt_args,
946 					  rpc_clnt_setup_test_and_add_xprt,
947 					  &rpcdata);
948 			if (xprtdata.cred)
949 				put_cred(xprtdata.cred);
950 		} else {
951 			if (da->da_transport == XPRT_TRANSPORT_TCP &&
952 				mds_srv->nfs_client->cl_proto ==
953 					XPRT_TRANSPORT_TCP_TLS)
954 				da->da_transport = XPRT_TRANSPORT_TCP_TLS;
955 			clp = nfs4_set_ds_client(mds_srv,
956 						&da->da_addr,
957 						da->da_addrlen,
958 						da->da_transport, timeo,
959 						retrans, minor_version);
960 			if (IS_ERR(clp))
961 				continue;
962 
963 			status = nfs4_init_ds_session(clp,
964 					mds_srv->nfs_client->cl_lease_time);
965 			if (status) {
966 				nfs_put_client(clp);
967 				clp = ERR_PTR(-EIO);
968 				continue;
969 			}
970 
971 		}
972 	}
973 
974 	if (IS_ERR(clp)) {
975 		status = PTR_ERR(clp);
976 		goto out;
977 	}
978 
979 	smp_wmb();
980 	WRITE_ONCE(ds->ds_clp, clp);
981 	dprintk("%s [new] addr: %s\n", __func__, ds->ds_remotestr);
982 out:
983 	return status;
984 }
985 
986 /*
987  * Create an rpc connection to the nfs4_pnfs_ds data server.
988  * Currently only supports IPv4 and IPv6 addresses.
989  * If connection fails, make devid unavailable and return a -errno.
990  */
nfs4_pnfs_ds_connect(struct nfs_server * mds_srv,struct nfs4_pnfs_ds * ds,struct nfs4_deviceid_node * devid,unsigned int timeo,unsigned int retrans,u32 version,u32 minor_version)991 int nfs4_pnfs_ds_connect(struct nfs_server *mds_srv, struct nfs4_pnfs_ds *ds,
992 			  struct nfs4_deviceid_node *devid, unsigned int timeo,
993 			  unsigned int retrans, u32 version, u32 minor_version)
994 {
995 	int err;
996 
997 	do {
998 		err = nfs4_wait_ds_connect(ds);
999 		if (err || ds->ds_clp)
1000 			goto out;
1001 		if (nfs4_test_deviceid_unavailable(devid))
1002 			return -ENODEV;
1003 	} while (test_and_set_bit(NFS4DS_CONNECTING, &ds->ds_state) != 0);
1004 
1005 	if (ds->ds_clp)
1006 		goto connect_done;
1007 
1008 	switch (version) {
1009 	case 3:
1010 		err = _nfs4_pnfs_v3_ds_connect(mds_srv, ds, timeo, retrans);
1011 		break;
1012 	case 4:
1013 		err = _nfs4_pnfs_v4_ds_connect(mds_srv, ds, timeo, retrans,
1014 					       minor_version);
1015 		break;
1016 	default:
1017 		dprintk("%s: unsupported DS version %d\n", __func__, version);
1018 		err = -EPROTONOSUPPORT;
1019 	}
1020 
1021 connect_done:
1022 	nfs4_clear_ds_conn_bit(ds);
1023 out:
1024 	/*
1025 	 * At this point the ds->ds_clp should be ready, but it might have
1026 	 * hit an error.
1027 	 */
1028 	if (!err) {
1029 		if (!ds->ds_clp || !nfs_client_init_is_complete(ds->ds_clp)) {
1030 			WARN_ON_ONCE(ds->ds_clp ||
1031 				!nfs4_test_deviceid_unavailable(devid));
1032 			return -EINVAL;
1033 		}
1034 		err = nfs_client_init_status(ds->ds_clp);
1035 	}
1036 
1037 	return err;
1038 }
1039 EXPORT_SYMBOL_GPL(nfs4_pnfs_ds_connect);
1040 
1041 /*
1042  * Currently only supports ipv4, ipv6 and one multi-path address.
1043  */
1044 struct nfs4_pnfs_ds_addr *
nfs4_decode_mp_ds_addr(struct net * net,struct xdr_stream * xdr,gfp_t gfp_flags)1045 nfs4_decode_mp_ds_addr(struct net *net, struct xdr_stream *xdr, gfp_t gfp_flags)
1046 {
1047 	struct nfs4_pnfs_ds_addr *da = NULL;
1048 	char *buf, *portstr;
1049 	__be16 port;
1050 	ssize_t nlen, rlen;
1051 	int tmp[2];
1052 	char *netid;
1053 	size_t len;
1054 	char *startsep = "";
1055 	char *endsep = "";
1056 
1057 
1058 	/* r_netid */
1059 	nlen = xdr_stream_decode_string_dup(xdr, &netid, XDR_MAX_NETOBJ,
1060 					    gfp_flags);
1061 	if (unlikely(nlen < 0))
1062 		goto out_err;
1063 
1064 	/* r_addr: ip/ip6addr with port in dec octets - see RFC 5665 */
1065 	/* port is ".ABC.DEF", 8 chars max */
1066 	rlen = xdr_stream_decode_string_dup(xdr, &buf, INET6_ADDRSTRLEN +
1067 					    IPV6_SCOPE_ID_LEN + 8, gfp_flags);
1068 	if (unlikely(rlen < 0))
1069 		goto out_free_netid;
1070 
1071 	/* replace port '.' with '-' */
1072 	portstr = strrchr(buf, '.');
1073 	if (!portstr) {
1074 		dprintk("%s: Failed finding expected dot in port\n",
1075 			__func__);
1076 		goto out_free_buf;
1077 	}
1078 	*portstr = '-';
1079 
1080 	/* find '.' between address and port */
1081 	portstr = strrchr(buf, '.');
1082 	if (!portstr) {
1083 		dprintk("%s: Failed finding expected dot between address and "
1084 			"port\n", __func__);
1085 		goto out_free_buf;
1086 	}
1087 	*portstr = '\0';
1088 
1089 	da = nfs4_pnfs_ds_addr_alloc(gfp_flags);
1090 	if (unlikely(!da))
1091 		goto out_free_buf;
1092 
1093 	if (!rpc_pton(net, buf, portstr-buf, (struct sockaddr *)&da->da_addr,
1094 		      sizeof(da->da_addr))) {
1095 		dprintk("%s: error parsing address %s\n", __func__, buf);
1096 		goto out_free_da;
1097 	}
1098 
1099 	portstr++;
1100 	sscanf(portstr, "%d-%d", &tmp[0], &tmp[1]);
1101 	port = htons((tmp[0] << 8) | (tmp[1]));
1102 
1103 	switch (da->da_addr.ss_family) {
1104 	case AF_INET:
1105 		((struct sockaddr_in *)&da->da_addr)->sin_port = port;
1106 		da->da_addrlen = sizeof(struct sockaddr_in);
1107 		break;
1108 
1109 	case AF_INET6:
1110 		((struct sockaddr_in6 *)&da->da_addr)->sin6_port = port;
1111 		da->da_addrlen = sizeof(struct sockaddr_in6);
1112 		startsep = "[";
1113 		endsep = "]";
1114 		break;
1115 
1116 	default:
1117 		dprintk("%s: unsupported address family: %u\n",
1118 			__func__, da->da_addr.ss_family);
1119 		goto out_free_da;
1120 	}
1121 
1122 	da->da_transport = xprt_find_transport_ident(netid);
1123 	if (da->da_transport < 0) {
1124 		dprintk("%s: ERROR: unknown r_netid \"%s\"\n",
1125 			__func__, netid);
1126 		goto out_free_da;
1127 	}
1128 
1129 	da->da_netid = netid;
1130 
1131 	/* save human readable address */
1132 	len = strlen(startsep) + strlen(buf) + strlen(endsep) + 7;
1133 	da->da_remotestr = kzalloc(len, gfp_flags);
1134 
1135 	/* NULL is ok, only used for dprintk */
1136 	if (da->da_remotestr)
1137 		snprintf(da->da_remotestr, len, "%s%s%s:%u", startsep,
1138 			 buf, endsep, ntohs(port));
1139 
1140 	dprintk("%s: Parsed DS addr %s\n", __func__, da->da_remotestr);
1141 	kfree(buf);
1142 	return da;
1143 
1144 out_free_da:
1145 	kfree(da);
1146 out_free_buf:
1147 	dprintk("%s: Error parsing DS addr: %s\n", __func__, buf);
1148 	kfree(buf);
1149 out_free_netid:
1150 	kfree(netid);
1151 out_err:
1152 	return NULL;
1153 }
1154 EXPORT_SYMBOL_GPL(nfs4_decode_mp_ds_addr);
1155 
1156 void
pnfs_layout_mark_request_commit(struct nfs_page * req,struct pnfs_layout_segment * lseg,struct nfs_commit_info * cinfo,u32 ds_commit_idx)1157 pnfs_layout_mark_request_commit(struct nfs_page *req,
1158 				struct pnfs_layout_segment *lseg,
1159 				struct nfs_commit_info *cinfo,
1160 				u32 ds_commit_idx)
1161 {
1162 	struct list_head *list;
1163 	struct pnfs_commit_array *array;
1164 	struct pnfs_commit_bucket *bucket;
1165 
1166 	mutex_lock(&NFS_I(cinfo->inode)->commit_mutex);
1167 	array = pnfs_lookup_commit_array(cinfo->ds, lseg);
1168 	if (!array || !pnfs_is_valid_lseg(lseg))
1169 		goto out_resched;
1170 	bucket = &array->buckets[ds_commit_idx];
1171 	list = &bucket->written;
1172 	/* Non-empty buckets hold a reference on the lseg.  That ref
1173 	 * is normally transferred to the COMMIT call and released
1174 	 * there.  It could also be released if the last req is pulled
1175 	 * off due to a rewrite, in which case it will be done in
1176 	 * pnfs_common_clear_request_commit
1177 	 */
1178 	if (!bucket->lseg)
1179 		bucket->lseg = pnfs_get_lseg(lseg);
1180 	set_bit(PG_COMMIT_TO_DS, &req->wb_flags);
1181 	cinfo->ds->nwritten++;
1182 
1183 	nfs_request_add_commit_list_locked(req, list, cinfo);
1184 	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1185 	nfs_folio_mark_unstable(nfs_page_to_folio(req), cinfo);
1186 	return;
1187 out_resched:
1188 	mutex_unlock(&NFS_I(cinfo->inode)->commit_mutex);
1189 	cinfo->completion_ops->resched_write(cinfo, req);
1190 }
1191 EXPORT_SYMBOL_GPL(pnfs_layout_mark_request_commit);
1192 
1193 int
pnfs_nfs_generic_sync(struct inode * inode,bool datasync)1194 pnfs_nfs_generic_sync(struct inode *inode, bool datasync)
1195 {
1196 	int ret;
1197 
1198 	if (!pnfs_layoutcommit_outstanding(inode))
1199 		return 0;
1200 	ret = nfs_commit_inode(inode, FLUSH_SYNC);
1201 	if (ret < 0)
1202 		return ret;
1203 	if (datasync)
1204 		return 0;
1205 	return pnfs_layoutcommit_inode(inode, true);
1206 }
1207 EXPORT_SYMBOL_GPL(pnfs_nfs_generic_sync);
1208 
1209