1 // SPDX-License-Identifier: GPL-2.0-only
2 /* Miscellaneous routines.
3  *
4  * Copyright (C) 2023 Red Hat, Inc. All Rights Reserved.
5  * Written by David Howells (dhowells@redhat.com)
6  */
7 
8 #include <linux/swap.h>
9 #include "internal.h"
10 
11 /**
12  * netfs_alloc_folioq_buffer - Allocate buffer space into a folio queue
13  * @mapping: Address space to set on the folio (or NULL).
14  * @_buffer: Pointer to the folio queue to add to (may point to a NULL; updated).
15  * @_cur_size: Current size of the buffer (updated).
16  * @size: Target size of the buffer.
17  * @gfp: The allocation constraints.
18  */
19 int netfs_alloc_folioq_buffer(struct address_space *mapping,
20 			      struct folio_queue **_buffer,
21 			      size_t *_cur_size, ssize_t size, gfp_t gfp)
22 {
23 	struct folio_queue *tail = *_buffer, *p;
24 
25 	size = round_up(size, PAGE_SIZE);
26 	if (*_cur_size >= size)
27 		return 0;
28 
29 	if (tail)
30 		while (tail->next)
31 			tail = tail->next;
32 
33 	do {
34 		struct folio *folio;
35 		int order = 0, slot;
36 
37 		if (!tail || folioq_full(tail)) {
38 			p = netfs_folioq_alloc(0, GFP_NOFS, netfs_trace_folioq_alloc_buffer);
39 			if (!p)
40 				return -ENOMEM;
41 			if (tail) {
42 				tail->next = p;
43 				p->prev = tail;
44 			} else {
45 				*_buffer = p;
46 			}
47 			tail = p;
48 		}
49 
50 		if (size - *_cur_size > PAGE_SIZE)
51 			order = umin(ilog2(size - *_cur_size) - PAGE_SHIFT,
52 				     MAX_PAGECACHE_ORDER);
53 
54 		folio = folio_alloc(gfp, order);
55 		if (!folio && order > 0)
56 			folio = folio_alloc(gfp, 0);
57 		if (!folio)
58 			return -ENOMEM;
59 
60 		folio->mapping = mapping;
61 		folio->index = *_cur_size / PAGE_SIZE;
62 		trace_netfs_folio(folio, netfs_folio_trace_alloc_buffer);
63 		slot = folioq_append_mark(tail, folio);
64 		*_cur_size += folioq_folio_size(tail, slot);
65 	} while (*_cur_size < size);
66 
67 	return 0;
68 }
69 EXPORT_SYMBOL(netfs_alloc_folioq_buffer);
70 
71 /**
72  * netfs_free_folioq_buffer - Free a folio queue.
73  * @fq: The start of the folio queue to free
74  *
75  * Free up a chain of folio_queues and, if marked, the marked folios they point
76  * to.
77  */
78 void netfs_free_folioq_buffer(struct folio_queue *fq)
79 {
80 	struct folio_queue *next;
81 	struct folio_batch fbatch;
82 
83 	folio_batch_init(&fbatch);
84 
85 	for (; fq; fq = next) {
86 		for (int slot = 0; slot < folioq_count(fq); slot++) {
87 			struct folio *folio = folioq_folio(fq, slot);
88 
89 			if (!folio ||
90 			    !folioq_is_marked(fq, slot))
91 				continue;
92 
93 			trace_netfs_folio(folio, netfs_folio_trace_put);
94 			if (folio_batch_add(&fbatch, folio))
95 				folio_batch_release(&fbatch);
96 		}
97 
98 		netfs_stat_d(&netfs_n_folioq);
99 		next = fq->next;
100 		kfree(fq);
101 	}
102 
103 	folio_batch_release(&fbatch);
104 }
105 EXPORT_SYMBOL(netfs_free_folioq_buffer);
106 
107 /*
108  * Reset the subrequest iterator to refer just to the region remaining to be
109  * read.  The iterator may or may not have been advanced by socket ops or
110  * extraction ops to an extent that may or may not match the amount actually
111  * read.
112  */
113 void netfs_reset_iter(struct netfs_io_subrequest *subreq)
114 {
115 	struct iov_iter *io_iter = &subreq->io_iter;
116 	size_t remain = subreq->len - subreq->transferred;
117 
118 	if (io_iter->count > remain)
119 		iov_iter_advance(io_iter, io_iter->count - remain);
120 	else if (io_iter->count < remain)
121 		iov_iter_revert(io_iter, remain - io_iter->count);
122 	iov_iter_truncate(&subreq->io_iter, remain);
123 }
124 
125 /**
126  * netfs_dirty_folio - Mark folio dirty and pin a cache object for writeback
127  * @mapping: The mapping the folio belongs to.
128  * @folio: The folio being dirtied.
129  *
130  * Set the dirty flag on a folio and pin an in-use cache object in memory so
131  * that writeback can later write to it.  This is intended to be called from
132  * the filesystem's ->dirty_folio() method.
133  *
134  * Return: true if the dirty flag was set on the folio, false otherwise.
135  */
136 bool netfs_dirty_folio(struct address_space *mapping, struct folio *folio)
137 {
138 	struct inode *inode = mapping->host;
139 	struct netfs_inode *ictx = netfs_inode(inode);
140 	struct fscache_cookie *cookie = netfs_i_cookie(ictx);
141 	bool need_use = false;
142 
143 	_enter("");
144 
145 	if (!filemap_dirty_folio(mapping, folio))
146 		return false;
147 	if (!fscache_cookie_valid(cookie))
148 		return true;
149 
150 	if (!(inode->i_state & I_PINNING_NETFS_WB)) {
151 		spin_lock(&inode->i_lock);
152 		if (!(inode->i_state & I_PINNING_NETFS_WB)) {
153 			inode->i_state |= I_PINNING_NETFS_WB;
154 			need_use = true;
155 		}
156 		spin_unlock(&inode->i_lock);
157 
158 		if (need_use)
159 			fscache_use_cookie(cookie, true);
160 	}
161 	return true;
162 }
163 EXPORT_SYMBOL(netfs_dirty_folio);
164 
165 /**
166  * netfs_unpin_writeback - Unpin writeback resources
167  * @inode: The inode on which the cookie resides
168  * @wbc: The writeback control
169  *
170  * Unpin the writeback resources pinned by netfs_dirty_folio().  This is
171  * intended to be called as/by the netfs's ->write_inode() method.
172  */
173 int netfs_unpin_writeback(struct inode *inode, struct writeback_control *wbc)
174 {
175 	struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode));
176 
177 	if (wbc->unpinned_netfs_wb)
178 		fscache_unuse_cookie(cookie, NULL, NULL);
179 	return 0;
180 }
181 EXPORT_SYMBOL(netfs_unpin_writeback);
182 
183 /**
184  * netfs_clear_inode_writeback - Clear writeback resources pinned by an inode
185  * @inode: The inode to clean up
186  * @aux: Auxiliary data to apply to the inode
187  *
188  * Clear any writeback resources held by an inode when the inode is evicted.
189  * This must be called before clear_inode() is called.
190  */
191 void netfs_clear_inode_writeback(struct inode *inode, const void *aux)
192 {
193 	struct fscache_cookie *cookie = netfs_i_cookie(netfs_inode(inode));
194 
195 	if (inode->i_state & I_PINNING_NETFS_WB) {
196 		loff_t i_size = i_size_read(inode);
197 		fscache_unuse_cookie(cookie, aux, &i_size);
198 	}
199 }
200 EXPORT_SYMBOL(netfs_clear_inode_writeback);
201 
202 /**
203  * netfs_invalidate_folio - Invalidate or partially invalidate a folio
204  * @folio: Folio proposed for release
205  * @offset: Offset of the invalidated region
206  * @length: Length of the invalidated region
207  *
208  * Invalidate part or all of a folio for a network filesystem.  The folio will
209  * be removed afterwards if the invalidated region covers the entire folio.
210  */
211 void netfs_invalidate_folio(struct folio *folio, size_t offset, size_t length)
212 {
213 	struct netfs_folio *finfo;
214 	struct netfs_inode *ctx = netfs_inode(folio_inode(folio));
215 	size_t flen = folio_size(folio);
216 
217 	_enter("{%lx},%zx,%zx", folio->index, offset, length);
218 
219 	if (offset == 0 && length == flen) {
220 		unsigned long long i_size = i_size_read(&ctx->inode);
221 		unsigned long long fpos = folio_pos(folio), end;
222 
223 		end = umin(fpos + flen, i_size);
224 		if (fpos < i_size && end > ctx->zero_point)
225 			ctx->zero_point = end;
226 	}
227 
228 	folio_wait_private_2(folio); /* [DEPRECATED] */
229 
230 	if (!folio_test_private(folio))
231 		return;
232 
233 	finfo = netfs_folio_info(folio);
234 
235 	if (offset == 0 && length >= flen)
236 		goto erase_completely;
237 
238 	if (finfo) {
239 		/* We have a partially uptodate page from a streaming write. */
240 		unsigned int fstart = finfo->dirty_offset;
241 		unsigned int fend = fstart + finfo->dirty_len;
242 		unsigned int iend = offset + length;
243 
244 		if (offset >= fend)
245 			return;
246 		if (iend <= fstart)
247 			return;
248 
249 		/* The invalidation region overlaps the data.  If the region
250 		 * covers the start of the data, we either move along the start
251 		 * or just erase the data entirely.
252 		 */
253 		if (offset <= fstart) {
254 			if (iend >= fend)
255 				goto erase_completely;
256 			/* Move the start of the data. */
257 			finfo->dirty_len = fend - iend;
258 			finfo->dirty_offset = offset;
259 			return;
260 		}
261 
262 		/* Reduce the length of the data if the invalidation region
263 		 * covers the tail part.
264 		 */
265 		if (iend >= fend) {
266 			finfo->dirty_len = offset - fstart;
267 			return;
268 		}
269 
270 		/* A partial write was split.  The caller has already zeroed
271 		 * it, so just absorb the hole.
272 		 */
273 	}
274 	return;
275 
276 erase_completely:
277 	netfs_put_group(netfs_folio_group(folio));
278 	folio_detach_private(folio);
279 	folio_clear_uptodate(folio);
280 	kfree(finfo);
281 	return;
282 }
283 EXPORT_SYMBOL(netfs_invalidate_folio);
284 
285 /**
286  * netfs_release_folio - Try to release a folio
287  * @folio: Folio proposed for release
288  * @gfp: Flags qualifying the release
289  *
290  * Request release of a folio and clean up its private state if it's not busy.
291  * Returns true if the folio can now be released, false if not
292  */
293 bool netfs_release_folio(struct folio *folio, gfp_t gfp)
294 {
295 	struct netfs_inode *ctx = netfs_inode(folio_inode(folio));
296 	unsigned long long end;
297 
298 	if (folio_test_dirty(folio))
299 		return false;
300 
301 	end = umin(folio_pos(folio) + folio_size(folio), i_size_read(&ctx->inode));
302 	if (end > ctx->zero_point)
303 		ctx->zero_point = end;
304 
305 	if (folio_test_private(folio))
306 		return false;
307 	if (unlikely(folio_test_private_2(folio))) { /* [DEPRECATED] */
308 		if (current_is_kswapd() || !(gfp & __GFP_FS))
309 			return false;
310 		folio_wait_private_2(folio);
311 	}
312 	fscache_note_page_release(netfs_i_cookie(ctx));
313 	return true;
314 }
315 EXPORT_SYMBOL(netfs_release_folio);
316 
317 /*
318  * Wake the collection work item.
319  */
320 void netfs_wake_collector(struct netfs_io_request *rreq)
321 {
322 	if (test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags) &&
323 	    !test_bit(NETFS_RREQ_RETRYING, &rreq->flags)) {
324 		queue_work(system_unbound_wq, &rreq->work);
325 	} else {
326 		trace_netfs_rreq(rreq, netfs_rreq_trace_wake_queue);
327 		wake_up(&rreq->waitq);
328 	}
329 }
330 
331 /*
332  * Mark a subrequest as no longer being in progress and, if need be, wake the
333  * collector.
334  */
335 void netfs_subreq_clear_in_progress(struct netfs_io_subrequest *subreq)
336 {
337 	struct netfs_io_request *rreq = subreq->rreq;
338 	struct netfs_io_stream *stream = &rreq->io_streams[subreq->stream_nr];
339 
340 	clear_bit_unlock(NETFS_SREQ_IN_PROGRESS, &subreq->flags);
341 	smp_mb__after_atomic(); /* Clear IN_PROGRESS before task state */
342 
343 	/* If we are at the head of the queue, wake up the collector. */
344 	if (list_is_first(&subreq->rreq_link, &stream->subrequests) ||
345 	    test_bit(NETFS_RREQ_RETRYING, &rreq->flags))
346 		netfs_wake_collector(rreq);
347 }
348 
349 /*
350  * Wait for all outstanding I/O in a stream to quiesce.
351  */
352 void netfs_wait_for_in_progress_stream(struct netfs_io_request *rreq,
353 				       struct netfs_io_stream *stream)
354 {
355 	struct netfs_io_subrequest *subreq;
356 	DEFINE_WAIT(myself);
357 
358 	list_for_each_entry(subreq, &stream->subrequests, rreq_link) {
359 		if (!test_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags))
360 			continue;
361 
362 		trace_netfs_rreq(rreq, netfs_rreq_trace_wait_queue);
363 		for (;;) {
364 			prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
365 
366 			if (!test_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags))
367 				break;
368 
369 			trace_netfs_sreq(subreq, netfs_sreq_trace_wait_for);
370 			schedule();
371 			trace_netfs_rreq(rreq, netfs_rreq_trace_woke_queue);
372 		}
373 	}
374 
375 	finish_wait(&rreq->waitq, &myself);
376 }
377 
378 /*
379  * Perform collection in app thread if not offloaded to workqueue.
380  */
381 static int netfs_collect_in_app(struct netfs_io_request *rreq,
382 				bool (*collector)(struct netfs_io_request *rreq))
383 {
384 	bool need_collect = false, inactive = true;
385 
386 	for (int i = 0; i < NR_IO_STREAMS; i++) {
387 		struct netfs_io_subrequest *subreq;
388 		struct netfs_io_stream *stream = &rreq->io_streams[i];
389 
390 		if (!stream->active)
391 			continue;
392 		inactive = false;
393 		trace_netfs_collect_stream(rreq, stream);
394 		subreq = list_first_entry_or_null(&stream->subrequests,
395 						  struct netfs_io_subrequest,
396 						  rreq_link);
397 		if (subreq &&
398 		    (!test_bit(NETFS_SREQ_IN_PROGRESS, &subreq->flags) ||
399 		     test_bit(NETFS_SREQ_MADE_PROGRESS, &subreq->flags))) {
400 			need_collect = true;
401 			break;
402 		}
403 	}
404 
405 	if (!need_collect && !inactive)
406 		return 0; /* Sleep */
407 
408 	__set_current_state(TASK_RUNNING);
409 	if (collector(rreq)) {
410 		/* Drop the ref from the NETFS_RREQ_IN_PROGRESS flag. */
411 		netfs_put_request(rreq, netfs_rreq_trace_put_work_ip);
412 		return 1; /* Done */
413 	}
414 
415 	if (inactive) {
416 		WARN(true, "Failed to collect inactive req R=%08x\n",
417 		     rreq->debug_id);
418 		cond_resched();
419 	}
420 	return 2; /* Again */
421 }
422 
423 /*
424  * Wait for a request to complete, successfully or otherwise.
425  */
426 static ssize_t netfs_wait_for_request(struct netfs_io_request *rreq,
427 				      bool (*collector)(struct netfs_io_request *rreq))
428 {
429 	DEFINE_WAIT(myself);
430 	ssize_t ret;
431 
432 	for (;;) {
433 		trace_netfs_rreq(rreq, netfs_rreq_trace_wait_queue);
434 		prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
435 
436 		if (!test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags)) {
437 			switch (netfs_collect_in_app(rreq, collector)) {
438 			case 0:
439 				break;
440 			case 1:
441 				goto all_collected;
442 			case 2:
443 				continue;
444 			}
445 		}
446 
447 		if (!test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags))
448 			break;
449 
450 		schedule();
451 		trace_netfs_rreq(rreq, netfs_rreq_trace_woke_queue);
452 	}
453 
454 all_collected:
455 	finish_wait(&rreq->waitq, &myself);
456 
457 	ret = rreq->error;
458 	if (ret == 0) {
459 		ret = rreq->transferred;
460 		switch (rreq->origin) {
461 		case NETFS_DIO_READ:
462 		case NETFS_DIO_WRITE:
463 		case NETFS_READ_SINGLE:
464 		case NETFS_UNBUFFERED_READ:
465 		case NETFS_UNBUFFERED_WRITE:
466 			break;
467 		default:
468 			if (rreq->submitted < rreq->len) {
469 				trace_netfs_failure(rreq, NULL, ret, netfs_fail_short_read);
470 				ret = -EIO;
471 			}
472 			break;
473 		}
474 	}
475 
476 	return ret;
477 }
478 
479 ssize_t netfs_wait_for_read(struct netfs_io_request *rreq)
480 {
481 	return netfs_wait_for_request(rreq, netfs_read_collection);
482 }
483 
484 ssize_t netfs_wait_for_write(struct netfs_io_request *rreq)
485 {
486 	return netfs_wait_for_request(rreq, netfs_write_collection);
487 }
488 
489 /*
490  * Wait for a paused operation to unpause or complete in some manner.
491  */
492 static void netfs_wait_for_pause(struct netfs_io_request *rreq,
493 				 bool (*collector)(struct netfs_io_request *rreq))
494 {
495 	DEFINE_WAIT(myself);
496 
497 	trace_netfs_rreq(rreq, netfs_rreq_trace_wait_pause);
498 
499 	for (;;) {
500 		trace_netfs_rreq(rreq, netfs_rreq_trace_wait_queue);
501 		prepare_to_wait(&rreq->waitq, &myself, TASK_UNINTERRUPTIBLE);
502 
503 		if (!test_bit(NETFS_RREQ_OFFLOAD_COLLECTION, &rreq->flags)) {
504 			switch (netfs_collect_in_app(rreq, collector)) {
505 			case 0:
506 				break;
507 			case 1:
508 				goto all_collected;
509 			case 2:
510 				continue;
511 			}
512 		}
513 
514 		if (!test_bit(NETFS_RREQ_IN_PROGRESS, &rreq->flags) ||
515 		    !test_bit(NETFS_RREQ_PAUSE, &rreq->flags))
516 			break;
517 
518 		schedule();
519 		trace_netfs_rreq(rreq, netfs_rreq_trace_woke_queue);
520 	}
521 
522 all_collected:
523 	finish_wait(&rreq->waitq, &myself);
524 }
525 
526 void netfs_wait_for_paused_read(struct netfs_io_request *rreq)
527 {
528 	return netfs_wait_for_pause(rreq, netfs_read_collection);
529 }
530 
531 void netfs_wait_for_paused_write(struct netfs_io_request *rreq)
532 {
533 	return netfs_wait_for_pause(rreq, netfs_write_collection);
534 }
535