1 // SPDX-License-Identifier: GPL-2.0-or-later
2 /* FS-Cache object state machine handler
8  * object state machine and the in-kernel representations.
32 #define STATE(n) (&__STATE_NAME(n))  macro
35  * Define a work state.  Work states are execution states.  No event processing
36  * is performed by them.  The function attached to a work state returns a
37  * pointer indicating the next state to which the state machine should
38  * transition.  Returning NO_TRANSIT repeats the current state, but goes back
51 #define transit_to(state) ({ prefetch(&STATE(state)->work); STATE(state); })  argument
56  * Define a wait state.  Wait states are event processing states.  No execution
57  * is performed by them.  Wait states are just tables of "if event X occurs,
58  * clear it and transition to state Y".  The dispatcher returns to the
59  * scheduler if none of the events in which the wait state has an interest are
70 #define TRANSIT_TO(state, emask) \  argument
71 	{ .events = (emask), .transit_to = STATE(state) }
74  * The object state machine.
108  * Out-of-band event transition tables.  These are for handling unexpected
109  * events, such as an I/O error.  If an OOB event occurs, the state machine
111  * state (acurrently executing work states will complete first).
113  * In such a situation, object->state remembers the state the machine should
151 	struct fscache_object *parent = object->parent;  in fscache_done_parent_op()
154 	       object->debug_id, parent->debug_id, parent->n_ops);  in fscache_done_parent_op()
156 	spin_lock_nested(&parent->lock, 1);  in fscache_done_parent_op()
157 	parent->n_obj_ops--;  in fscache_done_parent_op()
158 	parent->n_ops--;  in fscache_done_parent_op()
159 	if (parent->n_ops == 0)  in fscache_done_parent_op()
161 	spin_unlock(&parent->lock);  in fscache_done_parent_op()
165  * Object state machine dispatcher.
170 	const struct fscache_state *state, *new_state;  in fscache_object_sm_dispatcher()  local
173 	int event = -1;  in fscache_object_sm_dispatcher()
178 	       object->debug_id, object->state->name, object->events);  in fscache_object_sm_dispatcher()
180 	event_mask = object->event_mask;  in fscache_object_sm_dispatcher()
182 	object->event_mask = 0; /* Mask normal event handling */  in fscache_object_sm_dispatcher()
183 	state = object->state;  in fscache_object_sm_dispatcher()
185 	events = object->events;  in fscache_object_sm_dispatcher()
187 	/* Handle any out-of-band events (typically an error) */  in fscache_object_sm_dispatcher()
188 	if (events & object->oob_event_mask) {  in fscache_object_sm_dispatcher()
190 		       object->debug_id, events & object->oob_event_mask);  in fscache_object_sm_dispatcher()
192 		for (t = object->oob_table; t->events; t++) {  in fscache_object_sm_dispatcher()
193 			if (events & t->events) {  in fscache_object_sm_dispatcher()
194 				state = t->transit_to;  in fscache_object_sm_dispatcher()
195 				ASSERT(state->work != NULL);  in fscache_object_sm_dispatcher()
196 				event = fls(events & t->events) - 1;  in fscache_object_sm_dispatcher()
197 				__clear_bit(event, &object->oob_event_mask);  in fscache_object_sm_dispatcher()
198 				clear_bit(event, &object->events);  in fscache_object_sm_dispatcher()
205 	/* Wait states are just transition tables */  in fscache_object_sm_dispatcher()
206 	if (!state->work) {  in fscache_object_sm_dispatcher()
208 			for (t = state->transitions; t->events; t++) {  in fscache_object_sm_dispatcher()
209 				if (events & t->events) {  in fscache_object_sm_dispatcher()
210 					new_state = t->transit_to;  in fscache_object_sm_dispatcher()
211 					event = fls(events & t->events) - 1;  in fscache_object_sm_dispatcher()
212 					trace_fscache_osm(object, state,  in fscache_object_sm_dispatcher()
214 					clear_bit(event, &object->events);  in fscache_object_sm_dispatcher()
215 					_debug("{OBJ%x} ev %d: %s -> %s",  in fscache_object_sm_dispatcher()
216 					       object->debug_id, event,  in fscache_object_sm_dispatcher()
217 					       state->name, new_state->name);  in fscache_object_sm_dispatcher()
218 					object->state = state = new_state;  in fscache_object_sm_dispatcher()
231 	_debug("{OBJ%x} exec %s", object->debug_id, state->name);  in fscache_object_sm_dispatcher()
233 	trace_fscache_osm(object, state, false, oob, event);  in fscache_object_sm_dispatcher()
234 	new_state = state->work(object, event);  in fscache_object_sm_dispatcher()
235 	event = -1;  in fscache_object_sm_dispatcher()
237 		_debug("{OBJ%x} %s notrans", object->debug_id, state->name);  in fscache_object_sm_dispatcher()
238 		if (unlikely(state == STATE(OBJECT_DEAD))) {  in fscache_object_sm_dispatcher()
243 		event_mask = object->oob_event_mask;  in fscache_object_sm_dispatcher()
247 	_debug("{OBJ%x} %s -> %s",  in fscache_object_sm_dispatcher()
248 	       object->debug_id, state->name, new_state->name);  in fscache_object_sm_dispatcher()
249 	object->state = state = new_state;  in fscache_object_sm_dispatcher()
251 	if (state->work) {  in fscache_object_sm_dispatcher()
252 		if (unlikely(state == STATE(OBJECT_DEAD))) {  in fscache_object_sm_dispatcher()
259 	/* Transited to wait state */  in fscache_object_sm_dispatcher()
260 	event_mask = object->oob_event_mask;  in fscache_object_sm_dispatcher()
261 	for (t = state->transitions; t->events; t++)  in fscache_object_sm_dispatcher()
262 		event_mask |= t->events;  in fscache_object_sm_dispatcher()
265 	object->event_mask = event_mask;  in fscache_object_sm_dispatcher()
267 	events = object->events;  in fscache_object_sm_dispatcher()
282 	_enter("{OBJ%x}", object->debug_id);  in fscache_object_work_func()
291  * fscache_object_init - Initialise a cache object description
298  * See Documentation/filesystems/caching/backend-api.rst for a complete
307 	atomic_inc(&cache->object_count);  in fscache_object_init()
309 	object->state = STATE(WAIT_FOR_INIT);  in fscache_object_init()
310 	object->oob_table = fscache_osm_init_oob;  in fscache_object_init()
311 	object->flags = 1 << FSCACHE_OBJECT_IS_LIVE;  in fscache_object_init()
312 	spin_lock_init(&object->lock);  in fscache_object_init()
313 	INIT_LIST_HEAD(&object->cache_link);  in fscache_object_init()
314 	INIT_HLIST_NODE(&object->cookie_link);  in fscache_object_init()
315 	INIT_WORK(&object->work, fscache_object_work_func);  in fscache_object_init()
316 	INIT_LIST_HEAD(&object->dependents);  in fscache_object_init()
317 	INIT_LIST_HEAD(&object->dep_link);  in fscache_object_init()
318 	INIT_LIST_HEAD(&object->pending_ops);  in fscache_object_init()
319 	object->n_children = 0;  in fscache_object_init()
320 	object->n_ops = object->n_in_progress = object->n_exclusive = 0;  in fscache_object_init()
321 	object->events = 0;  in fscache_object_init()
322 	object->store_limit = 0;  in fscache_object_init()
323 	object->store_limit_l = 0;  in fscache_object_init()
324 	object->cache = cache;  in fscache_object_init()
325 	object->cookie = cookie;  in fscache_object_init()
327 	object->parent = NULL;  in fscache_object_init()
329 	RB_CLEAR_NODE(&object->objlist_link);  in fscache_object_init()
332 	object->oob_event_mask = 0;  in fscache_object_init()
333 	for (t = object->oob_table; t->events; t++)  in fscache_object_init()
334 		object->oob_event_mask |= t->events;  in fscache_object_init()
335 	object->event_mask = object->oob_event_mask;  in fscache_object_init()
336 	for (t = object->state->transitions; t->events; t++)  in fscache_object_init()
337 		object->event_mask |= t->events;  in fscache_object_init()
347 	spin_lock(&object->lock);  in fscache_mark_object_dead()
348 	clear_bit(FSCACHE_OBJECT_IS_LIVE, &object->flags);  in fscache_mark_object_dead()
349 	spin_unlock(&object->lock);  in fscache_mark_object_dead()
358 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_abort_initialisation()
360 	object->oob_event_mask = 0;  in fscache_abort_initialisation()
367  * - check the specified object's parent to see if we can make use of it
369  * - we may need to start the process of creating a parent and we need to wait
378 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_initialise_object()
380 	ASSERT(list_empty(&object->dep_link));  in fscache_initialise_object()
382 	parent = object->parent;  in fscache_initialise_object()
388 	_debug("parent: %s of:%lx", parent->state->name, parent->flags);  in fscache_initialise_object()
400 	_debug("wait");  in fscache_initialise_object()
402 	spin_lock(&parent->lock);  in fscache_initialise_object()
406 	    object->cache->ops->grab_object(object, fscache_obj_get_add_to_deps)) {  in fscache_initialise_object()
407 		list_add(&object->dep_link, &parent->dependents);  in fscache_initialise_object()
411 	spin_unlock(&parent->lock);  in fscache_initialise_object()
420 	_leave(" [wait]");  in fscache_initialise_object()
430 	struct fscache_object *parent = object->parent;  in fscache_parent_ready()
432 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_parent_ready()
436 	spin_lock(&parent->lock);  in fscache_parent_ready()
437 	parent->n_ops++;  in fscache_parent_ready()
438 	parent->n_obj_ops++;  in fscache_parent_ready()
439 	object->lookup_jif = jiffies;  in fscache_parent_ready()
440 	spin_unlock(&parent->lock);  in fscache_parent_ready()
448  * - we hold an "access lock" on the parent object, so the parent object cannot
454 	struct fscache_cookie *cookie = object->cookie;  in fscache_look_up_object()
455 	struct fscache_object *parent = object->parent;  in fscache_look_up_object()
458 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_look_up_object()
460 	object->oob_table = fscache_osm_lookup_oob;  in fscache_look_up_object()
463 	ASSERTCMP(parent->n_ops, >, 0);  in fscache_look_up_object()
464 	ASSERTCMP(parent->n_obj_ops, >, 0);  in fscache_look_up_object()
470 	    test_bit(FSCACHE_IOERROR, &object->cache->flags) ||  in fscache_look_up_object()
477 	       cookie->def->name, object->cache->tag->name);  in fscache_look_up_object()
481 	ret = object->cache->ops->lookup_object(object);  in fscache_look_up_object()
486 	if (ret == -ETIMEDOUT) {  in fscache_look_up_object()
504  * fscache_object_lookup_negative - Note negative cookie lookup
512 	struct fscache_cookie *cookie = object->cookie;  in fscache_object_lookup_negative()
514 	_enter("{OBJ%x,%s}", object->debug_id, object->state->name);  in fscache_object_lookup_negative()
516 	if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {  in fscache_object_lookup_negative()
522 		set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in fscache_object_lookup_negative()
523 		clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);  in fscache_object_lookup_negative()
525 		_debug("wake up lookup %p", &cookie->flags);  in fscache_object_lookup_negative()
526 		clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);  in fscache_object_lookup_negative()
527 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);  in fscache_object_lookup_negative()
534  * fscache_obtained_object - Note successful object lookup or creation
545 	struct fscache_cookie *cookie = object->cookie;  in fscache_obtained_object()
547 	_enter("{OBJ%x,%s}", object->debug_id, object->state->name);  in fscache_obtained_object()
551 	if (!test_and_set_bit(FSCACHE_OBJECT_IS_LOOKED_UP, &object->flags)) {  in fscache_obtained_object()
555 		clear_bit_unlock(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in fscache_obtained_object()
556 		clear_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);  in fscache_obtained_object()
561 		clear_bit_unlock(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags);  in fscache_obtained_object()
562 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);  in fscache_obtained_object()
567 	set_bit(FSCACHE_OBJECT_IS_AVAILABLE, &object->flags);  in fscache_obtained_object()
578 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_object_available()
580 	object->oob_table = fscache_osm_run_oob;  in fscache_object_available()
582 	spin_lock(&object->lock);  in fscache_object_available()
585 	if (object->n_in_progress == 0) {  in fscache_object_available()
586 		if (object->n_ops > 0) {  in fscache_object_available()
587 			ASSERTCMP(object->n_ops, >=, object->n_obj_ops);  in fscache_object_available()
590 			ASSERT(list_empty(&object->pending_ops));  in fscache_object_available()
593 	spin_unlock(&object->lock);  in fscache_object_available()
596 	object->cache->ops->lookup_complete(object);  in fscache_object_available()
599 	fscache_hist(fscache_obj_instantiate_histogram, object->lookup_jif);  in fscache_object_available()
612 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_jumpstart_dependents()
627 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_lookup_failure()
629 	object->oob_event_mask = 0;  in fscache_lookup_failure()
632 	object->cache->ops->lookup_complete(object);  in fscache_lookup_failure()
635 	set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags);  in fscache_lookup_failure()
637 	cookie = object->cookie;  in fscache_lookup_failure()
638 	set_bit(FSCACHE_COOKIE_UNAVAILABLE, &cookie->flags);  in fscache_lookup_failure()
639 	if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags))  in fscache_lookup_failure()
640 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);  in fscache_lookup_failure()
647  * Wait for completion of all active operations on this object and the death of
654 	       object->debug_id, object->n_ops, object->n_children, event);  in fscache_kill_object()
657 	object->oob_event_mask = 0;  in fscache_kill_object()
659 	if (test_bit(FSCACHE_OBJECT_RETIRED, &object->flags)) {  in fscache_kill_object()
661 		clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);  in fscache_kill_object()
665 	if (list_empty(&object->dependents) &&  in fscache_kill_object()
666 	    object->n_ops == 0 &&  in fscache_kill_object()
667 	    object->n_children == 0)  in fscache_kill_object()
670 	if (object->n_in_progress == 0) {  in fscache_kill_object()
671 		spin_lock(&object->lock);  in fscache_kill_object()
672 		if (object->n_ops > 0 && object->n_in_progress == 0)  in fscache_kill_object()
674 		spin_unlock(&object->lock);  in fscache_kill_object()
677 	if (!list_empty(&object->dependents))  in fscache_kill_object()
689 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_kill_dependents()
702 	struct fscache_object *parent = object->parent;  in fscache_drop_object()
703 	struct fscache_cookie *cookie = object->cookie;  in fscache_drop_object()
704 	struct fscache_cache *cache = object->cache;  in fscache_drop_object()
707 	_enter("{OBJ%x,%d},%d", object->debug_id, object->n_children, event);  in fscache_drop_object()
710 	ASSERT(!hlist_unhashed(&object->cookie_link));  in fscache_drop_object()
712 	if (test_bit(FSCACHE_COOKIE_AUX_UPDATED, &cookie->flags)) {  in fscache_drop_object()
720 	spin_lock(&cookie->lock);  in fscache_drop_object()
721 	hlist_del_init(&object->cookie_link);  in fscache_drop_object()
722 	if (hlist_empty(&cookie->backing_objects) &&  in fscache_drop_object()
723 	    test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))  in fscache_drop_object()
725 	spin_unlock(&cookie->lock);  in fscache_drop_object()
728 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);  in fscache_drop_object()
729 	if (test_and_clear_bit(FSCACHE_COOKIE_LOOKING_UP, &cookie->flags))  in fscache_drop_object()
730 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_LOOKING_UP);  in fscache_drop_object()
736 	spin_lock(&object->lock);  in fscache_drop_object()
737 	spin_unlock(&object->lock);  in fscache_drop_object()
740 	spin_lock(&cache->object_list_lock);  in fscache_drop_object()
741 	list_del_init(&object->cache_link);  in fscache_drop_object()
742 	spin_unlock(&cache->object_list_lock);  in fscache_drop_object()
745 	cache->ops->drop_object(object);  in fscache_drop_object()
751 		       parent->debug_id, parent->n_children);  in fscache_drop_object()
753 		spin_lock(&parent->lock);  in fscache_drop_object()
754 		parent->n_children--;  in fscache_drop_object()
755 		if (parent->n_children == 0)  in fscache_drop_object()
757 		spin_unlock(&parent->lock);  in fscache_drop_object()
758 		object->parent = NULL;  in fscache_drop_object()
778 	ret = object->cache->ops->grab_object(object, why) ? 0 : -EAGAIN;  in fscache_get_object()
790 	object->cache->ops->put_object(object, why);  in fscache_put_object()
795  * fscache_object_destroy - Note that a cache object is about to be destroyed
805 	fscache_cookie_put(object->cookie, fscache_cookie_put_object);  in fscache_object_destroy()
806 	object->cookie = NULL;  in fscache_object_destroy()
811  * enqueue an object for metadata-type processing
815 	_enter("{OBJ%x}", object->debug_id);  in fscache_enqueue_object()
821 		if (queue_work(fscache_object_wq, &object->work)) {  in fscache_enqueue_object()
832  * fscache_object_sleep_till_congested - Sleep until object wq is congested
846 	DEFINE_WAIT(wait);  in fscache_object_sleep_till_congested()
851 	add_wait_queue_exclusive(cong_wq, &wait);  in fscache_object_sleep_till_congested()
854 	finish_wait(cong_wq, &wait);  in fscache_object_sleep_till_congested()
861  * Enqueue the dependents of an object for metadata-type processing.
872 	_enter("{OBJ%x}", object->debug_id);  in fscache_enqueue_dependents()
874 	if (list_empty(&object->dependents))  in fscache_enqueue_dependents()
877 	spin_lock(&object->lock);  in fscache_enqueue_dependents()
879 	while (!list_empty(&object->dependents)) {  in fscache_enqueue_dependents()
880 		dep = list_entry(object->dependents.next,  in fscache_enqueue_dependents()
882 		list_del_init(&dep->dep_link);  in fscache_enqueue_dependents()
887 		if (!list_empty(&object->dependents) && need_resched()) {  in fscache_enqueue_dependents()
893 	spin_unlock(&object->lock);  in fscache_enqueue_dependents()
902 	_enter("{OBJ%x}", object->debug_id);  in fscache_dequeue_object()
904 	if (!list_empty(&object->dep_link)) {  in fscache_dequeue_object()
905 		spin_lock(&object->parent->lock);  in fscache_dequeue_object()
906 		list_del_init(&object->dep_link);  in fscache_dequeue_object()
907 		spin_unlock(&object->parent->lock);  in fscache_dequeue_object()
914  * fscache_check_aux - Ask the netfs whether an object on disk is still valid
919  * This function consults the netfs about the coherency state of an object.
920  * The caller must be holding a ref on cookie->n_active (held by
930 	if (!object->cookie->def->check_aux) {  in fscache_check_aux()
935 	result = object->cookie->def->check_aux(object->cookie->netfs_data,  in fscache_check_aux()
968 	struct fscache_cookie *cookie = object->cookie;  in _fscache_invalidate_object()
970 	_enter("{OBJ%x},%d", object->debug_id, event);  in _fscache_invalidate_object()
976 		ASSERT(radix_tree_empty(&object->cookie->stores));  in _fscache_invalidate_object()
977 		set_bit(FSCACHE_OBJECT_RETIRED, &object->flags);  in _fscache_invalidate_object()
984 	clear_bit(FSCACHE_OBJECT_PENDING_WRITE, &object->flags);  in _fscache_invalidate_object()
987 	/* Now we have to wait for in-progress reads and writes */  in _fscache_invalidate_object()
992 	fscache_operation_init(cookie, op, object->cache->ops->invalidate_object,  in _fscache_invalidate_object()
994 	op->flags = FSCACHE_OP_ASYNC |  in _fscache_invalidate_object()
999 	spin_lock(&cookie->lock);  in _fscache_invalidate_object()
1002 	spin_unlock(&cookie->lock);  in _fscache_invalidate_object()
1006 	 * stored in the cache and thus we can reinstate the data-check-skip  in _fscache_invalidate_object()
1009 	set_bit(FSCACHE_COOKIE_NO_DATA_YET, &cookie->flags);  in _fscache_invalidate_object()
1014 	if (test_and_clear_bit(FSCACHE_COOKIE_INVALIDATING, &cookie->flags))  in _fscache_invalidate_object()
1015 		wake_up_bit(&cookie->flags, FSCACHE_COOKIE_INVALIDATING);  in _fscache_invalidate_object()
1027 	spin_unlock(&cookie->lock);  in _fscache_invalidate_object()
1053 	object->cache->ops->update_object(object);  in fscache_update_aux_data()
1063 	_enter("{OBJ%x},%d", object->debug_id, event);  in fscache_update_object()
1072  * fscache_object_retrying_stale - Note retrying stale object
1075  * Note that an object lookup found an on-disk object that was adjudged to be
1085  * fscache_object_mark_killed - Note that an object was killed
1095 	if (test_and_set_bit(FSCACHE_OBJECT_KILLED_BY_CACHE, &object->flags)) {  in fscache_object_mark_killed()
1097 		       object->cache->identifier);  in fscache_object_mark_killed()
1128 			      &object->flags))  in fscache_object_dead()
1131 	WARN(true, "FS-Cache object redispatched after death");  in fscache_object_dead()