171  * @interval_sub: The interval subscription
175  * subscription. If the mm invokes invalidation during the critical section
196 	 * If the subscription has a different seq value under the user_lock  in mmu_interval_read_begin()
199 	 * If the subscription currently has the same seq value as the  in mmu_interval_read_begin()
304 	struct mmu_notifier *subscription;  in mn_hlist_release()  local
312 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_release()
320 		if (subscription->ops->release)  in mn_hlist_release()
321 			subscription->ops->release(subscription, mm);  in mn_hlist_release()
325 		subscription = hlist_entry(subscriptions->list.first,  in mn_hlist_release()
333 		hlist_del_init_rcu(&subscription->hlist);  in mn_hlist_release()
371 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_flush_young()  local
375 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_flush_young()
378 		if (subscription->ops->clear_flush_young)  in __mmu_notifier_clear_flush_young()
379 			young |= subscription->ops->clear_flush_young(  in __mmu_notifier_clear_flush_young()
380 				subscription, mm, start, end);  in __mmu_notifier_clear_flush_young()
391 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_young()  local
395 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_young()
398 		if (subscription->ops->clear_young)  in __mmu_notifier_clear_young()
399 			young |= subscription->ops->clear_young(subscription,  in __mmu_notifier_clear_young()
410 	struct mmu_notifier *subscription;  in __mmu_notifier_test_young()  local
414 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_test_young()
417 		if (subscription->ops->test_young) {  in __mmu_notifier_test_young()
418 			young = subscription->ops->test_young(subscription, mm,  in __mmu_notifier_test_young()
464 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_range_start()  local
469 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_range_start()
471 		const struct mmu_notifier_ops *ops = subscription->ops;  in mn_hlist_invalidate_range_start()
478 			_ret = ops->invalidate_range_start(subscription, range);  in mn_hlist_invalidate_range_start()
507 		hlist_for_each_entry_rcu(subscription, &subscriptions->list,  in mn_hlist_invalidate_range_start()
509 			if (!subscription->ops->invalidate_range_end)  in mn_hlist_invalidate_range_start()
512 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_range_start()
541 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_end()  local
545 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_end()
547 		if (subscription->ops->invalidate_range_end) {  in mn_hlist_invalidate_end()
550 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_end()
576 	struct mmu_notifier *subscription;  in __mmu_notifier_arch_invalidate_secondary_tlbs()  local
580 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_arch_invalidate_secondary_tlbs()
583 		if (subscription->ops->arch_invalidate_secondary_tlbs)  in __mmu_notifier_arch_invalidate_secondary_tlbs()
584 			subscription->ops->arch_invalidate_secondary_tlbs(  in __mmu_notifier_arch_invalidate_secondary_tlbs()
585 				subscription, mm,  in __mmu_notifier_arch_invalidate_secondary_tlbs()
596 int __mmu_notifier_register(struct mmu_notifier *subscription,  in __mmu_notifier_register()  argument
609 	if (WARN_ON_ONCE(subscription &&  in __mmu_notifier_register()
610 			 (subscription->ops->arch_invalidate_secondary_tlbs &&  in __mmu_notifier_register()
611 			 (subscription->ops->invalidate_range_start ||  in __mmu_notifier_register()
612 			  subscription->ops->invalidate_range_end))))  in __mmu_notifier_register()
657 	if (subscription) {  in __mmu_notifier_register()
660 		subscription->mm = mm;  in __mmu_notifier_register()
661 		subscription->users = 1;  in __mmu_notifier_register()
664 		hlist_add_head_rcu(&subscription->hlist,  in __mmu_notifier_register()
682  * @subscription: The notifier to attach
696  * While the caller has a mmu_notifier get the subscription->mm pointer will remain
699 int mmu_notifier_register(struct mmu_notifier *subscription,  in mmu_notifier_register()  argument
705 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_register()
714 	struct mmu_notifier *subscription;  in find_get_mmu_notifier()  local
717 	hlist_for_each_entry_rcu(subscription,  in find_get_mmu_notifier()
720 		if (subscription->ops != ops)  in find_get_mmu_notifier()
723 		if (likely(subscription->users != UINT_MAX))  in find_get_mmu_notifier()
724 			subscription->users++;  in find_get_mmu_notifier()
726 			subscription = ERR_PTR(-EOVERFLOW);  in find_get_mmu_notifier()
728 		return subscription;  in find_get_mmu_notifier()
754 	struct mmu_notifier *subscription;  in mmu_notifier_get_locked()  local
760 		subscription = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
761 		if (subscription)  in mmu_notifier_get_locked()
762 			return subscription;  in mmu_notifier_get_locked()
765 	subscription = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
766 	if (IS_ERR(subscription))  in mmu_notifier_get_locked()
767 		return subscription;  in mmu_notifier_get_locked()
768 	subscription->ops = ops;  in mmu_notifier_get_locked()
769 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_get_locked()
772 	return subscription;  in mmu_notifier_get_locked()
774 	subscription->ops->free_notifier(subscription);  in mmu_notifier_get_locked()
797 void mmu_notifier_unregister(struct mmu_notifier *subscription,  in mmu_notifier_unregister()  argument
802 	if (!hlist_unhashed(&subscription->hlist)) {  in mmu_notifier_unregister()
814 		if (subscription->ops->release)  in mmu_notifier_unregister()
815 			subscription->ops->release(subscription, mm);  in mmu_notifier_unregister()
823 		hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_unregister()
841 	struct mmu_notifier *subscription =  in mmu_notifier_free_rcu()  local
843 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_free_rcu()
845 	subscription->ops->free_notifier(subscription);  in mmu_notifier_free_rcu()
852  * @subscription: The notifier to act on
872 void mmu_notifier_put(struct mmu_notifier *subscription)  in mmu_notifier_put()  argument
874 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_put()
877 	if (WARN_ON(!subscription->users) || --subscription->users)  in mmu_notifier_put()
879 	hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_put()
882 	call_srcu(&srcu, &subscription->rcu, mmu_notifier_free_rcu);  in mmu_notifier_put()
942 		 * The starting seq for a subscription not under invalidation  in __mmu_interval_notifier_insert()
958  * @interval_sub: Interval subscription to register
1030  * @interval_sub: Interval subscription to unregister