169  * @interval_sub: The interval subscription
173  * subscription. If the mm invokes invalidation during the critical section
194 	 * If the subscription has a different seq value under the user_lock  in mmu_interval_read_begin()
197 	 * If the subscription currently has the same seq value as the  in mmu_interval_read_begin()
302 	struct mmu_notifier *subscription;  in mn_hlist_release()  local
310 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_release()
318 		if (subscription->ops->release)  in mn_hlist_release()
319 			subscription->ops->release(subscription, mm);  in mn_hlist_release()
323 		subscription = hlist_entry(subscriptions->list.first,  in mn_hlist_release()
331 		hlist_del_init_rcu(&subscription->hlist);  in mn_hlist_release()
369 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_flush_young()  local
373 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_flush_young()
376 		if (subscription->ops->clear_flush_young)  in __mmu_notifier_clear_flush_young()
377 			young |= subscription->ops->clear_flush_young(  in __mmu_notifier_clear_flush_young()
378 				subscription, mm, start, end);  in __mmu_notifier_clear_flush_young()
389 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_young()  local
393 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_clear_young()
396 		if (subscription->ops->clear_young)  in __mmu_notifier_clear_young()
397 			young |= subscription->ops->clear_young(subscription,  in __mmu_notifier_clear_young()
408 	struct mmu_notifier *subscription;  in __mmu_notifier_test_young()  local
412 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_test_young()
415 		if (subscription->ops->test_young) {  in __mmu_notifier_test_young()
416 			young = subscription->ops->test_young(subscription, mm,  in __mmu_notifier_test_young()
430 	struct mmu_notifier *subscription;  in __mmu_notifier_change_pte()  local
434 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_change_pte()
437 		if (subscription->ops->change_pte)  in __mmu_notifier_change_pte()
438 			subscription->ops->change_pte(subscription, mm, address,  in __mmu_notifier_change_pte()
479 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_range_start()  local
484 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_range_start()
486 		const struct mmu_notifier_ops *ops = subscription->ops;  in mn_hlist_invalidate_range_start()
493 			_ret = ops->invalidate_range_start(subscription, range);  in mn_hlist_invalidate_range_start()
522 		hlist_for_each_entry_rcu(subscription, &subscriptions->list,  in mn_hlist_invalidate_range_start()
524 			if (!subscription->ops->invalidate_range_end)  in mn_hlist_invalidate_range_start()
527 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_range_start()
556 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_end()  local
560 	hlist_for_each_entry_rcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_end()
562 		if (subscription->ops->invalidate_range_end) {  in mn_hlist_invalidate_end()
565 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_end()
591 	struct mmu_notifier *subscription;  in __mmu_notifier_arch_invalidate_secondary_tlbs()  local
595 	hlist_for_each_entry_rcu(subscription,  in __mmu_notifier_arch_invalidate_secondary_tlbs()
598 		if (subscription->ops->arch_invalidate_secondary_tlbs)  in __mmu_notifier_arch_invalidate_secondary_tlbs()
599 			subscription->ops->arch_invalidate_secondary_tlbs(  in __mmu_notifier_arch_invalidate_secondary_tlbs()
600 				subscription, mm,  in __mmu_notifier_arch_invalidate_secondary_tlbs()
611 int __mmu_notifier_register(struct mmu_notifier *subscription,  in __mmu_notifier_register()  argument
624 	if (WARN_ON_ONCE(subscription &&  in __mmu_notifier_register()
625 			 (subscription->ops->arch_invalidate_secondary_tlbs &&  in __mmu_notifier_register()
626 			 (subscription->ops->invalidate_range_start ||  in __mmu_notifier_register()
627 			  subscription->ops->invalidate_range_end))))  in __mmu_notifier_register()
672 	if (subscription) {  in __mmu_notifier_register()
675 		subscription->mm = mm;  in __mmu_notifier_register()
676 		subscription->users = 1;  in __mmu_notifier_register()
679 		hlist_add_head_rcu(&subscription->hlist,  in __mmu_notifier_register()
697  * @subscription: The notifier to attach
711  * While the caller has a mmu_notifier get the subscription->mm pointer will remain
714 int mmu_notifier_register(struct mmu_notifier *subscription,  in mmu_notifier_register()  argument
720 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_register()
729 	struct mmu_notifier *subscription;  in find_get_mmu_notifier()  local
732 	hlist_for_each_entry_rcu(subscription,  in find_get_mmu_notifier()
735 		if (subscription->ops != ops)  in find_get_mmu_notifier()
738 		if (likely(subscription->users != UINT_MAX))  in find_get_mmu_notifier()
739 			subscription->users++;  in find_get_mmu_notifier()
741 			subscription = ERR_PTR(-EOVERFLOW);  in find_get_mmu_notifier()
743 		return subscription;  in find_get_mmu_notifier()
769 	struct mmu_notifier *subscription;  in mmu_notifier_get_locked()  local
775 		subscription = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
776 		if (subscription)  in mmu_notifier_get_locked()
777 			return subscription;  in mmu_notifier_get_locked()
780 	subscription = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
781 	if (IS_ERR(subscription))  in mmu_notifier_get_locked()
782 		return subscription;  in mmu_notifier_get_locked()
783 	subscription->ops = ops;  in mmu_notifier_get_locked()
784 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_get_locked()
787 	return subscription;  in mmu_notifier_get_locked()
789 	subscription->ops->free_notifier(subscription);  in mmu_notifier_get_locked()
812 void mmu_notifier_unregister(struct mmu_notifier *subscription,  in mmu_notifier_unregister()  argument
817 	if (!hlist_unhashed(&subscription->hlist)) {  in mmu_notifier_unregister()
829 		if (subscription->ops->release)  in mmu_notifier_unregister()
830 			subscription->ops->release(subscription, mm);  in mmu_notifier_unregister()
838 		hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_unregister()
856 	struct mmu_notifier *subscription =  in mmu_notifier_free_rcu()  local
858 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_free_rcu()
860 	subscription->ops->free_notifier(subscription);  in mmu_notifier_free_rcu()
867  * @subscription: The notifier to act on
887 void mmu_notifier_put(struct mmu_notifier *subscription)  in mmu_notifier_put()  argument
889 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_put()
892 	if (WARN_ON(!subscription->users) || --subscription->users)  in mmu_notifier_put()
894 	hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_put()
897 	call_srcu(&srcu, &subscription->rcu, mmu_notifier_free_rcu);  in mmu_notifier_put()
957 		 * The starting seq for a subscription not under invalidation  in __mmu_interval_notifier_insert()
973  * @interval_sub: Interval subscription to register
1045  * @interval_sub: Interval subscription to unregister