330 	struct mmu_notifier *subscription;  in mn_hlist_release()  local
338 	hlist_for_each_entry_srcu(subscription, &subscriptions->list, hlist,  in mn_hlist_release()
346 		if (subscription->ops->release)  in mn_hlist_release()
347 			subscription->ops->release(subscription, mm);  in mn_hlist_release()
351 		subscription = hlist_entry(subscriptions->list.first,  in mn_hlist_release()
359 		hlist_del_init_rcu(&subscription->hlist);  in mn_hlist_release()
396 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_flush_young()  local
401 	hlist_for_each_entry_srcu(subscription,  in __mmu_notifier_clear_flush_young()
404 		if (subscription->ops->clear_flush_young)  in __mmu_notifier_clear_flush_young()
405 			young |= subscription->ops->clear_flush_young(  in __mmu_notifier_clear_flush_young()
406 				subscription, mm, start, end);  in __mmu_notifier_clear_flush_young()
416 	struct mmu_notifier *subscription;  in __mmu_notifier_clear_young()  local
421 	hlist_for_each_entry_srcu(subscription,  in __mmu_notifier_clear_young()
424 		if (subscription->ops->clear_young)  in __mmu_notifier_clear_young()
425 			young |= subscription->ops->clear_young(subscription,  in __mmu_notifier_clear_young()
436 	struct mmu_notifier *subscription;  in __mmu_notifier_test_young()  local
441 	hlist_for_each_entry_srcu(subscription,  in __mmu_notifier_test_young()
444 		if (subscription->ops->test_young) {  in __mmu_notifier_test_young()
445 			young = subscription->ops->test_young(subscription, mm,  in __mmu_notifier_test_young()
511 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_range_start()  local
516 	hlist_for_each_entry_srcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_range_start()
518 		const struct mmu_notifier_ops *ops = subscription->ops;  in mn_hlist_invalidate_range_start()
525 			_ret = ops->invalidate_range_start(subscription, range);  in mn_hlist_invalidate_range_start()
554 		hlist_for_each_entry_srcu(subscription, &subscriptions->list,  in mn_hlist_invalidate_range_start()
556 			if (!subscription->ops->invalidate_range_end)  in mn_hlist_invalidate_range_start()
559 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_range_start()
588 	struct mmu_notifier *subscription;  in mn_hlist_invalidate_end()  local
592 	hlist_for_each_entry_srcu(subscription, &subscriptions->list, hlist,  in mn_hlist_invalidate_end()
594 		if (subscription->ops->invalidate_range_end) {  in mn_hlist_invalidate_end()
597 			subscription->ops->invalidate_range_end(subscription,  in mn_hlist_invalidate_end()
623 	struct mmu_notifier *subscription;  in __mmu_notifier_arch_invalidate_secondary_tlbs()  local
627 	hlist_for_each_entry_srcu(subscription,  in __mmu_notifier_arch_invalidate_secondary_tlbs()
630 		if (subscription->ops->arch_invalidate_secondary_tlbs)  in __mmu_notifier_arch_invalidate_secondary_tlbs()
631 			subscription->ops->arch_invalidate_secondary_tlbs(  in __mmu_notifier_arch_invalidate_secondary_tlbs()
632 				subscription, mm,  in __mmu_notifier_arch_invalidate_secondary_tlbs()
643 int __mmu_notifier_register(struct mmu_notifier *subscription,  in __mmu_notifier_register()  argument
656 	if (WARN_ON_ONCE(subscription &&  in __mmu_notifier_register()
657 			 (subscription->ops->arch_invalidate_secondary_tlbs &&  in __mmu_notifier_register()
658 			 (subscription->ops->invalidate_range_start ||  in __mmu_notifier_register()
659 			  subscription->ops->invalidate_range_end))))  in __mmu_notifier_register()
703 	if (subscription) {  in __mmu_notifier_register()
706 		subscription->mm = mm;  in __mmu_notifier_register()
707 		subscription->users = 1;  in __mmu_notifier_register()
710 		hlist_add_head_rcu(&subscription->hlist,  in __mmu_notifier_register()
745 int mmu_notifier_register(struct mmu_notifier *subscription,  in mmu_notifier_register()  argument
751 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_register()
760 	struct mmu_notifier *subscription;  in find_get_mmu_notifier()  local
763 	hlist_for_each_entry_srcu(subscription,  in find_get_mmu_notifier()
766 		if (subscription->ops != ops)  in find_get_mmu_notifier()
769 		if (likely(subscription->users != UINT_MAX))  in find_get_mmu_notifier()
770 			subscription->users++;  in find_get_mmu_notifier()
772 			subscription = ERR_PTR(-EOVERFLOW);  in find_get_mmu_notifier()
774 		return subscription;  in find_get_mmu_notifier()
800 	struct mmu_notifier *subscription;  in mmu_notifier_get_locked()  local
806 		subscription = find_get_mmu_notifier(mm, ops);  in mmu_notifier_get_locked()
807 		if (subscription)  in mmu_notifier_get_locked()
808 			return subscription;  in mmu_notifier_get_locked()
811 	subscription = ops->alloc_notifier(mm);  in mmu_notifier_get_locked()
812 	if (IS_ERR(subscription))  in mmu_notifier_get_locked()
813 		return subscription;  in mmu_notifier_get_locked()
814 	subscription->ops = ops;  in mmu_notifier_get_locked()
815 	ret = __mmu_notifier_register(subscription, mm);  in mmu_notifier_get_locked()
818 	return subscription;  in mmu_notifier_get_locked()
820 	subscription->ops->free_notifier(subscription);  in mmu_notifier_get_locked()
843 void mmu_notifier_unregister(struct mmu_notifier *subscription,  in mmu_notifier_unregister()  argument
848 	if (!hlist_unhashed(&subscription->hlist)) {  in mmu_notifier_unregister()
860 		if (subscription->ops->release)  in mmu_notifier_unregister()
861 			subscription->ops->release(subscription, mm);  in mmu_notifier_unregister()
869 		hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_unregister()
887 	struct mmu_notifier *subscription =  in mmu_notifier_free_rcu()  local
889 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_free_rcu()
891 	subscription->ops->free_notifier(subscription);  in mmu_notifier_free_rcu()
918 void mmu_notifier_put(struct mmu_notifier *subscription)  in mmu_notifier_put()  argument
920 	struct mm_struct *mm = subscription->mm;  in mmu_notifier_put()
923 	if (WARN_ON(!subscription->users) || --subscription->users)  in mmu_notifier_put()
925 	hlist_del_init_rcu(&subscription->hlist);  in mmu_notifier_put()
928 	call_srcu(&srcu, &subscription->rcu, mmu_notifier_free_rcu);  in mmu_notifier_put()