20 void __mmap_lock_do_trace_start_locking(struct mm_struct *mm, bool write);
21 void __mmap_lock_do_trace_acquire_returned(struct mm_struct *mm, bool write,
23 void __mmap_lock_do_trace_released(struct mm_struct *mm, bool write);
25 static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm,  in __mmap_lock_trace_start_locking()  argument
29 		__mmap_lock_do_trace_start_locking(mm, write);  in __mmap_lock_trace_start_locking()
32 static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm,  in __mmap_lock_trace_acquire_returned()  argument
36 		__mmap_lock_do_trace_acquire_returned(mm, write, success);  in __mmap_lock_trace_acquire_returned()
39 static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)  in __mmap_lock_trace_released()  argument
42 		__mmap_lock_do_trace_released(mm, write);  in __mmap_lock_trace_released()
47 static inline void __mmap_lock_trace_start_locking(struct mm_struct *mm,  in __mmap_lock_trace_start_locking()  argument
52 static inline void __mmap_lock_trace_acquire_returned(struct mm_struct *mm,  in __mmap_lock_trace_acquire_returned()  argument
57 static inline void __mmap_lock_trace_released(struct mm_struct *mm, bool write)  in __mmap_lock_trace_released()  argument
63 static inline void mmap_assert_locked(struct mm_struct *mm)  in mmap_assert_locked()  argument
65 	lockdep_assert_held(&mm->mmap_lock);  in mmap_assert_locked()
66 	VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm);  in mmap_assert_locked()
69 static inline void mmap_assert_write_locked(struct mm_struct *mm)  in mmap_assert_write_locked()  argument
71 	lockdep_assert_held_write(&mm->mmap_lock);  in mmap_assert_write_locked()
72 	VM_BUG_ON_MM(!rwsem_is_locked(&mm->mmap_lock), mm);  in mmap_assert_write_locked()
84 static inline void vma_end_write_all(struct mm_struct *mm)  in vma_end_write_all()  argument
86 	mmap_assert_write_locked(mm);  in vma_end_write_all()
88 	 * Nobody can concurrently modify mm->mm_lock_seq due to exclusive  in vma_end_write_all()
94 	smp_store_release(&mm->mm_lock_seq, mm->mm_lock_seq + 1);  in vma_end_write_all()
97 static inline void vma_end_write_all(struct mm_struct *mm) {}  in vma_end_write_all()  argument
100 static inline void mmap_init_lock(struct mm_struct *mm)  in mmap_init_lock()  argument
102 	init_rwsem(&mm->mmap_lock);  in mmap_init_lock()
105 static inline void mmap_write_lock(struct mm_struct *mm)  in mmap_write_lock()  argument
107 	__mmap_lock_trace_start_locking(mm, true);  in mmap_write_lock()
108 	down_write(&mm->mmap_lock);  in mmap_write_lock()
109 	__mmap_lock_trace_acquire_returned(mm, true, true);  in mmap_write_lock()
112 static inline void mmap_write_lock_nested(struct mm_struct *mm, int subclass)  in mmap_write_lock_nested()  argument
114 	__mmap_lock_trace_start_locking(mm, true);  in mmap_write_lock_nested()
115 	down_write_nested(&mm->mmap_lock, subclass);  in mmap_write_lock_nested()
116 	__mmap_lock_trace_acquire_returned(mm, true, true);  in mmap_write_lock_nested()
119 static inline int mmap_write_lock_killable(struct mm_struct *mm)  in mmap_write_lock_killable()  argument
123 	__mmap_lock_trace_start_locking(mm, true);  in mmap_write_lock_killable()
124 	ret = down_write_killable(&mm->mmap_lock);  in mmap_write_lock_killable()
125 	__mmap_lock_trace_acquire_returned(mm, true, ret == 0);  in mmap_write_lock_killable()
129 static inline void mmap_write_unlock(struct mm_struct *mm)  in mmap_write_unlock()  argument
131 	__mmap_lock_trace_released(mm, true);  in mmap_write_unlock()
132 	vma_end_write_all(mm);  in mmap_write_unlock()
133 	up_write(&mm->mmap_lock);  in mmap_write_unlock()
136 static inline void mmap_write_downgrade(struct mm_struct *mm)  in mmap_write_downgrade()  argument
138 	__mmap_lock_trace_acquire_returned(mm, false, true);  in mmap_write_downgrade()
139 	vma_end_write_all(mm);  in mmap_write_downgrade()
140 	downgrade_write(&mm->mmap_lock);  in mmap_write_downgrade()
143 static inline void mmap_read_lock(struct mm_struct *mm)  in mmap_read_lock()  argument
145 	__mmap_lock_trace_start_locking(mm, false);  in mmap_read_lock()
146 	down_read(&mm->mmap_lock);  in mmap_read_lock()
147 	__mmap_lock_trace_acquire_returned(mm, false, true);  in mmap_read_lock()
150 static inline int mmap_read_lock_killable(struct mm_struct *mm)  in mmap_read_lock_killable()  argument
154 	__mmap_lock_trace_start_locking(mm, false);  in mmap_read_lock_killable()
155 	ret = down_read_killable(&mm->mmap_lock);  in mmap_read_lock_killable()
156 	__mmap_lock_trace_acquire_returned(mm, false, ret == 0);  in mmap_read_lock_killable()
160 static inline bool mmap_read_trylock(struct mm_struct *mm)  in mmap_read_trylock()  argument
164 	__mmap_lock_trace_start_locking(mm, false);  in mmap_read_trylock()
165 	ret = down_read_trylock(&mm->mmap_lock) != 0;  in mmap_read_trylock()
166 	__mmap_lock_trace_acquire_returned(mm, false, ret);  in mmap_read_trylock()
170 static inline void mmap_read_unlock(struct mm_struct *mm)  in mmap_read_unlock()  argument
172 	__mmap_lock_trace_released(mm, false);  in mmap_read_unlock()
173 	up_read(&mm->mmap_lock);  in mmap_read_unlock()
176 static inline void mmap_read_unlock_non_owner(struct mm_struct *mm)  in mmap_read_unlock_non_owner()  argument
178 	__mmap_lock_trace_released(mm, false);  in mmap_read_unlock_non_owner()
179 	up_read_non_owner(&mm->mmap_lock);  in mmap_read_unlock_non_owner()
182 static inline int mmap_lock_is_contended(struct mm_struct *mm)  in mmap_lock_is_contended()  argument
184 	return rwsem_is_contended(&mm->mmap_lock);  in mmap_lock_is_contended()