105 alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode);
108 __kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms)  in __kick_dmap_free_worker()  argument
113 	free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100,  in __kick_dmap_free_worker()
115 	if (fcd->nr_free_ranges < free_threshold)  in __kick_dmap_free_worker()
116 		queue_delayed_work(system_long_wq, &fcd->free_work,  in __kick_dmap_free_worker()
120 static void kick_dmap_free_worker(struct fuse_conn_dax *fcd,  in kick_dmap_free_worker()  argument
123 	spin_lock(&fcd->lock);  in kick_dmap_free_worker()
124 	__kick_dmap_free_worker(fcd, delay_ms);  in kick_dmap_free_worker()
125 	spin_unlock(&fcd->lock);  in kick_dmap_free_worker()
128 static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd)  in alloc_dax_mapping()  argument
132 	spin_lock(&fcd->lock);  in alloc_dax_mapping()
133 	dmap = list_first_entry_or_null(&fcd->free_ranges,  in alloc_dax_mapping()
137 		WARN_ON(fcd->nr_free_ranges <= 0);  in alloc_dax_mapping()
138 		fcd->nr_free_ranges--;  in alloc_dax_mapping()
140 	__kick_dmap_free_worker(fcd, 0);  in alloc_dax_mapping()
141 	spin_unlock(&fcd->lock);  in alloc_dax_mapping()
147 static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd,  in __dmap_remove_busy_list()  argument
151 	WARN_ON(fcd->nr_busy_ranges == 0);  in __dmap_remove_busy_list()
152 	fcd->nr_busy_ranges--;  in __dmap_remove_busy_list()
155 static void dmap_remove_busy_list(struct fuse_conn_dax *fcd,  in dmap_remove_busy_list()  argument
158 	spin_lock(&fcd->lock);  in dmap_remove_busy_list()
159 	__dmap_remove_busy_list(fcd, dmap);  in dmap_remove_busy_list()
160 	spin_unlock(&fcd->lock);  in dmap_remove_busy_list()
164 static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd,  in __dmap_add_to_free_pool()  argument
167 	list_add_tail(&dmap->list, &fcd->free_ranges);  in __dmap_add_to_free_pool()
168 	fcd->nr_free_ranges++;  in __dmap_add_to_free_pool()
169 	wake_up(&fcd->range_waitq);  in __dmap_add_to_free_pool()
172 static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd,  in dmap_add_to_free_pool()  argument
176 	spin_lock(&fcd->lock);  in dmap_add_to_free_pool()
177 	__dmap_add_to_free_pool(fcd, dmap);  in dmap_add_to_free_pool()
178 	spin_unlock(&fcd->lock);  in dmap_add_to_free_pool()
186 	struct fuse_conn_dax *fcd = fm->fc->dax;  in fuse_setup_one_mapping()  local
193 	WARN_ON(fcd->nr_free_ranges < 0);  in fuse_setup_one_mapping()
224 		spin_lock(&fcd->lock);  in fuse_setup_one_mapping()
225 		list_add_tail(&dmap->busy_list, &fcd->busy_ranges);  in fuse_setup_one_mapping()
226 		fcd->nr_busy_ranges++;  in fuse_setup_one_mapping()
227 		spin_unlock(&fcd->lock);  in fuse_setup_one_mapping()
291 static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd,  in dmap_reinit_add_to_free_pool()  argument
297 	__dmap_remove_busy_list(fcd, dmap);  in dmap_reinit_add_to_free_pool()
300 	__dmap_add_to_free_pool(fcd, dmap);  in dmap_reinit_add_to_free_pool()
309 static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd,  in inode_reclaim_dmap_range()  argument
345 	spin_lock(&fcd->lock);  in inode_reclaim_dmap_range()
348 		dmap_reinit_add_to_free_pool(fcd, dmap);  in inode_reclaim_dmap_range()
350 	spin_unlock(&fcd->lock);  in inode_reclaim_dmap_range()
437 	struct fuse_conn_dax *fcd = fc->dax;  in fuse_setup_new_dax_mapping()  local
455 		alloc_dmap = alloc_dax_mapping(fcd);  in fuse_setup_new_dax_mapping()
459 		alloc_dmap = alloc_dax_mapping_reclaim(fcd, inode);  in fuse_setup_new_dax_mapping()
481 		dmap_add_to_free_pool(fcd, alloc_dmap);  in fuse_setup_new_dax_mapping()
490 		dmap_add_to_free_pool(fcd, alloc_dmap);  in fuse_setup_new_dax_mapping()
762 	struct fuse_conn_dax *fcd = fc->dax;  in __fuse_dax_fault()  local
768 	if (retry && !(fcd->nr_free_ranges > 0))  in __fuse_dax_fault()
769 		wait_event(fcd->range_waitq, (fcd->nr_free_ranges > 0));  in __fuse_dax_fault()
912 inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,  in inode_inline_reclaim_one_dmap()  argument
973 	dmap_remove_busy_list(fcd, dmap);  in inode_inline_reclaim_one_dmap()
988 alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode)  in alloc_dax_mapping_reclaim()  argument
996 		dmap = alloc_dax_mapping(fcd);  in alloc_dax_mapping_reclaim()
1000 		dmap = inode_inline_reclaim_one_dmap(fcd, inode, &retry);  in alloc_dax_mapping_reclaim()
1026 		if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) {  in alloc_dax_mapping_reclaim()
1027 			if (wait_event_killable_exclusive(fcd->range_waitq,  in alloc_dax_mapping_reclaim()
1028 					(fcd->nr_free_ranges > 0))) {  in alloc_dax_mapping_reclaim()
1035 static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd,  in lookup_and_reclaim_dmap_locked()  argument
1061 	spin_lock(&fcd->lock);  in lookup_and_reclaim_dmap_locked()
1062 	dmap_reinit_add_to_free_pool(fcd, dmap);  in lookup_and_reclaim_dmap_locked()
1063 	spin_unlock(&fcd->lock);  in lookup_and_reclaim_dmap_locked()
1074 static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd,  in lookup_and_reclaim_dmap()  argument
1093 	ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx);  in lookup_and_reclaim_dmap()
1100 static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd,  in try_to_free_dmap_chunks()  argument
1114 		spin_lock(&fcd->lock);  in try_to_free_dmap_chunks()
1116 		if (!fcd->nr_busy_ranges) {  in try_to_free_dmap_chunks()
1117 			spin_unlock(&fcd->lock);  in try_to_free_dmap_chunks()
1121 		list_for_each_entry_safe(pos, temp, &fcd->busy_ranges,  in try_to_free_dmap_chunks()
1141 			list_move_tail(&dmap->busy_list, &fcd->busy_ranges);  in try_to_free_dmap_chunks()
1145 		spin_unlock(&fcd->lock);  in try_to_free_dmap_chunks()
1149 		ret = lookup_and_reclaim_dmap(fcd, inode, start_idx, end_idx);  in try_to_free_dmap_chunks()
1161 	struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax,  in fuse_dax_free_mem_worker()  local
1163 	ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK);  in fuse_dax_free_mem_worker()
1170 	kick_dmap_free_worker(fcd, 1);  in fuse_dax_free_mem_worker()
1195 static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd)  in fuse_dax_mem_range_init()  argument
1203 	init_waitqueue_head(&fcd->range_waitq);  in fuse_dax_mem_range_init()
1204 	INIT_LIST_HEAD(&fcd->free_ranges);  in fuse_dax_mem_range_init()
1205 	INIT_LIST_HEAD(&fcd->busy_ranges);  in fuse_dax_mem_range_init()
1206 	INIT_DELAYED_WORK(&fcd->free_work, fuse_dax_free_mem_worker);  in fuse_dax_mem_range_init()
1209 	nr_pages = dax_direct_access(fcd->dev, 0, PHYS_PFN(dax_size),  in fuse_dax_mem_range_init()
1235 		list_add_tail(&range->list, &fcd->free_ranges);  in fuse_dax_mem_range_init()
1238 	fcd->nr_free_ranges = nr_ranges;  in fuse_dax_mem_range_init()
1239 	fcd->nr_ranges = nr_ranges;  in fuse_dax_mem_range_init()
1243 	fuse_free_dax_mem_ranges(&fcd->free_ranges);  in fuse_dax_mem_range_init()
1250 	struct fuse_conn_dax *fcd;  in fuse_dax_conn_alloc()  local
1258 	fcd = kzalloc_obj(*fcd);  in fuse_dax_conn_alloc()
1259 	if (!fcd)  in fuse_dax_conn_alloc()
1262 	spin_lock_init(&fcd->lock);  in fuse_dax_conn_alloc()
1263 	fcd->dev = dax_dev;  in fuse_dax_conn_alloc()
1264 	err = fuse_dax_mem_range_init(fcd);  in fuse_dax_conn_alloc()
1266 		kfree(fcd);  in fuse_dax_conn_alloc()
1270 	fc->dax = fcd;  in fuse_dax_conn_alloc()
1348 	struct fuse_conn_dax *fcd = fc->dax;  in fuse_dax_cancel_work()  local
1350 	if (fcd)  in fuse_dax_cancel_work()
1351 		cancel_delayed_work_sync(&fcd->free_work);  in fuse_dax_cancel_work()