42 static inline unsigned btree_cache_can_free(struct btree_cache *bc)  in btree_cache_can_free()  argument
44 	return max_t(int, 0, bc->used - bc->reserve);  in btree_cache_can_free()
47 static void btree_node_to_freedlist(struct btree_cache *bc, struct btree *b)  in btree_node_to_freedlist()  argument
50 		list_move(&b->list, &bc->freed_pcpu);  in btree_node_to_freedlist()
52 		list_move(&b->list, &bc->freed_nonpcpu);  in btree_node_to_freedlist()
57 	struct btree_cache *bc = &c->btree_cache;  in btree_node_data_free()  local
72 	bc->used--;  in btree_node_data_free()
74 	btree_node_to_freedlist(bc, b);  in btree_node_data_free()
135 	struct btree_cache *bc = &c->btree_cache;  in __bch2_btree_node_mem_alloc()  local
149 	bc->used++;  in __bch2_btree_node_mem_alloc()
150 	list_add(&b->list, &bc->freeable);  in __bch2_btree_node_mem_alloc()
156 void bch2_btree_node_hash_remove(struct btree_cache *bc, struct btree *b)  in bch2_btree_node_hash_remove()  argument
158 	int ret = rhashtable_remove_fast(&bc->table, &b->hash, bch_btree_cache_params);  in bch2_btree_node_hash_remove()
166 int __bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b)  in __bch2_btree_node_hash_insert()  argument
171 	return rhashtable_lookup_insert_fast(&bc->table, &b->hash,  in __bch2_btree_node_hash_insert()
175 int bch2_btree_node_hash_insert(struct btree_cache *bc, struct btree *b,  in bch2_btree_node_hash_insert()  argument
183 	mutex_lock(&bc->lock);  in bch2_btree_node_hash_insert()
184 	ret = __bch2_btree_node_hash_insert(bc, b);  in bch2_btree_node_hash_insert()
186 		list_add_tail(&b->list, &bc->live);  in bch2_btree_node_hash_insert()
187 	mutex_unlock(&bc->lock);  in bch2_btree_node_hash_insert()
193 static inline struct btree *btree_cache_find(struct btree_cache *bc,  in btree_cache_find()  argument
198 	return rhashtable_lookup_fast(&bc->table, &v, bch_btree_cache_params);  in btree_cache_find()
207 	struct btree_cache *bc = &c->btree_cache;  in __btree_node_reclaim()  local
210 	lockdep_assert_held(&bc->lock);  in __btree_node_reclaim()
290 	struct btree_cache *bc = &c->btree_cache;  in bch2_btree_cache_scan()  local
298 	bool trigger_writes = atomic_read(&bc->dirty) + nr >=  in bch2_btree_cache_scan()
299 		bc->used * 3 / 4;  in bch2_btree_cache_scan()
304 	mutex_lock(&bc->lock);  in bch2_btree_cache_scan()
314 	can_free = btree_cache_can_free(bc);  in bch2_btree_cache_scan()
318 	list_for_each_entry_safe(b, t, &bc->freeable, list) {  in bch2_btree_cache_scan()
339 	list_for_each_entry_safe(b, t, &bc->live, list) {  in bch2_btree_cache_scan()
348 			bch2_btree_node_hash_remove(bc, b);  in bch2_btree_cache_scan()
359 			list_move(&bc->live, &b->list);  in bch2_btree_cache_scan()
360 			mutex_unlock(&bc->lock);  in bch2_btree_cache_scan()
365 			mutex_lock(&bc->lock);  in bch2_btree_cache_scan()
373 	if (&t->list != &bc->live)  in bch2_btree_cache_scan()
374 		list_move_tail(&bc->live, &t->list);  in bch2_btree_cache_scan()
376 	mutex_unlock(&bc->lock);  in bch2_btree_cache_scan()
388 	struct btree_cache *bc = &c->btree_cache;  in bch2_btree_cache_count()  local
393 	return btree_cache_can_free(bc);  in bch2_btree_cache_count()
398 	struct btree_cache *bc = &c->btree_cache;  in bch2_fs_btree_cache_exit()  local
402 	shrinker_free(bc->shrink);  in bch2_fs_btree_cache_exit()
406 	mutex_lock(&bc->lock);  in bch2_fs_btree_cache_exit()
409 		list_move(&c->verify_data->list, &bc->live);  in bch2_fs_btree_cache_exit()
417 			list_add(&r->b->list, &bc->live);  in bch2_fs_btree_cache_exit()
420 	list_splice(&bc->freeable, &bc->live);  in bch2_fs_btree_cache_exit()
422 	while (!list_empty(&bc->live)) {  in bch2_fs_btree_cache_exit()
423 		b = list_first_entry(&bc->live, struct btree, list);  in bch2_fs_btree_cache_exit()
434 	list_splice(&bc->freed_pcpu, &bc->freed_nonpcpu);  in bch2_fs_btree_cache_exit()
436 	while (!list_empty(&bc->freed_nonpcpu)) {  in bch2_fs_btree_cache_exit()
437 		b = list_first_entry(&bc->freed_nonpcpu, struct btree, list);  in bch2_fs_btree_cache_exit()
443 	mutex_unlock(&bc->lock);  in bch2_fs_btree_cache_exit()
446 	if (bc->table_init_done)  in bch2_fs_btree_cache_exit()
447 		rhashtable_destroy(&bc->table);  in bch2_fs_btree_cache_exit()
452 	struct btree_cache *bc = &c->btree_cache;  in bch2_fs_btree_cache_init()  local
457 	ret = rhashtable_init(&bc->table, &bch_btree_cache_params);  in bch2_fs_btree_cache_init()
461 	bc->table_init_done = true;  in bch2_fs_btree_cache_init()
465 	for (i = 0; i < bc->reserve; i++)  in bch2_fs_btree_cache_init()
469 	list_splice_init(&bc->live, &bc->freeable);  in bch2_fs_btree_cache_init()
476 	bc->shrink = shrink;  in bch2_fs_btree_cache_init()
488 void bch2_fs_btree_cache_init_early(struct btree_cache *bc)  in bch2_fs_btree_cache_init_early()  argument
490 	mutex_init(&bc->lock);  in bch2_fs_btree_cache_init_early()
491 	INIT_LIST_HEAD(&bc->live);  in bch2_fs_btree_cache_init_early()
492 	INIT_LIST_HEAD(&bc->freeable);  in bch2_fs_btree_cache_init_early()
493 	INIT_LIST_HEAD(&bc->freed_pcpu);  in bch2_fs_btree_cache_init_early()
494 	INIT_LIST_HEAD(&bc->freed_nonpcpu);  in bch2_fs_btree_cache_init_early()
506 	struct btree_cache *bc = &c->btree_cache;  in bch2_btree_cache_cannibalize_unlock()  local
508 	if (bc->alloc_lock == current) {  in bch2_btree_cache_cannibalize_unlock()
510 		bc->alloc_lock = NULL;  in bch2_btree_cache_cannibalize_unlock()
511 		closure_wake_up(&bc->alloc_wait);  in bch2_btree_cache_cannibalize_unlock()
518 	struct btree_cache *bc = &c->btree_cache;  in bch2_btree_cache_cannibalize_lock()  local
521 	old = cmpxchg(&bc->alloc_lock, NULL, current);  in bch2_btree_cache_cannibalize_lock()
530 	closure_wait(&bc->alloc_wait, cl);  in bch2_btree_cache_cannibalize_lock()
533 	old = cmpxchg(&bc->alloc_lock, NULL, current);  in bch2_btree_cache_cannibalize_lock()
536 		closure_wake_up(&bc->alloc_wait);  in bch2_btree_cache_cannibalize_lock()
550 	struct btree_cache *bc = &c->btree_cache;  in btree_node_cannibalize()  local
553 	list_for_each_entry_reverse(b, &bc->live, list)  in btree_node_cannibalize()
558 		list_for_each_entry_reverse(b, &bc->live, list)  in btree_node_cannibalize()
574 	struct btree_cache *bc = &c->btree_cache;  in bch2_btree_node_mem_alloc()  local
576 		? &bc->freed_pcpu  in bch2_btree_node_mem_alloc()
577 		: &bc->freed_nonpcpu;  in bch2_btree_node_mem_alloc()
583 	mutex_lock(&bc->lock);  in bch2_btree_node_mem_alloc()
597 		mutex_unlock(&bc->lock);  in bch2_btree_node_mem_alloc()
602 		mutex_lock(&bc->lock);  in bch2_btree_node_mem_alloc()
615 	list_for_each_entry(b2, &bc->freeable, list)  in bch2_btree_node_mem_alloc()
619 			btree_node_to_freedlist(bc, b2);  in bch2_btree_node_mem_alloc()
625 	mutex_unlock(&bc->lock);  in bch2_btree_node_mem_alloc()
633 	mutex_lock(&bc->lock);  in bch2_btree_node_mem_alloc()
634 	bc->used++;  in bch2_btree_node_mem_alloc()
636 	mutex_unlock(&bc->lock);  in bch2_btree_node_mem_alloc()
657 	mutex_lock(&bc->lock);  in bch2_btree_node_mem_alloc()
660 	if (bc->alloc_lock == current) {  in bch2_btree_node_mem_alloc()
663 		bch2_btree_node_hash_remove(bc, b2);  in bch2_btree_node_mem_alloc()
668 			btree_node_to_freedlist(bc, b2);  in bch2_btree_node_mem_alloc()
676 		mutex_unlock(&bc->lock);  in bch2_btree_node_mem_alloc()
682 	mutex_unlock(&bc->lock);  in bch2_btree_node_mem_alloc()
697 	struct btree_cache *bc = &c->btree_cache;  in bch2_btree_node_fill()  local
723 	if (bch2_btree_node_hash_insert(bc, b, level, btree_id)) {  in bch2_btree_node_fill()
729 		mutex_lock(&bc->lock);  in bch2_btree_node_fill()
730 		list_add(&b->list, &bc->freeable);  in bch2_btree_node_fill()
731 		mutex_unlock(&bc->lock);  in bch2_btree_node_fill()
815 	struct btree_cache *bc = &c->btree_cache;  in __bch2_btree_node_get()  local
823 	b = btree_cache_find(bc, k);  in __bch2_btree_node_get()
1012 	struct btree_cache *bc = &c->btree_cache;  in bch2_btree_node_get_noiter()  local
1025 	b = btree_cache_find(bc, k);  in bch2_btree_node_get_noiter()
1096 	struct btree_cache *bc = &c->btree_cache;  in bch2_btree_node_prefetch()  local
1102 	b = btree_cache_find(bc, k);  in bch2_btree_node_prefetch()
1114 	struct btree_cache *bc = &c->btree_cache;  in bch2_btree_node_evict()  local
1117 	b = btree_cache_find(bc, k);  in bch2_btree_node_evict()
1141 	mutex_lock(&bc->lock);  in bch2_btree_node_evict()
1143 	bch2_btree_node_hash_remove(bc, b);  in bch2_btree_node_evict()
1144 	mutex_unlock(&bc->lock);  in bch2_btree_node_evict()