83 				 struct bkey_s_c k)  in bch2_alloc_unpack_v1()  argument
85 	const struct bch_alloc *in = bkey_s_c_to_alloc(k).v;  in bch2_alloc_unpack_v1()
97 				struct bkey_s_c k)  in bch2_alloc_unpack_v2()  argument
99 	struct bkey_s_c_alloc_v2 a = bkey_s_c_to_alloc_v2(k);  in bch2_alloc_unpack_v2()
130 				struct bkey_s_c k)  in bch2_alloc_unpack_v3()  argument
132 	struct bkey_s_c_alloc_v3 a = bkey_s_c_to_alloc_v3(k);  in bch2_alloc_unpack_v3()
165 static struct bkey_alloc_unpacked bch2_alloc_unpack(struct bkey_s_c k)  in bch2_alloc_unpack()  argument
169 	switch (k.k->type) {  in bch2_alloc_unpack()
171 		bch2_alloc_unpack_v1(&ret, k);  in bch2_alloc_unpack()
174 		bch2_alloc_unpack_v2(&ret, k);  in bch2_alloc_unpack()
177 		bch2_alloc_unpack_v3(&ret, k);  in bch2_alloc_unpack()
195 int bch2_alloc_v1_invalid(struct bch_fs *c, struct bkey_s_c k,  in bch2_alloc_v1_invalid()  argument
199 	struct bkey_s_c_alloc a = bkey_s_c_to_alloc(k);  in bch2_alloc_v1_invalid()
203 	bkey_fsck_err_on(bkey_val_u64s(a.k) < bch_alloc_v1_val_u64s(a.v), c, err,  in bch2_alloc_v1_invalid()
206 			 bkey_val_u64s(a.k), bch_alloc_v1_val_u64s(a.v));  in bch2_alloc_v1_invalid()
211 int bch2_alloc_v2_invalid(struct bch_fs *c, struct bkey_s_c k,  in bch2_alloc_v2_invalid()  argument
218 	bkey_fsck_err_on(bch2_alloc_unpack_v2(&u, k), c, err,  in bch2_alloc_v2_invalid()
225 int bch2_alloc_v3_invalid(struct bch_fs *c, struct bkey_s_c k,  in bch2_alloc_v3_invalid()  argument
232 	bkey_fsck_err_on(bch2_alloc_unpack_v3(&u, k), c, err,  in bch2_alloc_v3_invalid()
239 int bch2_alloc_v4_invalid(struct bch_fs *c, struct bkey_s_c k,  in bch2_alloc_v4_invalid()  argument
242 	struct bkey_s_c_alloc_v4 a = bkey_s_c_to_alloc_v4(k);  in bch2_alloc_v4_invalid()
245 	bkey_fsck_err_on(alloc_v4_u64s(a.v) > bkey_val_u64s(k.k), c, err,  in bch2_alloc_v4_invalid()
248 			 alloc_v4_u64s(a.v), bkey_val_u64s(k.k));  in bch2_alloc_v4_invalid()
297 void bch2_alloc_v4_swab(struct bkey_s k)  in bch2_alloc_v4_swab()  argument
299 	struct bch_alloc_v4 *a = bkey_s_to_alloc_v4(k).v;  in bch2_alloc_v4_swab()
320 void bch2_alloc_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)  in bch2_alloc_to_text()  argument
323 	const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &_a);  in bch2_alloc_to_text()
355 void __bch2_alloc_to_v4(struct bkey_s_c k, struct bch_alloc_v4 *out)  in __bch2_alloc_to_v4()  argument
357 	if (k.k->type == KEY_TYPE_alloc_v4) {  in __bch2_alloc_to_v4()
360 		*out = *bkey_s_c_to_alloc_v4(k).v;  in __bch2_alloc_to_v4()
371 		struct bkey_alloc_unpacked u = bch2_alloc_unpack(k);  in __bch2_alloc_to_v4()
392 __bch2_alloc_to_v4_mut(struct btree_trans *trans, struct bkey_s_c k)  in __bch2_alloc_to_v4_mut()  argument
396 	ret = bch2_trans_kmalloc(trans, max(bkey_bytes(k.k), sizeof(struct bkey_i_alloc_v4)));  in __bch2_alloc_to_v4_mut()
400 	if (k.k->type == KEY_TYPE_alloc_v4) {  in __bch2_alloc_to_v4_mut()
403 		bkey_reassemble(&ret->k_i, k);  in __bch2_alloc_to_v4_mut()
416 		ret->k.p = k.k->p;  in __bch2_alloc_to_v4_mut()
417 		bch2_alloc_to_v4(k, &ret->v);  in __bch2_alloc_to_v4_mut()
422 … struct bkey_i_alloc_v4 *bch2_alloc_to_v4_mut_inlined(struct btree_trans *trans, struct bkey_s_c k)  in bch2_alloc_to_v4_mut_inlined()  argument
426 	if (likely(k.k->type == KEY_TYPE_alloc_v4) &&  in bch2_alloc_to_v4_mut_inlined()
427 	    ((a = bkey_s_c_to_alloc_v4(k), true) &&  in bch2_alloc_to_v4_mut_inlined()
429 		return bch2_bkey_make_mut_noupdate_typed(trans, k, alloc_v4);  in bch2_alloc_to_v4_mut_inlined()
431 	return __bch2_alloc_to_v4_mut(trans, k);  in bch2_alloc_to_v4_mut_inlined()
434 struct bkey_i_alloc_v4 *bch2_alloc_to_v4_mut(struct btree_trans *trans, struct bkey_s_c k)  in bch2_alloc_to_v4_mut()  argument
436 	return bch2_alloc_to_v4_mut_inlined(trans, k);  in bch2_alloc_to_v4_mut()
443 	struct bkey_s_c k;  in bch2_trans_start_alloc_update()  local
447 	k = bch2_bkey_get_iter(trans, iter, BTREE_ID_alloc, pos,  in bch2_trans_start_alloc_update()
451 	ret = bkey_err(k);  in bch2_trans_start_alloc_update()
455 	a = bch2_alloc_to_v4_mut_inlined(trans, k);  in bch2_trans_start_alloc_update()
480 static unsigned alloc_gen(struct bkey_s_c k, unsigned offset)  in alloc_gen()  argument
482 	return k.k->type == KEY_TYPE_bucket_gens  in alloc_gen()
483 		? bkey_s_c_to_bucket_gens(k).v->gens[offset]  in alloc_gen()
487 int bch2_bucket_gens_invalid(struct bch_fs *c, struct bkey_s_c k,  in bch2_bucket_gens_invalid()  argument
493 	bkey_fsck_err_on(bkey_val_bytes(k.k) != sizeof(struct bch_bucket_gens), c, err,  in bch2_bucket_gens_invalid()
496 			 bkey_val_bytes(k.k), sizeof(struct bch_bucket_gens));  in bch2_bucket_gens_invalid()
501 void bch2_bucket_gens_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k)  in bch2_bucket_gens_to_text()  argument
503 	struct bkey_s_c_bucket_gens g = bkey_s_c_to_bucket_gens(k);  in bch2_bucket_gens_to_text()
521 				 BTREE_ITER_PREFETCH, k, ({  in bch2_bucket_gens_init()
526 		if (!bch2_dev_bucket_exists(c, k.k->p))  in bch2_bucket_gens_init()
530 		u8 gen = bch2_alloc_to_v4(k, &a)->gen;  in bch2_bucket_gens_init()
545 			g.k.p = pos;  in bch2_bucket_gens_init()
573 					 BTREE_ITER_PREFETCH, k, ({  in bch2_alloc_read()
574 			u64 start = bucket_gens_pos_to_alloc(k.k->p, 0).offset;  in bch2_alloc_read()
575 			u64 end = bucket_gens_pos_to_alloc(bpos_nosnap_successor(k.k->p), 0).offset;  in bch2_alloc_read()
577 			if (k.k->type != KEY_TYPE_bucket_gens)  in bch2_alloc_read()
580 			const struct bch_bucket_gens *g = bkey_s_c_to_bucket_gens(k).v;  in bch2_alloc_read()
586 			if (!bch2_dev_exists2(c, k.k->p.inode))  in bch2_alloc_read()
589 			struct bch_dev *ca = bch_dev_bkey_exists(c, k.k->p.inode);  in bch2_alloc_read()
599 					 BTREE_ITER_PREFETCH, k, ({  in bch2_alloc_read()
604 			if (!bch2_dev_bucket_exists(c, k.k->p))  in bch2_alloc_read()
607 			struct bch_dev *ca = bch_dev_bkey_exists(c, k.k->p.inode);  in bch2_alloc_read()
610 			*bucket_gen(ca, k.k->p.offset) = bch2_alloc_to_v4(k, &a)->gen;  in bch2_alloc_read()
630 	struct bch_dev *ca = bch_dev_bkey_exists(c, alloc_k.k->p.inode);  in bch2_bucket_do_index()
633 	struct bkey_i *k;  in bch2_bucket_do_index()  local
644 	k = bch2_trans_kmalloc_nomemzero(trans, sizeof(*k));  in bch2_bucket_do_index()
645 	if (IS_ERR(k))  in bch2_bucket_do_index()
646 		return PTR_ERR(k);  in bch2_bucket_do_index()
648 	bkey_init(&k->k);  in bch2_bucket_do_index()
649 	k->k.type = new_type;  in bch2_bucket_do_index()
654 		k->k.p = alloc_freespace_pos(alloc_k.k->p, *a);  in bch2_bucket_do_index()
655 		bch2_key_resize(&k->k, 1);  in bch2_bucket_do_index()
659 		k->k.p = alloc_k.k->p;  in bch2_bucket_do_index()
666 			     bkey_start_pos(&k->k),  in bch2_bucket_do_index()
674 	    bch2_trans_inconsistent_on(old.k->type != old_type, trans,  in bch2_bucket_do_index()
681 			bch2_bkey_types[old.k->type],  in bch2_bucket_do_index()
688 	ret = bch2_trans_update(trans, &iter, k, 0);  in bch2_bucket_do_index()
702 	struct bkey_s_c k;  in bch2_bucket_gen_update()  local
710 	k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_bucket_gens, pos,  in bch2_bucket_gen_update()
713 	ret = bkey_err(k);  in bch2_bucket_gen_update()
717 	if (k.k->type != KEY_TYPE_bucket_gens) {  in bch2_bucket_gen_update()
719 		g->k.p = iter.pos;  in bch2_bucket_gen_update()
721 		bkey_reassemble(&g->k_i, k);  in bch2_bucket_gen_update()
739 	if (bch2_trans_inconsistent_on(!bch2_dev_bucket_exists(c, new.k->p), trans,  in bch2_trigger_alloc()
743 	struct bch_dev *ca = bch_dev_bkey_exists(c, new.k->p.inode);  in bch2_trigger_alloc()
762 		    !bch2_bucket_is_open_safe(c, new.k->p.inode, new.k->p.offset)) {  in bch2_trigger_alloc()
783 			ret = bch2_lru_change(trans, new.k->p.inode,  in bch2_trigger_alloc()
784 					      bucket_to_u64(new.k->p),  in bch2_trigger_alloc()
791 						bch_dev_bkey_exists(c, new.k->p.inode));  in bch2_trigger_alloc()
795 					bucket_to_u64(new.k->p),  in bch2_trigger_alloc()
802 			ret = bch2_bucket_gen_update(trans, new.k->p, new_a->gen);  in bch2_trigger_alloc()
814 			ret = bch2_update_cached_sectors_list(trans, new.k->p.inode,  in bch2_trigger_alloc()
829 		    new.k->type == KEY_TYPE_alloc_v4) {  in bch2_trigger_alloc()
849 					new.k->p.inode, new.k->p.offset,  in bch2_trigger_alloc()
860 			*bucket_gen(ca, new.k->p.offset) = new_a->gen;  in bch2_trigger_alloc()
888 		struct bucket *g = gc_bucket(ca, new.k->p.offset);  in bch2_trigger_alloc()
913 	struct bkey_s_c k = bch2_btree_iter_peek_slot(iter);  in bch2_get_key_or_hole()  local
915 	if (bkey_err(k))  in bch2_get_key_or_hole()
916 		return k;  in bch2_get_key_or_hole()
918 	if (k.k->type) {  in bch2_get_key_or_hole()
919 		return k;  in bch2_get_key_or_hole()
927 		if (!bpos_eq(path->l[0].b->key.k.p, SPOS_MAX))  in bch2_get_key_or_hole()
928 			end = bkey_min(end, bpos_nosnap_successor(path->l[0].b->key.k.p));  in bch2_get_key_or_hole()
936 		k = bch2_btree_iter_peek_upto(&iter2, end);  in bch2_get_key_or_hole()
942 		if (bkey_err(k))  in bch2_get_key_or_hole()
943 			return k;  in bch2_get_key_or_hole()
984 	struct bkey_s_c k;  in bch2_get_key_or_real_bucket_hole()  local
986 	k = bch2_get_key_or_hole(iter, POS_MAX, hole);  in bch2_get_key_or_real_bucket_hole()
987 	if (bkey_err(k))  in bch2_get_key_or_real_bucket_hole()
988 		return k;  in bch2_get_key_or_real_bucket_hole()
990 	if (!k.k->type) {  in bch2_get_key_or_real_bucket_hole()
991 		struct bpos bucket = bkey_start_pos(k.k);  in bch2_get_key_or_real_bucket_hole()
1001 		if (!bch2_dev_bucket_exists(c, k.k->p)) {  in bch2_get_key_or_real_bucket_hole()
1008 	return k;  in bch2_get_key_or_real_bucket_hole()
1025 	struct bkey_s_c k;  in bch2_check_alloc_key()  local
1029 	if (fsck_err_on(!bch2_dev_bucket_exists(c, alloc_k.k->p), c,  in bch2_check_alloc_key()
1032 			alloc_k.k->p.inode, alloc_k.k->p.offset))  in bch2_check_alloc_key()
1035 	ca = bch_dev_bkey_exists(c, alloc_k.k->p.inode);  in bch2_check_alloc_key()
1042 	bch2_btree_iter_set_pos(discard_iter, alloc_k.k->p);  in bch2_check_alloc_key()
1043 	k = bch2_btree_iter_peek_slot(discard_iter);  in bch2_check_alloc_key()
1044 	ret = bkey_err(k);  in bch2_check_alloc_key()
1048 	if (k.k->type != discard_key_type &&  in bch2_check_alloc_key()
1053 		      bch2_bkey_types[k.k->type],  in bch2_check_alloc_key()
1063 		bkey_init(&update->k);  in bch2_check_alloc_key()
1064 		update->k.type	= discard_key_type;  in bch2_check_alloc_key()
1065 		update->k.p	= discard_iter->pos;  in bch2_check_alloc_key()
1073 	bch2_btree_iter_set_pos(freespace_iter, alloc_freespace_pos(alloc_k.k->p, *a));  in bch2_check_alloc_key()
1074 	k = bch2_btree_iter_peek_slot(freespace_iter);  in bch2_check_alloc_key()
1075 	ret = bkey_err(k);  in bch2_check_alloc_key()
1079 	if (k.k->type != freespace_key_type &&  in bch2_check_alloc_key()
1084 		      bch2_bkey_types[k.k->type],  in bch2_check_alloc_key()
1095 		bkey_init(&update->k);  in bch2_check_alloc_key()
1096 		update->k.type	= freespace_key_type;  in bch2_check_alloc_key()
1097 		update->k.p	= freespace_iter->pos;  in bch2_check_alloc_key()
1098 		bch2_key_resize(&update->k, 1);  in bch2_check_alloc_key()
1105 	bch2_btree_iter_set_pos(bucket_gens_iter, alloc_gens_pos(alloc_k.k->p, &gens_offset));  in bch2_check_alloc_key()
1106 	k = bch2_btree_iter_peek_slot(bucket_gens_iter);  in bch2_check_alloc_key()
1107 	ret = bkey_err(k);  in bch2_check_alloc_key()
1111 	if (a->gen != alloc_gen(k, gens_offset) &&  in bch2_check_alloc_key()
1116 		      alloc_gen(k, gens_offset), a->gen,  in bch2_check_alloc_key()
1126 		if (k.k->type == KEY_TYPE_bucket_gens) {  in bch2_check_alloc_key()
1127 			bkey_reassemble(&g->k_i, k);  in bch2_check_alloc_key()
1130 			g->k.p = alloc_gens_pos(alloc_k.k->p, &gens_offset);  in bch2_check_alloc_key()
1153 	struct bkey_s_c k;  in bch2_check_alloc_hole_freespace()  local
1163 	k = bch2_btree_iter_peek_slot(freespace_iter);  in bch2_check_alloc_hole_freespace()
1164 	ret = bkey_err(k);  in bch2_check_alloc_hole_freespace()
1168 	*end = bkey_min(k.k->p, *end);  in bch2_check_alloc_hole_freespace()
1170 	if (k.k->type != KEY_TYPE_set &&  in bch2_check_alloc_hole_freespace()
1185 		bkey_init(&update->k);  in bch2_check_alloc_hole_freespace()
1186 		update->k.type	= KEY_TYPE_set;  in bch2_check_alloc_hole_freespace()
1187 		update->k.p	= freespace_iter->pos;  in bch2_check_alloc_hole_freespace()
1188 		bch2_key_resize(&update->k,  in bch2_check_alloc_hole_freespace()
1209 	struct bkey_s_c k;  in bch2_check_alloc_hole_bucket_gens()  local
1216 	k = bch2_btree_iter_peek_slot(bucket_gens_iter);  in bch2_check_alloc_hole_bucket_gens()
1217 	ret = bkey_err(k);  in bch2_check_alloc_hole_bucket_gens()
1225 	if (k.k->type == KEY_TYPE_bucket_gens) {  in bch2_check_alloc_hole_bucket_gens()
1229 		bkey_reassemble(&g.k_i, k);  in bch2_check_alloc_hole_bucket_gens()
1235 					bucket_gens_pos_to_alloc(k.k->p, i).inode,  in bch2_check_alloc_hole_bucket_gens()
1236 					bucket_gens_pos_to_alloc(k.k->p, i).offset,  in bch2_check_alloc_hole_bucket_gens()
1258 	*end = bkey_min(*end, bucket_gens_pos_to_alloc(bpos_nosnap_successor(k.k->p), 0));  in bch2_check_alloc_hole_bucket_gens()
1332 			       struct bkey_s_c k)  in bch2_check_bucket_gens_key()  argument
1337 	u64 start = bucket_gens_pos_to_alloc(k.k->p, 0).offset;  in bch2_check_bucket_gens_key()
1338 	u64 end = bucket_gens_pos_to_alloc(bpos_nosnap_successor(k.k->p), 0).offset;  in bch2_check_bucket_gens_key()
1344 	BUG_ON(k.k->type != KEY_TYPE_bucket_gens);  in bch2_check_bucket_gens_key()
1345 	bkey_reassemble(&g.k_i, k);  in bch2_check_bucket_gens_key()
1348 	dev_exists = bch2_dev_exists2(c, k.k->p.inode);  in bch2_check_bucket_gens_key()
1353 				(bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {  in bch2_check_bucket_gens_key()
1359 	ca = bch_dev_bkey_exists(c, k.k->p.inode);  in bch2_check_bucket_gens_key()
1364 			(bch2_bkey_val_to_text(&buf, c, k), buf.buf))) {  in bch2_check_bucket_gens_key()
1406 	struct bkey_s_c k;  in bch2_check_alloc_info()  local
1423 		k = bch2_get_key_or_real_bucket_hole(&iter, &hole);  in bch2_check_alloc_info()
1424 		ret = bkey_err(k);  in bch2_check_alloc_info()
1428 		if (!k.k)  in bch2_check_alloc_info()
1431 		if (k.k->type) {  in bch2_check_alloc_info()
1432 			next = bpos_nosnap_successor(k.k->p);  in bch2_check_alloc_info()
1435 						   k, &iter,  in bch2_check_alloc_info()
1442 			next = k.k->p;  in bch2_check_alloc_info()
1445 						    bkey_start_pos(k.k),  in bch2_check_alloc_info()
1449 						    bkey_start_pos(k.k),  in bch2_check_alloc_info()
1478 			BTREE_ITER_PREFETCH, k,  in bch2_check_alloc_info()
1487 		k = bch2_btree_iter_peek(&iter);  in bch2_check_alloc_info()
1488 		if (!k.k)  in bch2_check_alloc_info()
1491 		ret = bkey_err(k) ?:  in bch2_check_alloc_info()
1499 			bch2_bkey_val_to_text(&buf, c, k);  in bch2_check_alloc_info()
1514 			BTREE_ITER_PREFETCH, k,  in bch2_check_alloc_info()
1516 		bch2_check_bucket_gens_key(trans, &iter, k));  in bch2_check_alloc_info()
1535 	if (!alloc_k.k)  in bch2_check_alloc_to_lru_ref()
1569 			     lru_pos(alloc_k.k->p.inode,  in bch2_check_alloc_to_lru_ref()
1570 				     bucket_to_u64(alloc_k.k->p),  in bch2_check_alloc_to_lru_ref()
1576 	if (fsck_err_on(lru_k.k->type != KEY_TYPE_set, c,  in bch2_check_alloc_to_lru_ref()
1583 				   alloc_k.k->p.inode,  in bch2_check_alloc_to_lru_ref()
1584 				   bucket_to_u64(alloc_k.k->p),  in bch2_check_alloc_to_lru_ref()
1600 				POS_MIN, BTREE_ITER_PREFETCH, k,  in bch2_check_alloc_to_lru_refs()
1641 	struct bkey_s_c k;  in bch2_discard_one_bucket()  local
1669 	k = bch2_bkey_get_iter(trans, &iter, BTREE_ID_alloc,  in bch2_discard_one_bucket()
1672 	ret = bkey_err(k);  in bch2_discard_one_bucket()
1676 	a = bch2_alloc_to_v4_mut(trans, k);  in bch2_discard_one_bucket()
1694 				(bch2_bkey_val_to_text(&buf, c, k), buf.buf));  in bch2_discard_one_bucket()
1705 				(bch2_bkey_val_to_text(&buf, c, k), buf.buf));  in bch2_discard_one_bucket()
1720 				     k.k->p.offset * ca->mi.bucket_size,  in bch2_discard_one_bucket()
1764 				   BTREE_ID_need_discard, POS_MIN, 0, k,  in bch2_do_discards_work()
1791 	struct bpos bucket = u64_to_bucket(lru_k.k->p.offset);  in invalidate_one_bucket()
1883 				BTREE_ITER_INTENT, k,  in bch2_do_invalidates_work()
1884 			invalidate_one_bucket(trans, &iter, k, &nr_to_invalidate));  in bch2_do_invalidates_work()
1908 	struct bkey_s_c k;  in bch2_dev_freespace_init()  local
1939 		k = bch2_get_key_or_hole(&iter, end, &hole);  in bch2_dev_freespace_init()
1940 		ret = bkey_err(k);  in bch2_dev_freespace_init()
1944 		if (k.k->type) {  in bch2_dev_freespace_init()
1950 			const struct bch_alloc_v4 *a = bch2_alloc_to_v4(k, &a_convert);  in bch2_dev_freespace_init()
1952 			ret =   bch2_bucket_do_index(trans, k, a, true) ?:  in bch2_dev_freespace_init()
1967 			bkey_init(&freespace->k);  in bch2_dev_freespace_init()
1968 			freespace->k.type	= KEY_TYPE_set;  in bch2_dev_freespace_init()
1969 			freespace->k.p		= k.k->p;  in bch2_dev_freespace_init()
1970 			freespace->k.size	= k.k->size;  in bch2_dev_freespace_init()
1978 			bch2_btree_iter_set_pos(&iter, k.k->p);  in bch2_dev_freespace_init()