Lines Matching full:as

208 static void bch2_btree_node_free_never_used(struct btree_update *as,  in bch2_btree_node_free_never_used()  argument
212 struct bch_fs *c = as->c; in bch2_btree_node_free_never_used()
213 struct prealloc_nodes *p = &as->prealloc_nodes[b->c.lock.readers != NULL]; in bch2_btree_node_free_never_used()
218 BUG_ON(b->will_make_reachable != (1UL|(unsigned long) as)); in bch2_btree_node_free_never_used()
221 closure_put(&as->cl); in bch2_btree_node_free_never_used()
321 static struct btree *bch2_btree_node_alloc(struct btree_update *as, in bch2_btree_node_alloc() argument
325 struct bch_fs *c = as->c; in bch2_btree_node_alloc()
327 struct prealloc_nodes *p = &as->prealloc_nodes[!!level]; in bch2_btree_node_alloc()
344 b->c.btree_id = as->btree_id; in bch2_btree_node_alloc()
351 SET_BTREE_NODE_ID(b->data, as->btree_id); in bch2_btree_node_alloc()
366 ret = bch2_btree_node_hash_insert(&c->btree_cache, b, level, as->btree_id); in bch2_btree_node_alloc()
387 static struct btree *bch2_btree_node_alloc_replacement(struct btree_update *as, in bch2_btree_node_alloc_replacement() argument
391 struct btree *n = bch2_btree_node_alloc(as, trans, b->c.level); in bch2_btree_node_alloc_replacement()
398 if (!bch2_btree_node_format_fits(as->c, b, b->nr, &format)) in bch2_btree_node_alloc_replacement()
409 bch2_btree_sort_into(as->c, n, b); in bch2_btree_node_alloc_replacement()
415 static struct btree *__btree_root_alloc(struct btree_update *as, in __btree_root_alloc() argument
418 struct btree *b = bch2_btree_node_alloc(as, trans, level); in __btree_root_alloc()
430 static void bch2_btree_reserve_put(struct btree_update *as, struct btree_trans *trans) in bch2_btree_reserve_put() argument
432 struct bch_fs *c = as->c; in bch2_btree_reserve_put()
435 for (p = as->prealloc_nodes; in bch2_btree_reserve_put()
436 p < as->prealloc_nodes + ARRAY_SIZE(as->prealloc_nodes); in bch2_btree_reserve_put()
467 struct btree_update *as, in bch2_btree_reserve_get() argument
487 struct prealloc_nodes *p = as->prealloc_nodes + interior; in bch2_btree_reserve_get()
490 b = __bch2_btree_node_alloc(trans, &as->disk_res, cl, in bch2_btree_reserve_get()
507 static void bch2_btree_update_free(struct btree_update *as, struct btree_trans *trans) in bch2_btree_update_free() argument
509 struct bch_fs *c = as->c; in bch2_btree_update_free()
511 if (as->took_gc_lock) in bch2_btree_update_free()
513 as->took_gc_lock = false; in bch2_btree_update_free()
515 bch2_journal_pin_drop(&c->journal, &as->journal); in bch2_btree_update_free()
516 bch2_journal_pin_flush(&c->journal, &as->journal); in bch2_btree_update_free()
517 bch2_disk_reservation_put(c, &as->disk_res); in bch2_btree_update_free()
518 bch2_btree_reserve_put(as, trans); in bch2_btree_update_free()
521 as->start_time); in bch2_btree_update_free()
524 list_del(&as->unwritten_list); in bch2_btree_update_free()
525 list_del(&as->list); in bch2_btree_update_free()
527 closure_debug_destroy(&as->cl); in bch2_btree_update_free()
528 mempool_free(as, &c->btree_interior_update_pool); in bch2_btree_update_free()
539 static void btree_update_add_key(struct btree_update *as, in btree_update_add_key() argument
545 ARRAY_SIZE(as->_old_keys)); in btree_update_add_key()
558 struct btree_update *as) in btree_update_nodes_written_trans() argument
560 struct jset_entry *e = bch2_trans_jset_entry_alloc(trans, as->journal_u64s); in btree_update_nodes_written_trans()
565 memcpy(e, as->journal_entries, as->journal_u64s * sizeof(u64)); in btree_update_nodes_written_trans()
567 trans->journal_pin = &as->journal; in btree_update_nodes_written_trans()
569 for_each_keylist_key(&as->old_keys, k) { in btree_update_nodes_written_trans()
572 ret = bch2_key_trigger_old(trans, as->btree_id, level, bkey_i_to_s_c(k), in btree_update_nodes_written_trans()
578 for_each_keylist_key(&as->new_keys, k) { in btree_update_nodes_written_trans()
581 ret = bch2_key_trigger_new(trans, as->btree_id, level, bkey_i_to_s(k), in btree_update_nodes_written_trans()
590 static void btree_update_nodes_written(struct btree_update *as) in btree_update_nodes_written() argument
592 struct bch_fs *c = as->c; in btree_update_nodes_written()
602 * node here, but it won't have been marked as allocated and we'll see in btree_update_nodes_written()
614 for (i = 0; i < as->nr_old_nodes; i++) { in btree_update_nodes_written()
617 b = as->old_nodes[i]; in btree_update_nodes_written()
623 if (seq == as->old_nodes_seq[i]) in btree_update_nodes_written()
639 * which may require allocations as well. in btree_update_nodes_written()
641 ret = commit_do(trans, &as->disk_res, &journal_seq, in btree_update_nodes_written()
646 btree_update_nodes_written_trans(trans, as)); in btree_update_nodes_written()
652 if (as->b) { in btree_update_nodes_written()
654 b = as->b; in btree_update_nodes_written()
656 as->btree_id, b->c.level, b->key.k.p); in btree_update_nodes_written()
690 list_del(&as->write_blocked_list); in btree_update_nodes_written()
698 if (as->b == b) { in btree_update_nodes_written()
730 bch2_journal_pin_drop(&c->journal, &as->journal); in btree_update_nodes_written()
733 for (i = 0; i < as->nr_new_nodes; i++) { in btree_update_nodes_written()
734 b = as->new_nodes[i]; in btree_update_nodes_written()
736 BUG_ON(b->will_make_reachable != (unsigned long) as); in btree_update_nodes_written()
742 for (i = 0; i < as->nr_new_nodes; i++) { in btree_update_nodes_written()
743 b = as->new_nodes[i]; in btree_update_nodes_written()
750 for (i = 0; i < as->nr_open_buckets; i++) in btree_update_nodes_written()
751 bch2_open_bucket_put(c, c->open_buckets + as->open_buckets[i]); in btree_update_nodes_written()
753 bch2_btree_update_free(as, trans); in btree_update_nodes_written()
761 struct btree_update *as; in btree_interior_update_work() local
765 as = list_first_entry_or_null(&c->btree_interior_updates_unwritten, in btree_interior_update_work()
767 if (as && !as->nodes_written) in btree_interior_update_work()
768 as = NULL; in btree_interior_update_work()
771 if (!as) in btree_interior_update_work()
774 btree_update_nodes_written(as); in btree_interior_update_work()
780 closure_type(as, struct btree_update, cl); in CLOSURE_CALLBACK()
781 struct bch_fs *c = as->c; in CLOSURE_CALLBACK()
784 as->nodes_written = true; in CLOSURE_CALLBACK()
792 * block @b from being written until @as completes
794 static void btree_update_updated_node(struct btree_update *as, struct btree *b) in btree_update_updated_node() argument
796 struct bch_fs *c = as->c; in btree_update_updated_node()
799 list_add_tail(&as->unwritten_list, &c->btree_interior_updates_unwritten); in btree_update_updated_node()
801 BUG_ON(as->mode != BTREE_INTERIOR_NO_UPDATE); in btree_update_updated_node()
805 as->mode = BTREE_INTERIOR_UPDATING_NODE; in btree_update_updated_node()
806 as->b = b; in btree_update_updated_node()
809 list_add(&as->write_blocked_list, &b->write_blocked); in btree_update_updated_node()
820 static void btree_update_reparent(struct btree_update *as, in btree_update_reparent() argument
823 struct bch_fs *c = as->c; in btree_update_reparent()
830 bch2_journal_pin_copy(&c->journal, &as->journal, &child->journal, in btree_update_reparent()
834 static void btree_update_updated_root(struct btree_update *as, struct btree *b) in btree_update_updated_root() argument
837 struct bch_fs *c = as->c; in btree_update_updated_root()
839 BUG_ON(as->mode != BTREE_INTERIOR_NO_UPDATE); in btree_update_updated_root()
841 BUG_ON(as->journal_u64s + jset_u64s(insert->k.u64s) > in btree_update_updated_root()
842 ARRAY_SIZE(as->journal_entries)); in btree_update_updated_root()
844 as->journal_u64s += in btree_update_updated_root()
845 journal_entry_set((void *) &as->journal_entries[as->journal_u64s], in btree_update_updated_root()
851 list_add_tail(&as->unwritten_list, &c->btree_interior_updates_unwritten); in btree_update_updated_root()
853 as->mode = BTREE_INTERIOR_UPDATING_ROOT; in btree_update_updated_root()
860 * This causes @as to wait on @b to be written, before it gets to
866 * And it adds @b to the list of @as's new nodes, so that we can update sector
869 static void bch2_btree_update_add_new_node(struct btree_update *as, struct btree *b) in bch2_btree_update_add_new_node() argument
871 struct bch_fs *c = as->c; in bch2_btree_update_add_new_node()
873 closure_get(&as->cl); in bch2_btree_update_add_new_node()
876 BUG_ON(as->nr_new_nodes >= ARRAY_SIZE(as->new_nodes)); in bch2_btree_update_add_new_node()
879 as->new_nodes[as->nr_new_nodes++] = b; in bch2_btree_update_add_new_node()
880 b->will_make_reachable = 1UL|(unsigned long) as; in bch2_btree_update_add_new_node()
885 btree_update_add_key(as, &as->new_keys, b); in bch2_btree_update_add_new_node()
901 struct btree_update *as; in btree_update_drop_new_node() local
907 * When b->will_make_reachable != 0, it owns a ref on as->cl that's in btree_update_drop_new_node()
913 as = (struct btree_update *) (v & ~1UL); in btree_update_drop_new_node()
915 if (!as) { in btree_update_drop_new_node()
920 for (i = 0; i < as->nr_new_nodes; i++) in btree_update_drop_new_node()
921 if (as->new_nodes[i] == b) in btree_update_drop_new_node()
926 array_remove_item(as->new_nodes, as->nr_new_nodes, i); in btree_update_drop_new_node()
930 closure_put(&as->cl); in btree_update_drop_new_node()
933 static void bch2_btree_update_get_open_buckets(struct btree_update *as, struct btree *b) in bch2_btree_update_get_open_buckets() argument
936 as->open_buckets[as->nr_open_buckets++] = in bch2_btree_update_get_open_buckets()
951 static void bch2_btree_interior_update_will_free_node(struct btree_update *as, in bch2_btree_interior_update_will_free_node() argument
954 struct bch_fs *c = as->c; in bch2_btree_interior_update_will_free_node()
975 btree_update_reparent(as, p); in bch2_btree_interior_update_will_free_node()
997 bch2_journal_pin_copy(&c->journal, &as->journal, &w->journal, in bch2_btree_interior_update_will_free_node()
1002 bch2_journal_pin_copy(&c->journal, &as->journal, &w->journal, in bch2_btree_interior_update_will_free_node()
1019 btree_update_add_key(as, &as->old_keys, b); in bch2_btree_interior_update_will_free_node()
1021 as->old_nodes[as->nr_old_nodes] = b; in bch2_btree_interior_update_will_free_node()
1022 as->old_nodes_seq[as->nr_old_nodes] = b->data->keys.seq; in bch2_btree_interior_update_will_free_node()
1023 as->nr_old_nodes++; in bch2_btree_interior_update_will_free_node()
1026 static void bch2_btree_update_done(struct btree_update *as, struct btree_trans *trans) in bch2_btree_update_done() argument
1028 struct bch_fs *c = as->c; in bch2_btree_update_done()
1029 u64 start_time = as->start_time; in bch2_btree_update_done()
1031 BUG_ON(as->mode == BTREE_INTERIOR_NO_UPDATE); in bch2_btree_update_done()
1033 if (as->took_gc_lock) in bch2_btree_update_done()
1034 up_read(&as->c->gc_lock); in bch2_btree_update_done()
1035 as->took_gc_lock = false; in bch2_btree_update_done()
1037 bch2_btree_reserve_put(as, trans); in bch2_btree_update_done()
1039 continue_at(&as->cl, btree_update_set_nodes_written, in bch2_btree_update_done()
1040 as->c->btree_interior_update_worker); in bch2_btree_update_done()
1051 struct btree_update *as; in bch2_btree_update_start() local
1116 as = mempool_alloc(&c->btree_interior_update_pool, GFP_NOFS); in bch2_btree_update_start()
1117 memset(as, 0, sizeof(*as)); in bch2_btree_update_start()
1118 closure_init(&as->cl, NULL); in bch2_btree_update_start()
1119 as->c = c; in bch2_btree_update_start()
1120 as->start_time = start_time; in bch2_btree_update_start()
1121 as->mode = BTREE_INTERIOR_NO_UPDATE; in bch2_btree_update_start()
1122 as->took_gc_lock = true; in bch2_btree_update_start()
1123 as->btree_id = path->btree_id; in bch2_btree_update_start()
1124 as->update_level = update_level; in bch2_btree_update_start()
1125 INIT_LIST_HEAD(&as->list); in bch2_btree_update_start()
1126 INIT_LIST_HEAD(&as->unwritten_list); in bch2_btree_update_start()
1127 INIT_LIST_HEAD(&as->write_blocked_list); in bch2_btree_update_start()
1128 bch2_keylist_init(&as->old_keys, as->_old_keys); in bch2_btree_update_start()
1129 bch2_keylist_init(&as->new_keys, as->_new_keys); in bch2_btree_update_start()
1130 bch2_keylist_init(&as->parent_keys, as->inline_keys); in bch2_btree_update_start()
1133 list_add_tail(&as->list, &c->btree_interior_update_list); in bch2_btree_update_start()
1148 ret = bch2_disk_reservation_get(c, &as->disk_res, in bch2_btree_update_start()
1155 ret = bch2_btree_reserve_get(trans, as, nr_nodes, flags, NULL); in bch2_btree_update_start()
1174 ret = bch2_btree_reserve_get(trans, as, nr_nodes, flags, &cl); in bch2_btree_update_start()
1192 return as; in bch2_btree_update_start()
1194 bch2_btree_update_free(as, trans); in bch2_btree_update_start()
1221 static void bch2_btree_set_root(struct btree_update *as, in bch2_btree_set_root() argument
1226 struct bch_fs *c = as->c; in bch2_btree_set_root()
1241 btree_update_updated_root(as, b); in bch2_btree_set_root()
1255 static void bch2_insert_fixup_btree_ptr(struct btree_update *as, in bch2_insert_fixup_btree_ptr() argument
1262 struct bch_fs *c = as->c; in bch2_insert_fixup_btree_ptr()
1288 BUG_ON(as->journal_u64s + jset_u64s(insert->k.u64s) > in bch2_insert_fixup_btree_ptr()
1289 ARRAY_SIZE(as->journal_entries)); in bch2_insert_fixup_btree_ptr()
1291 as->journal_u64s += in bch2_insert_fixup_btree_ptr()
1292 journal_entry_set((void *) &as->journal_entries[as->journal_u64s], in bch2_insert_fixup_btree_ptr()
1317 __bch2_btree_insert_keys_interior(struct btree_update *as, in __bch2_btree_insert_keys_interior() argument
1339 bch2_insert_fixup_btree_ptr(as, trans, path, b, &node_iter, insert); in __bch2_btree_insert_keys_interior()
1348 static void __btree_split_node(struct btree_update *as, in __btree_split_node() argument
1443 btree_node_interior_verify(as->c, n[i]); in __btree_split_node()
1458 static void btree_split_insert_keys(struct btree_update *as, in btree_split_insert_keys() argument
1472 __bch2_btree_insert_keys_interior(as, trans, path, b, node_iter, keys); in btree_split_insert_keys()
1474 btree_node_interior_verify(as->c, b); in btree_split_insert_keys()
1478 static int btree_split(struct btree_update *as, struct btree_trans *trans, in btree_split() argument
1482 struct bch_fs *c = as->c; in btree_split()
1492 bch2_btree_interior_update_will_free_node(as, b); in btree_split()
1499 n[0] = n1 = bch2_btree_node_alloc(as, trans, b->c.level); in btree_split()
1500 n[1] = n2 = bch2_btree_node_alloc(as, trans, b->c.level); in btree_split()
1502 __btree_split_node(as, trans, b, n); in btree_split()
1505 btree_split_insert_keys(as, trans, path, n1, keys); in btree_split()
1506 btree_split_insert_keys(as, trans, path, n2, keys); in btree_split()
1513 bch2_btree_update_add_new_node(as, n1); in btree_split()
1514 bch2_btree_update_add_new_node(as, n2); in btree_split()
1518 path1 = get_unlocked_mut_path(trans, as->btree_id, n1->c.level, n1->key.k.p); in btree_split()
1523 path2 = get_unlocked_mut_path(trans, as->btree_id, n2->c.level, n2->key.k.p); in btree_split()
1533 bch2_keylist_add(&as->parent_keys, &n1->key); in btree_split()
1534 bch2_keylist_add(&as->parent_keys, &n2->key); in btree_split()
1538 n3 = __btree_root_alloc(as, trans, b->c.level + 1); in btree_split()
1540 bch2_btree_update_add_new_node(as, n3); in btree_split()
1552 btree_split_insert_keys(as, trans, path, n3, &as->parent_keys); in btree_split()
1557 n1 = bch2_btree_node_alloc_replacement(as, trans, b); in btree_split()
1560 btree_split_insert_keys(as, trans, path, n1, keys); in btree_split()
1565 bch2_btree_update_add_new_node(as, n1); in btree_split()
1568 path1 = get_unlocked_mut_path(trans, as->btree_id, n1->c.level, n1->key.k.p); in btree_split()
1574 bch2_keylist_add(&as->parent_keys, &n1->key); in btree_split()
1581 ret = bch2_btree_insert_node(as, trans, path, parent, &as->parent_keys, flags); in btree_split()
1585 bch2_btree_set_root(as, trans, trans->paths + path, n3); in btree_split()
1588 bch2_btree_set_root(as, trans, trans->paths + path, n1); in btree_split()
1592 bch2_btree_update_get_open_buckets(as, n3); in btree_split()
1596 bch2_btree_update_get_open_buckets(as, n2); in btree_split()
1599 bch2_btree_update_get_open_buckets(as, n1); in btree_split()
1640 bch2_btree_node_free_never_used(as, trans, n3); in btree_split()
1642 bch2_btree_node_free_never_used(as, trans, n2); in btree_split()
1643 bch2_btree_node_free_never_used(as, trans, n1); in btree_split()
1648 bch2_btree_insert_keys_interior(struct btree_update *as, in bch2_btree_insert_keys_interior() argument
1657 __bch2_btree_insert_keys_interior(as, trans, path, b, in bch2_btree_insert_keys_interior()
1660 btree_update_updated_node(as, b); in bch2_btree_insert_keys_interior()
1671 * @as: btree_update object
1680 * Inserts as many keys as it can into a given btree node, splitting it if full.
1684 static int bch2_btree_insert_node(struct btree_update *as, struct btree_trans *trans, in bch2_btree_insert_node() argument
1688 struct bch_fs *c = as->c; in bch2_btree_insert_node()
1698 BUG_ON(!as || as->b); in bch2_btree_insert_node()
1714 bch2_btree_insert_keys_interior(as, trans, path, b, keys); in bch2_btree_insert_node()
1737 if (b->c.level >= as->update_level) { in bch2_btree_insert_node()
1742 return btree_split(as, trans, path_idx, b, keys, flags); in bch2_btree_insert_node()
1752 struct btree_update *as; in bch2_btree_split_leaf() local
1756 as = bch2_btree_update_start(trans, trans->paths + path, in bch2_btree_split_leaf()
1759 if (IS_ERR(as)) in bch2_btree_split_leaf()
1760 return PTR_ERR(as); in bch2_btree_split_leaf()
1762 ret = btree_split(as, trans, path, b, NULL, flags); in bch2_btree_split_leaf()
1764 bch2_btree_update_free(as, trans); in bch2_btree_split_leaf()
1768 bch2_btree_update_done(as, trans); in bch2_btree_split_leaf()
1785 struct btree_update *as; in __bch2_foreground_maybe_merge() local
1877 as = bch2_btree_update_start(trans, trans->paths + path, level, false, in __bch2_foreground_maybe_merge()
1879 ret = PTR_ERR_OR_ZERO(as); in __bch2_foreground_maybe_merge()
1885 bch2_btree_interior_update_will_free_node(as, b); in __bch2_foreground_maybe_merge()
1886 bch2_btree_interior_update_will_free_node(as, m); in __bch2_foreground_maybe_merge()
1888 n = bch2_btree_node_alloc(as, trans, b->c.level); in __bch2_foreground_maybe_merge()
1904 bch2_btree_update_add_new_node(as, n); in __bch2_foreground_maybe_merge()
1914 bch2_keylist_add(&as->parent_keys, &delete); in __bch2_foreground_maybe_merge()
1915 bch2_keylist_add(&as->parent_keys, &n->key); in __bch2_foreground_maybe_merge()
1919 ret = bch2_btree_insert_node(as, trans, path, parent, &as->parent_keys, flags); in __bch2_foreground_maybe_merge()
1925 bch2_btree_update_get_open_buckets(as, n); in __bch2_foreground_maybe_merge()
1937 bch2_btree_update_done(as, trans); in __bch2_foreground_maybe_merge()
1948 bch2_btree_node_free_never_used(as, trans, n); in __bch2_foreground_maybe_merge()
1949 bch2_btree_update_free(as, trans); in __bch2_foreground_maybe_merge()
1960 struct btree_update *as; in bch2_btree_node_rewrite() local
1968 as = bch2_btree_update_start(trans, path, b->c.level, false, flags); in bch2_btree_node_rewrite()
1969 ret = PTR_ERR_OR_ZERO(as); in bch2_btree_node_rewrite()
1973 bch2_btree_interior_update_will_free_node(as, b); in bch2_btree_node_rewrite()
1975 n = bch2_btree_node_alloc_replacement(as, trans, b); in bch2_btree_node_rewrite()
1978 bch2_btree_update_add_new_node(as, n); in bch2_btree_node_rewrite()
1989 bch2_keylist_add(&as->parent_keys, &n->key); in bch2_btree_node_rewrite()
1990 ret = bch2_btree_insert_node(as, trans, iter->path, in bch2_btree_node_rewrite()
1991 parent, &as->parent_keys, flags); in bch2_btree_node_rewrite()
1995 bch2_btree_set_root(as, trans, btree_iter_path(trans, iter), n); in bch2_btree_node_rewrite()
1998 bch2_btree_update_get_open_buckets(as, n); in bch2_btree_node_rewrite()
2006 bch2_btree_update_done(as, trans); in bch2_btree_node_rewrite()
2013 bch2_btree_node_free_never_used(as, trans, n); in bch2_btree_node_rewrite()
2014 bch2_btree_update_free(as, trans); in bch2_btree_node_rewrite()
2395 struct btree_update *as; in bch2_btree_updates_to_text() local
2398 list_for_each_entry(as, &c->btree_interior_update_list, list) in bch2_btree_updates_to_text()
2400 as, in bch2_btree_updates_to_text()
2401 as->mode, in bch2_btree_updates_to_text()
2402 as->nodes_written, in bch2_btree_updates_to_text()
2403 closure_nr_remaining(&as->cl), in bch2_btree_updates_to_text()
2404 as->journal.seq); in bch2_btree_updates_to_text()