Lines Matching +full:k +full:- +full:to +full:- +full:j
1 // SPDX-License-Identifier: GPL-2.0
19 * We have two parallel sets of counters to be concerned with, and both must be
22 * - Persistent/on disk accounting, stored in the accounting btree and updated
23 * via btree write buffer updates that treat new accounting keys as deltas to
24 * apply to existing values. But reading from a write buffer btree is
27 * - In memory accounting, where accounting is stored as an array of percpu
31 * Cheap to read, but non persistent.
35 * the output of these triggers are deltas to the various counters.
38 * counters in the disk accounting key are deltas that will be applied to the
42 * To do a disk accounting update:
43 * - initialize a disk_accounting_pos, to specify which counter is being update
44 * - initialize counter deltas, as an array of 1-3 s64s
45 * - call bch2_disk_accounting_mod()
47 * This queues up the accounting update to be done at transaction commit time.
50 * The transaction commit path is responsible for propagating updates to the in
55 * buffer; this is used by journal replay to determine which updates have been
61 * bch2_accounting_mem_mod() tells it to.
71 static inline void accounting_key_init(struct bkey_i *k, struct disk_accounting_pos *pos, in accounting_key_init() argument
74 struct bkey_i_accounting *acc = bkey_accounting_init(k); in accounting_key_init()
76 acc->k.p = disk_accounting_pos_to_bpos(pos); in accounting_key_init()
77 set_bkey_val_u64s(&acc->k, sizeof(struct bch_accounting) / sizeof(u64) + nr); in accounting_key_init()
79 memcpy_u64s_small(acc->v.d, d, nr); in accounting_key_init()
85 struct disk_accounting_pos *k, in bch2_disk_accounting_mod() argument
89 switch (k->type) { in bch2_disk_accounting_mod()
91 bubble_sort(k->replicas.devs, k->replicas.nr_devs, u8_cmp); in bch2_disk_accounting_mod()
97 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i; in bch2_disk_accounting_mod()
99 accounting_key_init(&k_i.k, k, d, nr); in bch2_disk_accounting_mod()
102 int ret = bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true); in bch2_disk_accounting_mod()
103 if (ret == -BCH_ERR_btree_insert_need_mark_replicas) in bch2_disk_accounting_mod()
105 bch2_accounting_update_sb_one(trans->c, disk_accounting_pos_to_bpos(k))) ?: in bch2_disk_accounting_mod()
106 bch2_accounting_mem_add(trans, bkey_i_to_s_c_accounting(&k_i.k), true); in bch2_disk_accounting_mod()
109 return bch2_trans_update_buffered(trans, BTREE_ID_accounting, &k_i.k); in bch2_disk_accounting_mod()
143 int bch2_accounting_validate(struct bch_fs *c, struct bkey_s_c k, in bch2_accounting_validate() argument
147 bpos_to_disk_accounting_pos(&acc_k, k.k->p); in bch2_accounting_validate()
152 bversion_zero(k.k->bversion), in bch2_accounting_validate()
202 bkey_fsck_err_on(bch2_accounting_counters(k.k) != bch2_accounting_type_nr_counters[acc_k.type], in bch2_accounting_validate()
205 bch2_accounting_counters(k.k), bch2_accounting_type_nr_counters[acc_k.type]); in bch2_accounting_validate()
210 void bch2_accounting_key_to_text(struct printbuf *out, struct disk_accounting_pos *k) in bch2_accounting_key_to_text() argument
212 if (k->type >= BCH_DISK_ACCOUNTING_TYPE_NR) { in bch2_accounting_key_to_text()
213 prt_printf(out, "unknown type %u", k->type); in bch2_accounting_key_to_text()
217 prt_str(out, disk_accounting_type_strs[k->type]); in bch2_accounting_key_to_text()
220 switch (k->type) { in bch2_accounting_key_to_text()
224 prt_printf(out, "replicas=%u", k->persistent_reserved.nr_replicas); in bch2_accounting_key_to_text()
227 bch2_replicas_entry_to_text(out, &k->replicas); in bch2_accounting_key_to_text()
230 prt_printf(out, "dev=%u data_type=", k->dev_data_type.dev); in bch2_accounting_key_to_text()
231 bch2_prt_data_type(out, k->dev_data_type.data_type); in bch2_accounting_key_to_text()
234 bch2_prt_compression_type(out, k->compression.type); in bch2_accounting_key_to_text()
237 prt_printf(out, "id=%u", k->snapshot.id); in bch2_accounting_key_to_text()
241 bch2_btree_id_to_text(out, k->btree.id); in bch2_accounting_key_to_text()
246 void bch2_accounting_to_text(struct printbuf *out, struct bch_fs *c, struct bkey_s_c k) in bch2_accounting_to_text() argument
248 struct bkey_s_c_accounting acc = bkey_s_c_to_accounting(k); in bch2_accounting_to_text()
250 bpos_to_disk_accounting_pos(&acc_k, k.k->p); in bch2_accounting_to_text()
254 for (unsigned i = 0; i < bch2_accounting_counters(k.k); i++) in bch2_accounting_to_text()
255 prt_printf(out, " %lli", acc.v->d[i]); in bch2_accounting_to_text()
258 void bch2_accounting_swab(struct bkey_s k) in bch2_accounting_swab() argument
260 for (u64 *p = (u64 *) k.v; in bch2_accounting_swab()
261 p < (u64 *) bkey_val_end(k); in bch2_accounting_swab()
269 unsafe_memcpy(r, &acc->replicas, in __accounting_to_replicas()
270 replicas_entry_bytes(&acc->replicas), in __accounting_to_replicas()
302 for (struct jset_entry *i = trans->journal_entries; in bch2_accounting_update_sb()
303 i != (void *) ((u64 *) trans->journal_entries + trans->journal_entries_u64s); in bch2_accounting_update_sb()
305 if (jset_entry_is_key(i) && i->start->k.type == KEY_TYPE_accounting) { in bch2_accounting_update_sb()
306 int ret = bch2_accounting_update_sb_one(trans->c, i->start->k.p); in bch2_accounting_update_sb()
316 struct bch_accounting_mem *acc = &c->accounting; in __bch2_accounting_mem_insert()
319 if (eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), in __bch2_accounting_mem_insert()
320 accounting_pos_cmp, &a.k->p) < acc->k.nr) in __bch2_accounting_mem_insert()
324 .pos = a.k->p, in __bch2_accounting_mem_insert()
325 .bversion = a.k->bversion, in __bch2_accounting_mem_insert()
326 .nr_counters = bch2_accounting_counters(a.k), in __bch2_accounting_mem_insert()
334 if (acc->gc_running) { in __bch2_accounting_mem_insert()
341 if (darray_push(&acc->k, n)) in __bch2_accounting_mem_insert()
344 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), in __bch2_accounting_mem_insert()
358 return -BCH_ERR_ENOMEM_disk_accounting; in __bch2_accounting_mem_insert()
367 accounting_to_replicas(&r.e, a.k->p) && in bch2_accounting_mem_insert()
369 return -BCH_ERR_btree_insert_need_mark_replicas; in bch2_accounting_mem_insert()
371 percpu_up_read(&c->mark_lock); in bch2_accounting_mem_insert()
372 percpu_down_write(&c->mark_lock); in bch2_accounting_mem_insert()
374 percpu_up_write(&c->mark_lock); in bch2_accounting_mem_insert()
375 percpu_down_read(&c->mark_lock); in bch2_accounting_mem_insert()
385 accounting_to_replicas(&r.e, a.k->p) && in bch2_accounting_mem_insert_locked()
387 return -BCH_ERR_btree_insert_need_mark_replicas; in bch2_accounting_mem_insert_locked()
394 for (unsigned i = 0; i < e->nr_counters; i++) in accounting_mem_entry_is_zero()
395 if (percpu_u64_get(e->v[0] + i) || in accounting_mem_entry_is_zero()
396 (e->v[1] && in accounting_mem_entry_is_zero()
397 percpu_u64_get(e->v[1] + i))) in accounting_mem_entry_is_zero()
404 struct bch_accounting_mem *acc = &c->accounting; in bch2_accounting_mem_gc()
406 percpu_down_write(&c->mark_lock); in bch2_accounting_mem_gc()
407 struct accounting_mem_entry *dst = acc->k.data; in bch2_accounting_mem_gc()
409 darray_for_each(acc->k, src) { in bch2_accounting_mem_gc()
411 free_percpu(src->v[0]); in bch2_accounting_mem_gc()
412 free_percpu(src->v[1]); in bch2_accounting_mem_gc()
418 acc->k.nr = dst - acc->k.data; in bch2_accounting_mem_gc()
419 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), in bch2_accounting_mem_gc()
421 percpu_up_write(&c->mark_lock); in bch2_accounting_mem_gc()
429 * with something more general, it exists to support the ioctl used by the
434 struct bch_accounting_mem *acc = &c->accounting; in bch2_fs_replicas_usage_read()
439 percpu_down_read(&c->mark_lock); in bch2_fs_replicas_usage_read()
440 darray_for_each(acc->k, i) { in bch2_fs_replicas_usage_read()
446 if (!accounting_to_replicas(&u.r.r, i->pos)) in bch2_fs_replicas_usage_read()
450 bch2_accounting_mem_read_counters(acc, i - acc->k.data, §ors, 1, false); in bch2_fs_replicas_usage_read()
458 usage->nr += replicas_usage_bytes(&u.r); in bch2_fs_replicas_usage_read()
460 percpu_up_read(&c->mark_lock); in bch2_fs_replicas_usage_read()
470 struct bch_accounting_mem *acc = &c->accounting; in bch2_fs_accounting_read()
475 percpu_down_read(&c->mark_lock); in bch2_fs_accounting_read()
476 darray_for_each(acc->k, i) { in bch2_fs_accounting_read()
478 bpos_to_disk_accounting_pos(&a_p, i->pos); in bch2_fs_accounting_read()
484 sizeof(u64) * i->nr_counters); in bch2_fs_accounting_read()
490 set_bkey_val_u64s(&a_out->k, i->nr_counters); in bch2_fs_accounting_read()
491 a_out->k.p = i->pos; in bch2_fs_accounting_read()
492 bch2_accounting_mem_read_counters(acc, i - acc->k.data, in bch2_fs_accounting_read()
493 a_out->v.d, i->nr_counters, false); in bch2_fs_accounting_read()
496 out_buf->nr += bkey_bytes(&a_out->k); in bch2_fs_accounting_read()
499 percpu_up_read(&c->mark_lock); in bch2_fs_accounting_read()
508 darray_for_each(acc->k, e) { in bch2_accounting_free_counters()
509 free_percpu(e->v[gc]); in bch2_accounting_free_counters()
510 e->v[gc] = NULL; in bch2_accounting_free_counters()
516 struct bch_accounting_mem *acc = &c->accounting; in bch2_gc_accounting_start()
519 percpu_down_write(&c->mark_lock); in bch2_gc_accounting_start()
520 darray_for_each(acc->k, e) { in bch2_gc_accounting_start()
521 e->v[1] = __alloc_percpu_gfp(e->nr_counters * sizeof(u64), in bch2_gc_accounting_start()
523 if (!e->v[1]) { in bch2_gc_accounting_start()
525 ret = -BCH_ERR_ENOMEM_disk_accounting; in bch2_gc_accounting_start()
530 acc->gc_running = !ret; in bch2_gc_accounting_start()
531 percpu_up_write(&c->mark_lock); in bch2_gc_accounting_start()
538 struct bch_accounting_mem *acc = &c->accounting; in bch2_gc_accounting_done()
544 percpu_down_write(&c->mark_lock); in bch2_gc_accounting_done()
546 unsigned idx = eytzinger0_find_ge(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), in bch2_gc_accounting_done()
549 if (idx >= acc->k.nr) in bch2_gc_accounting_done()
552 struct accounting_mem_entry *e = acc->k.data + idx; in bch2_gc_accounting_done()
553 pos = bpos_successor(e->pos); in bch2_gc_accounting_done()
556 bpos_to_disk_accounting_pos(&acc_k, e->pos); in bch2_gc_accounting_done()
564 unsigned nr = e->nr_counters; in bch2_gc_accounting_done()
574 for (unsigned j = 0; j < nr; j++) in bch2_gc_accounting_done() local
575 prt_printf(&buf, " %llu", dst_v[j]); in bch2_gc_accounting_done()
578 for (unsigned j = 0; j < nr; j++) in bch2_gc_accounting_done() local
579 prt_printf(&buf, " %llu", src_v[j]); in bch2_gc_accounting_done()
581 for (unsigned j = 0; j < nr; j++) in bch2_gc_accounting_done() local
582 src_v[j] -= dst_v[j]; in bch2_gc_accounting_done()
585 percpu_up_write(&c->mark_lock); in bch2_gc_accounting_done()
588 percpu_down_write(&c->mark_lock); in bch2_gc_accounting_done()
592 if (!test_bit(BCH_FS_may_go_rw, &c->flags)) { in bch2_gc_accounting_done()
593 memset(&trans->fs_usage_delta, 0, sizeof(trans->fs_usage_delta)); in bch2_gc_accounting_done()
594 struct { __BKEY_PADDED(k, BCH_ACCOUNTING_MAX_COUNTERS); } k_i; in bch2_gc_accounting_done()
596 accounting_key_init(&k_i.k, &acc_k, src_v, nr); in bch2_gc_accounting_done()
598 bkey_i_to_s_c_accounting(&k_i.k), in bch2_gc_accounting_done()
602 struct bch_fs_usage_base *dst = this_cpu_ptr(c->usage); in bch2_gc_accounting_done()
603 struct bch_fs_usage_base *src = &trans->fs_usage_delta; in bch2_gc_accounting_done()
612 percpu_up_write(&c->mark_lock); in bch2_gc_accounting_done()
619 static int accounting_read_key(struct btree_trans *trans, struct bkey_s_c k) in accounting_read_key() argument
621 struct bch_fs *c = trans->c; in accounting_read_key()
623 if (k.k->type != KEY_TYPE_accounting) in accounting_read_key()
626 percpu_down_read(&c->mark_lock); in accounting_read_key()
627 int ret = bch2_accounting_mem_mod_locked(trans, bkey_s_c_to_accounting(k), in accounting_read_key()
629 percpu_up_read(&c->mark_lock); in accounting_read_key()
637 struct bch_fs *c = trans->c; in bch2_disk_accounting_validate_late()
639 int ret = 0, invalid_dev = -1; in bch2_disk_accounting_validate_late()
669 percpu_up_write(&c->mark_lock); in bch2_disk_accounting_validate_late()
673 percpu_down_write(&c->mark_lock); in bch2_disk_accounting_validate_late()
691 "accounting entry points to invalid device %i\n%s", in bch2_disk_accounting_validate_late()
697 v[i] = -v[i]; in bch2_disk_accounting_validate_late()
701 -BCH_ERR_remove_disk_accounting_entry; in bch2_disk_accounting_validate_late()
703 ret = -BCH_ERR_remove_disk_accounting_entry; in bch2_disk_accounting_validate_late()
714 struct bch_accounting_mem *acc = &c->accounting; in bch2_accounting_read()
719 * We might run more than once if we rewind to start topology repair or in bch2_accounting_read()
720 * btree node scan - and those might cause us to get different results, in bch2_accounting_read()
725 percpu_down_write(&c->mark_lock); in bch2_accounting_read()
726 darray_for_each(acc->k, e) in bch2_accounting_read()
727 percpu_memset(e->v[0], 0, sizeof(u64) * e->nr_counters); in bch2_accounting_read()
729 percpu_memset(ca->usage, 0, sizeof(*ca->usage)); in bch2_accounting_read()
730 percpu_memset(c->usage, 0, sizeof(*c->usage)); in bch2_accounting_read()
731 percpu_up_write(&c->mark_lock); in bch2_accounting_read()
738 BTREE_ITER_prefetch|BTREE_ITER_all_snapshots, k, ({ in bch2_accounting_read()
740 struct bkey_s_c k = bch2_btree_path_peek_slot_exact(btree_iter_path(trans, &iter), &u); in bch2_accounting_read()
742 if (k.k->type != KEY_TYPE_accounting) in bch2_accounting_read()
746 bpos_to_disk_accounting_pos(&acc_k, k.k->p); in bch2_accounting_read()
759 accounting_read_key(trans, k); in bch2_accounting_read()
764 struct journal_keys *keys = &c->journal_keys; in bch2_accounting_read()
765 struct journal_key *dst = keys->data; in bch2_accounting_read()
766 move_gap(keys, keys->nr); in bch2_accounting_read()
769 if (i->k->k.type == KEY_TYPE_accounting) { in bch2_accounting_read()
771 bpos_to_disk_accounting_pos(&acc_k, i->k->k.p); in bch2_accounting_read()
776 struct bkey_s_c k = bkey_i_to_s_c(i->k); in bch2_accounting_read() local
777 unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr, in bch2_accounting_read()
778 sizeof(acc->k.data[0]), in bch2_accounting_read()
779 accounting_pos_cmp, &k.k->p); in bch2_accounting_read()
781 bool applied = idx < acc->k.nr && in bch2_accounting_read()
782 bversion_cmp(acc->k.data[idx].bversion, k.k->bversion) >= 0; in bch2_accounting_read()
788 i[1].k->k.type == KEY_TYPE_accounting && in bch2_accounting_read()
790 WARN_ON(bversion_cmp(i[0].k->k.bversion, i[1].k->k.bversion) >= 0); in bch2_accounting_read()
794 bch2_accounting_accumulate(bkey_i_to_accounting(i[1].k), in bch2_accounting_read()
795 bkey_s_c_to_accounting(k)); in bch2_accounting_read()
799 ret = accounting_read_key(trans, k); in bch2_accounting_read()
806 keys->gap = keys->nr = dst - keys->data; in bch2_accounting_read()
808 percpu_down_write(&c->mark_lock); in bch2_accounting_read()
810 darray_for_each_reverse(acc->k, i) { in bch2_accounting_read()
812 bpos_to_disk_accounting_pos(&acc_k, i->pos); in bch2_accounting_read()
817 for (unsigned j = 0; j < i->nr_counters; j++) in bch2_accounting_read() local
818 v[j] = percpu_u64_get(i->v[0] + j); in bch2_accounting_read()
822 * nonexistent - it might point to an invalid device. in bch2_accounting_read()
824 * Remove it, so that if it's re-added it gets re-marked in the in bch2_accounting_read()
827 ret = bch2_is_zero(v, sizeof(v[0]) * i->nr_counters) in bch2_accounting_read()
828 ? -BCH_ERR_remove_disk_accounting_entry in bch2_accounting_read()
829 : bch2_disk_accounting_validate_late(trans, acc_k, v, i->nr_counters); in bch2_accounting_read()
831 if (ret == -BCH_ERR_remove_disk_accounting_entry) { in bch2_accounting_read()
832 free_percpu(i->v[0]); in bch2_accounting_read()
833 free_percpu(i->v[1]); in bch2_accounting_read()
834 darray_remove_item(&acc->k, i); in bch2_accounting_read()
843 eytzinger0_sort(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), in bch2_accounting_read()
847 struct bch_fs_usage_base *usage = this_cpu_ptr(c->usage); in bch2_accounting_read()
849 for (unsigned i = 0; i < acc->k.nr; i++) { in bch2_accounting_read()
850 struct disk_accounting_pos k; in bch2_accounting_read() local
851 bpos_to_disk_accounting_pos(&k, acc->k.data[i].pos); in bch2_accounting_read()
856 switch (k.type) { in bch2_accounting_read()
858 usage->reserved += v[0] * k.persistent_reserved.nr_replicas; in bch2_accounting_read()
861 fs_usage_data_type_to_base(usage, k.replicas.data_type, v[0]); in bch2_accounting_read()
865 struct bch_dev *ca = bch2_dev_rcu_noerror(c, k.dev_data_type.dev); in bch2_accounting_read()
867 struct bch_dev_usage_type __percpu *d = &ca->usage->d[k.dev_data_type.data_type]; in bch2_accounting_read()
868 percpu_u64_set(&d->buckets, v[0]); in bch2_accounting_read()
869 percpu_u64_set(&d->sectors, v[1]); in bch2_accounting_read()
870 percpu_u64_set(&d->fragmented, v[2]); in bch2_accounting_read()
872 if (k.dev_data_type.data_type == BCH_DATA_sb || in bch2_accounting_read()
873 k.dev_data_type.data_type == BCH_DATA_journal) in bch2_accounting_read()
874 usage->hidden += v[0] * ca->mi.bucket_size; in bch2_accounting_read()
882 percpu_up_write(&c->mark_lock); in bch2_accounting_read()
895 BTREE_ITER_all_snapshots, k, NULL, NULL, 0, ({ in bch2_dev_usage_remove()
897 bpos_to_disk_accounting_pos(&acc, k.k->p); in bch2_dev_usage_remove()
901 ? bch2_btree_bit_mod_buffered(trans, BTREE_ID_accounting, k.k->p, 0) in bch2_dev_usage_remove()
909 struct bch_fs *c = ca->fs; in bch2_dev_usage_init()
910 u64 v[3] = { ca->mi.nbuckets - ca->mi.first_bucket, 0, 0 }; in bch2_dev_usage_init()
915 .dev = ca->dev_idx, in bch2_dev_usage_init()
931 BTREE_ITER_all_snapshots, k, ({ in bch2_verify_accounting_clean()
933 struct bkey_s_c_accounting a = bkey_s_c_to_accounting(k); in bch2_verify_accounting_clean()
934 unsigned nr = bch2_accounting_counters(k.k); in bch2_verify_accounting_clean()
937 bpos_to_disk_accounting_pos(&acc_k, k.k->p); in bch2_verify_accounting_clean()
950 bch2_accounting_mem_read(c, k.k->p, v, nr); in bch2_verify_accounting_clean()
952 if (memcmp(a.v->d, v, nr * sizeof(u64))) { in bch2_verify_accounting_clean()
955 bch2_bkey_val_to_text(&buf, c, k); in bch2_verify_accounting_clean()
957 for (unsigned j = 0; j < nr; j++) in bch2_verify_accounting_clean()
958 prt_printf(&buf, " %llu", v[j]); in bch2_verify_accounting_clean()
967 base.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0]; in bch2_verify_accounting_clean()
970 fs_usage_data_type_to_base(&base, acc_k.replicas.data_type, a.v->d[0]); in bch2_verify_accounting_clean()
980 v[0] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].buckets); in bch2_verify_accounting_clean()
981 v[1] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].sectors); in bch2_verify_accounting_clean()
982 v[2] = percpu_u64_get(&ca->usage->d[acc_k.dev_data_type.data_type].fragmented); in bch2_verify_accounting_clean()
985 if (memcmp(a.v->d, v, 3 * sizeof(u64))) { in bch2_verify_accounting_clean()
988 bch2_bkey_val_to_text(&buf, c, k); in bch2_verify_accounting_clean()
990 for (unsigned j = 0; j < nr; j++) in bch2_verify_accounting_clean()
991 prt_printf(&buf, " %llu", v[j]); in bch2_verify_accounting_clean()
1003 acc_u64s_percpu(&base_inmem.hidden, &c->usage->hidden, sizeof(base_inmem) / sizeof(u64)); in bch2_verify_accounting_clean()
1023 lockdep_assert_held(&c->mark_lock); in bch2_accounting_gc_free()
1025 struct bch_accounting_mem *acc = &c->accounting; in bch2_accounting_gc_free()
1028 acc->gc_running = false; in bch2_accounting_gc_free()
1033 struct bch_accounting_mem *acc = &c->accounting; in bch2_fs_accounting_exit()
1036 darray_exit(&acc->k); in bch2_fs_accounting_exit()