1 /* SPDX-License-Identifier: GPL-2.0 */ 2 #ifndef _BCACHEFS_DISK_ACCOUNTING_H 3 #define _BCACHEFS_DISK_ACCOUNTING_H 4 5 #include "btree_update.h" 6 #include "eytzinger.h" 7 #include "sb-members.h" 8 9 static inline void bch2_u64s_neg(u64 *v, unsigned nr) 10 { 11 for (unsigned i = 0; i < nr; i++) 12 v[i] = -v[i]; 13 } 14 15 static inline unsigned bch2_accounting_counters(const struct bkey *k) 16 { 17 return bkey_val_u64s(k) - offsetof(struct bch_accounting, d) / sizeof(u64); 18 } 19 20 static inline void bch2_accounting_neg(struct bkey_s_accounting a) 21 { 22 bch2_u64s_neg(a.v->d, bch2_accounting_counters(a.k)); 23 } 24 25 static inline bool bch2_accounting_key_is_zero(struct bkey_s_c_accounting a) 26 { 27 for (unsigned i = 0; i < bch2_accounting_counters(a.k); i++) 28 if (a.v->d[i]) 29 return false; 30 return true; 31 } 32 33 static inline void bch2_accounting_accumulate(struct bkey_i_accounting *dst, 34 struct bkey_s_c_accounting src) 35 { 36 for (unsigned i = 0; 37 i < min(bch2_accounting_counters(&dst->k), 38 bch2_accounting_counters(src.k)); 39 i++) 40 dst->v.d[i] += src.v->d[i]; 41 42 if (bversion_cmp(dst->k.bversion, src.k->bversion) < 0) 43 dst->k.bversion = src.k->bversion; 44 } 45 46 static inline void fs_usage_data_type_to_base(struct bch_fs_usage_base *fs_usage, 47 enum bch_data_type data_type, 48 s64 sectors) 49 { 50 switch (data_type) { 51 case BCH_DATA_btree: 52 fs_usage->btree += sectors; 53 break; 54 case BCH_DATA_user: 55 case BCH_DATA_parity: 56 fs_usage->data += sectors; 57 break; 58 case BCH_DATA_cached: 59 fs_usage->cached += sectors; 60 break; 61 default: 62 break; 63 } 64 } 65 66 static inline void bpos_to_disk_accounting_pos(struct disk_accounting_pos *acc, struct bpos p) 67 { 68 BUILD_BUG_ON(sizeof(*acc) != sizeof(p)); 69 70 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 71 acc->_pad = p; 72 #else 73 memcpy_swab(acc, &p, sizeof(p)); 74 #endif 75 } 76 77 static inline struct bpos disk_accounting_pos_to_bpos(struct disk_accounting_pos *acc) 78 { 79 struct bpos p; 80 #if __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__ 81 p = acc->_pad; 82 #else 83 memcpy_swab(&p, acc, sizeof(p)); 84 #endif 85 return p; 86 } 87 88 int bch2_disk_accounting_mod(struct btree_trans *, struct disk_accounting_pos *, 89 s64 *, unsigned, bool); 90 91 #define disk_accounting_key_init(_k, _type, ...) \ 92 do { \ 93 memset(&(_k), 0, sizeof(_k)); \ 94 (_k).type = BCH_DISK_ACCOUNTING_##_type; \ 95 (_k)._type = (struct bch_acct_##_type) { __VA_ARGS__ }; \ 96 } while (0) 97 98 #define bch2_disk_accounting_mod2_nr(_trans, _gc, _v, _nr, ...) \ 99 ({ \ 100 struct disk_accounting_pos pos; \ 101 disk_accounting_key_init(pos, __VA_ARGS__); \ 102 bch2_disk_accounting_mod(trans, &pos, _v, _nr, _gc); \ 103 }) 104 105 #define bch2_disk_accounting_mod2(_trans, _gc, _v, ...) \ 106 bch2_disk_accounting_mod2_nr(_trans, _gc, _v, ARRAY_SIZE(_v), __VA_ARGS__) 107 108 int bch2_mod_dev_cached_sectors(struct btree_trans *, unsigned, s64, bool); 109 110 int bch2_accounting_validate(struct bch_fs *, struct bkey_s_c, 111 struct bkey_validate_context); 112 void bch2_accounting_key_to_text(struct printbuf *, struct disk_accounting_pos *); 113 void bch2_accounting_to_text(struct printbuf *, struct bch_fs *, struct bkey_s_c); 114 void bch2_accounting_swab(struct bkey_s); 115 116 #define bch2_bkey_ops_accounting ((struct bkey_ops) { \ 117 .key_validate = bch2_accounting_validate, \ 118 .val_to_text = bch2_accounting_to_text, \ 119 .swab = bch2_accounting_swab, \ 120 .min_val_size = 8, \ 121 }) 122 123 int bch2_accounting_update_sb(struct btree_trans *); 124 125 static inline int accounting_pos_cmp(const void *_l, const void *_r) 126 { 127 const struct bpos *l = _l, *r = _r; 128 129 return bpos_cmp(*l, *r); 130 } 131 132 enum bch_accounting_mode { 133 BCH_ACCOUNTING_normal, 134 BCH_ACCOUNTING_gc, 135 BCH_ACCOUNTING_read, 136 }; 137 138 int bch2_accounting_mem_insert(struct bch_fs *, struct bkey_s_c_accounting, enum bch_accounting_mode); 139 void bch2_accounting_mem_gc(struct bch_fs *); 140 141 static inline bool bch2_accounting_is_mem(struct disk_accounting_pos acc) 142 { 143 return acc.type < BCH_DISK_ACCOUNTING_TYPE_NR && 144 acc.type != BCH_DISK_ACCOUNTING_inum; 145 } 146 147 /* 148 * Update in memory counters so they match the btree update we're doing; called 149 * from transaction commit path 150 */ 151 static inline int bch2_accounting_mem_mod_locked(struct btree_trans *trans, 152 struct bkey_s_c_accounting a, 153 enum bch_accounting_mode mode) 154 { 155 struct bch_fs *c = trans->c; 156 struct bch_accounting_mem *acc = &c->accounting; 157 struct disk_accounting_pos acc_k; 158 bpos_to_disk_accounting_pos(&acc_k, a.k->p); 159 bool gc = mode == BCH_ACCOUNTING_gc; 160 161 if (gc && !acc->gc_running) 162 return 0; 163 164 if (!bch2_accounting_is_mem(acc_k)) 165 return 0; 166 167 if (mode == BCH_ACCOUNTING_normal) { 168 switch (acc_k.type) { 169 case BCH_DISK_ACCOUNTING_persistent_reserved: 170 trans->fs_usage_delta.reserved += acc_k.persistent_reserved.nr_replicas * a.v->d[0]; 171 break; 172 case BCH_DISK_ACCOUNTING_replicas: 173 fs_usage_data_type_to_base(&trans->fs_usage_delta, acc_k.replicas.data_type, a.v->d[0]); 174 break; 175 case BCH_DISK_ACCOUNTING_dev_data_type: 176 rcu_read_lock(); 177 struct bch_dev *ca = bch2_dev_rcu_noerror(c, acc_k.dev_data_type.dev); 178 if (ca) { 179 this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].buckets, a.v->d[0]); 180 this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].sectors, a.v->d[1]); 181 this_cpu_add(ca->usage->d[acc_k.dev_data_type.data_type].fragmented, a.v->d[2]); 182 } 183 rcu_read_unlock(); 184 break; 185 } 186 } 187 188 unsigned idx; 189 190 while ((idx = eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 191 accounting_pos_cmp, &a.k->p)) >= acc->k.nr) { 192 int ret = bch2_accounting_mem_insert(c, a, mode); 193 if (ret) 194 return ret; 195 } 196 197 struct accounting_mem_entry *e = &acc->k.data[idx]; 198 199 EBUG_ON(bch2_accounting_counters(a.k) != e->nr_counters); 200 201 for (unsigned i = 0; i < bch2_accounting_counters(a.k); i++) 202 this_cpu_add(e->v[gc][i], a.v->d[i]); 203 return 0; 204 } 205 206 static inline int bch2_accounting_mem_add(struct btree_trans *trans, struct bkey_s_c_accounting a, bool gc) 207 { 208 percpu_down_read(&trans->c->mark_lock); 209 int ret = bch2_accounting_mem_mod_locked(trans, a, gc ? BCH_ACCOUNTING_gc : BCH_ACCOUNTING_normal); 210 percpu_up_read(&trans->c->mark_lock); 211 return ret; 212 } 213 214 static inline void bch2_accounting_mem_read_counters(struct bch_accounting_mem *acc, 215 unsigned idx, u64 *v, unsigned nr, bool gc) 216 { 217 memset(v, 0, sizeof(*v) * nr); 218 219 if (unlikely(idx >= acc->k.nr)) 220 return; 221 222 struct accounting_mem_entry *e = &acc->k.data[idx]; 223 224 nr = min_t(unsigned, nr, e->nr_counters); 225 226 for (unsigned i = 0; i < nr; i++) 227 v[i] = percpu_u64_get(e->v[gc] + i); 228 } 229 230 static inline void bch2_accounting_mem_read(struct bch_fs *c, struct bpos p, 231 u64 *v, unsigned nr) 232 { 233 percpu_down_read(&c->mark_lock); 234 struct bch_accounting_mem *acc = &c->accounting; 235 unsigned idx = eytzinger0_find(acc->k.data, acc->k.nr, sizeof(acc->k.data[0]), 236 accounting_pos_cmp, &p); 237 238 bch2_accounting_mem_read_counters(acc, idx, v, nr, false); 239 percpu_up_read(&c->mark_lock); 240 } 241 242 static inline struct bversion journal_pos_to_bversion(struct journal_res *res, unsigned offset) 243 { 244 EBUG_ON(!res->ref); 245 246 return (struct bversion) { 247 .hi = res->seq >> 32, 248 .lo = (res->seq << 32) | (res->offset + offset), 249 }; 250 } 251 252 static inline int bch2_accounting_trans_commit_hook(struct btree_trans *trans, 253 struct bkey_i_accounting *a, 254 unsigned commit_flags) 255 { 256 a->k.bversion = journal_pos_to_bversion(&trans->journal_res, 257 (u64 *) a - (u64 *) trans->journal_entries); 258 259 EBUG_ON(bversion_zero(a->k.bversion)); 260 261 return likely(!(commit_flags & BCH_TRANS_COMMIT_skip_accounting_apply)) 262 ? bch2_accounting_mem_mod_locked(trans, accounting_i_to_s_c(a), BCH_ACCOUNTING_normal) 263 : 0; 264 } 265 266 static inline void bch2_accounting_trans_commit_revert(struct btree_trans *trans, 267 struct bkey_i_accounting *a_i, 268 unsigned commit_flags) 269 { 270 if (likely(!(commit_flags & BCH_TRANS_COMMIT_skip_accounting_apply))) { 271 struct bkey_s_accounting a = accounting_i_to_s(a_i); 272 273 bch2_accounting_neg(a); 274 bch2_accounting_mem_mod_locked(trans, a.c, BCH_ACCOUNTING_normal); 275 bch2_accounting_neg(a); 276 } 277 } 278 279 int bch2_fs_replicas_usage_read(struct bch_fs *, darray_char *); 280 int bch2_fs_accounting_read(struct bch_fs *, darray_char *, unsigned); 281 282 int bch2_gc_accounting_start(struct bch_fs *); 283 int bch2_gc_accounting_done(struct bch_fs *); 284 285 int bch2_accounting_read(struct bch_fs *); 286 287 int bch2_dev_usage_remove(struct bch_fs *, unsigned); 288 int bch2_dev_usage_init(struct bch_dev *, bool); 289 290 void bch2_verify_accounting_clean(struct bch_fs *c); 291 292 void bch2_accounting_gc_free(struct bch_fs *); 293 void bch2_fs_accounting_exit(struct bch_fs *); 294 295 #endif /* _BCACHEFS_DISK_ACCOUNTING_H */ 296