Lines Matching +full:mi +full:- +full:v
1 // SPDX-License-Identifier: GPL-2.0
58 * - They track buckets that have been partially allocated, allowing for
59 * sub-bucket sized allocations - they're used by the sector allocator below
61 * - They provide a reference to the buckets they own that mark and sweep GC
66 * with a reference to an open bucket - the caller is required to put that
74 memset(ca->alloc_cursor, 0, sizeof(ca->alloc_cursor)); in bch2_reset_alloc_cursors()
80 open_bucket_idx_t idx = ob - c->open_buckets; in bch2_open_bucket_hash_add()
81 open_bucket_idx_t *slot = open_bucket_hashslot(c, ob->dev, ob->bucket); in bch2_open_bucket_hash_add()
83 ob->hash = *slot; in bch2_open_bucket_hash_add()
89 open_bucket_idx_t idx = ob - c->open_buckets; in bch2_open_bucket_hash_remove()
90 open_bucket_idx_t *slot = open_bucket_hashslot(c, ob->dev, ob->bucket); in bch2_open_bucket_hash_remove()
94 slot = &c->open_buckets[*slot].hash; in bch2_open_bucket_hash_remove()
97 *slot = ob->hash; in bch2_open_bucket_hash_remove()
98 ob->hash = 0; in bch2_open_bucket_hash_remove()
105 if (ob->ec) { in __bch2_open_bucket_put()
106 ec_stripe_new_put(c, ob->ec, STRIPE_REF_io); in __bch2_open_bucket_put()
110 spin_lock(&ob->lock); in __bch2_open_bucket_put()
111 ob->valid = false; in __bch2_open_bucket_put()
112 ob->data_type = 0; in __bch2_open_bucket_put()
113 spin_unlock(&ob->lock); in __bch2_open_bucket_put()
115 spin_lock(&c->freelist_lock); in __bch2_open_bucket_put()
118 ob->freelist = c->open_buckets_freelist; in __bch2_open_bucket_put()
119 c->open_buckets_freelist = ob - c->open_buckets; in __bch2_open_bucket_put()
121 c->open_buckets_nr_free++; in __bch2_open_bucket_put()
122 ca->nr_open_buckets--; in __bch2_open_bucket_put()
123 spin_unlock(&c->freelist_lock); in __bch2_open_bucket_put()
125 closure_wake_up(&c->open_buckets_wait); in __bch2_open_bucket_put()
136 if (ob->dev == dev && ob->ec) in bch2_open_bucket_write_error()
144 BUG_ON(!c->open_buckets_freelist || !c->open_buckets_nr_free); in bch2_open_bucket_alloc()
146 ob = c->open_buckets + c->open_buckets_freelist; in bch2_open_bucket_alloc()
147 c->open_buckets_freelist = ob->freelist; in bch2_open_bucket_alloc()
148 atomic_set(&ob->pin, 1); in bch2_open_bucket_alloc()
149 ob->data_type = 0; in bch2_open_bucket_alloc()
151 c->open_buckets_nr_free--; in bch2_open_bucket_alloc()
157 if (c->curr_recovery_pass > BCH_RECOVERY_PASS_trans_mark_dev_sbs) in is_superblock_bucket()
165 BUG_ON(c->open_buckets_partial_nr >= in open_bucket_free_unused()
166 ARRAY_SIZE(c->open_buckets_partial)); in open_bucket_free_unused()
168 spin_lock(&c->freelist_lock); in open_bucket_free_unused()
170 bch2_dev_rcu(c, ob->dev)->nr_partial_buckets++; in open_bucket_free_unused()
173 ob->on_partial_list = true; in open_bucket_free_unused()
174 c->open_buckets_partial[c->open_buckets_partial_nr++] = in open_bucket_free_unused()
175 ob - c->open_buckets; in open_bucket_free_unused()
176 spin_unlock(&c->freelist_lock); in open_bucket_free_unused()
178 closure_wake_up(&c->open_buckets_wait); in open_bucket_free_unused()
179 closure_wake_up(&c->freelist_wait); in open_bucket_free_unused()
187 s->skipped_open++; in may_alloc_bucket()
192 bch2_bucket_journal_seq_ready(&c->buckets_waiting_for_journal, in may_alloc_bucket()
194 if (journal_seq_ready > c->journal.flushed_seq_ondisk) { in may_alloc_bucket()
195 if (journal_seq_ready > c->journal.flushing_seq) in may_alloc_bucket()
196 s->need_journal_commit++; in may_alloc_bucket()
197 s->skipped_need_journal_commit++; in may_alloc_bucket()
201 if (bch2_bucket_nocow_is_locked(&c->nocow_locks, bucket)) { in may_alloc_bucket()
202 s->skipped_nocow++; in may_alloc_bucket()
218 if (unlikely(ca->buckets_nouse && test_bit(bucket, ca->buckets_nouse))) { in __try_alloc_bucket()
219 s->skipped_nouse++; in __try_alloc_bucket()
223 spin_lock(&c->freelist_lock); in __try_alloc_bucket()
225 if (unlikely(c->open_buckets_nr_free <= bch2_open_buckets_reserved(watermark))) { in __try_alloc_bucket()
227 closure_wait(&c->open_buckets_wait, cl); in __try_alloc_bucket()
229 track_event_change(&c->times[BCH_TIME_blocked_allocate_open_bucket], true); in __try_alloc_bucket()
230 spin_unlock(&c->freelist_lock); in __try_alloc_bucket()
231 return ERR_PTR(-BCH_ERR_open_buckets_empty); in __try_alloc_bucket()
235 if (bch2_bucket_is_open(c, ca->dev_idx, bucket)) { in __try_alloc_bucket()
236 spin_unlock(&c->freelist_lock); in __try_alloc_bucket()
237 s->skipped_open++; in __try_alloc_bucket()
243 spin_lock(&ob->lock); in __try_alloc_bucket()
244 ob->valid = true; in __try_alloc_bucket()
245 ob->sectors_free = ca->mi.bucket_size; in __try_alloc_bucket()
246 ob->dev = ca->dev_idx; in __try_alloc_bucket()
247 ob->gen = gen; in __try_alloc_bucket()
248 ob->bucket = bucket; in __try_alloc_bucket()
249 spin_unlock(&ob->lock); in __try_alloc_bucket()
251 ca->nr_open_buckets++; in __try_alloc_bucket()
254 track_event_change(&c->times[BCH_TIME_blocked_allocate_open_bucket], false); in __try_alloc_bucket()
255 track_event_change(&c->times[BCH_TIME_blocked_allocate], false); in __try_alloc_bucket()
257 spin_unlock(&c->freelist_lock); in __try_alloc_bucket()
267 struct bch_fs *c = trans->c; in try_alloc_bucket()
268 u64 b = freespace_iter->pos.offset & ~(~0ULL << 56); in try_alloc_bucket()
270 if (!may_alloc_bucket(c, POS(ca->dev_idx, b), s)) in try_alloc_bucket()
293 struct bch_fs *c = trans->c; in bch2_bucket_alloc_early()
297 u64 first_bucket = ca->mi.first_bucket; in bch2_bucket_alloc_early()
298 u64 *dev_alloc_cursor = &ca->alloc_cursor[s->btree_bitmap]; in bch2_bucket_alloc_early()
313 for_each_btree_key_norestart(trans, iter, BTREE_ID_alloc, POS(ca->dev_idx, alloc_cursor), in bch2_bucket_alloc_early()
315 u64 bucket = k.k->p.offset; in bch2_bucket_alloc_early()
317 if (bkey_ge(k.k->p, POS(ca->dev_idx, ca->mi.nbuckets))) in bch2_bucket_alloc_early()
320 if (s->btree_bitmap != BTREE_BITMAP_ANY && in bch2_bucket_alloc_early()
321 s->btree_bitmap != bch2_dev_btree_bitmap_marked_sectors(ca, in bch2_bucket_alloc_early()
322 bucket_to_sector(ca, bucket), ca->mi.bucket_size)) { in bch2_bucket_alloc_early()
323 if (s->btree_bitmap == BTREE_BITMAP_YES && in bch2_bucket_alloc_early()
324 bucket_to_sector(ca, bucket) > 64ULL << ca->mi.btree_bitmap_shift) in bch2_bucket_alloc_early()
329 1ULL << ca->mi.btree_bitmap_shift)); in bch2_bucket_alloc_early()
330 bch2_btree_iter_set_pos(trans, &iter, POS(ca->dev_idx, bucket)); in bch2_bucket_alloc_early()
331 s->buckets_seen++; in bch2_bucket_alloc_early()
332 s->skipped_mi_btree_bitmap++; in bch2_bucket_alloc_early()
338 if (a->data_type != BCH_DATA_free) in bch2_bucket_alloc_early()
342 ck = bch2_bkey_get_iter(trans, &citer, BTREE_ID_alloc, k.k->p, BTREE_ITER_cached); in bch2_bucket_alloc_early()
348 if (a->data_type != BCH_DATA_free) in bch2_bucket_alloc_early()
351 s->buckets_seen++; in bch2_bucket_alloc_early()
353 ob = may_alloc_bucket(c, k.k->p, s) in bch2_bucket_alloc_early()
354 ? __try_alloc_bucket(c, ca, k.k->p.offset, a->gen, in bch2_bucket_alloc_early()
389 u64 *dev_alloc_cursor = &ca->alloc_cursor[s->btree_bitmap]; in bch2_bucket_alloc_freelist()
390 u64 alloc_start = max_t(u64, ca->mi.first_bucket, READ_ONCE(*dev_alloc_cursor)); in bch2_bucket_alloc_freelist()
395 POS(ca->dev_idx, alloc_cursor), in bch2_bucket_alloc_freelist()
396 POS(ca->dev_idx, U64_MAX), in bch2_bucket_alloc_freelist()
399 * peek normally dosen't trim extents - they can span iter.pos, in bch2_bucket_alloc_freelist()
402 iter.k.size = iter.k.p.offset - iter.pos.offset; in bch2_bucket_alloc_freelist()
405 s->buckets_seen++; in bch2_bucket_alloc_freelist()
408 if (s->btree_bitmap != BTREE_BITMAP_ANY && in bch2_bucket_alloc_freelist()
409 s->btree_bitmap != bch2_dev_btree_bitmap_marked_sectors(ca, in bch2_bucket_alloc_freelist()
410 bucket_to_sector(ca, bucket), ca->mi.bucket_size)) { in bch2_bucket_alloc_freelist()
411 if (s->btree_bitmap == BTREE_BITMAP_YES && in bch2_bucket_alloc_freelist()
412 bucket_to_sector(ca, bucket) > 64ULL << ca->mi.btree_bitmap_shift) in bch2_bucket_alloc_freelist()
417 1ULL << ca->mi.btree_bitmap_shift)); in bch2_bucket_alloc_freelist()
420 bch2_btree_iter_set_pos(trans, &iter, POS(ca->dev_idx, alloc_cursor)); in bch2_bucket_alloc_freelist()
421 s->skipped_mi_btree_bitmap++; in bch2_bucket_alloc_freelist()
433 iter.k.size--; in bch2_bucket_alloc_freelist()
448 if (!ob && alloc_start > ca->mi.first_bucket) { in bch2_bucket_alloc_freelist()
449 alloc_cursor = alloc_start = ca->mi.first_bucket; in bch2_bucket_alloc_freelist()
468 prt_printf(&buf, "dev\t%s (%u)\n", ca->name, ca->dev_idx); in trace_bucket_alloc2()
472 prt_printf(&buf, "free\t%llu\n", usage->buckets[BCH_DATA_free]); in trace_bucket_alloc2()
476 c->copygc_wait - atomic64_read(&c->io_clock[WRITE].now)); in trace_bucket_alloc2()
477 prt_printf(&buf, "seen\t%llu\n", s->buckets_seen); in trace_bucket_alloc2()
478 prt_printf(&buf, "open\t%llu\n", s->skipped_open); in trace_bucket_alloc2()
479 prt_printf(&buf, "need journal commit\t%llu\n", s->skipped_need_journal_commit); in trace_bucket_alloc2()
480 prt_printf(&buf, "nocow\t%llu\n", s->skipped_nocow); in trace_bucket_alloc2()
481 prt_printf(&buf, "nouse\t%llu\n", s->skipped_nouse); in trace_bucket_alloc2()
482 prt_printf(&buf, "mi_btree_bitmap\t%llu\n", s->skipped_mi_btree_bitmap); in trace_bucket_alloc2()
485 prt_printf(&buf, "allocated\t%llu\n", ob->bucket); in trace_bucket_alloc2()
496 * bch2_bucket_alloc_trans - allocate a single bucket from a specific device
515 struct bch_fs *c = trans->c; in bch2_bucket_alloc_trans()
517 bool freespace = READ_ONCE(ca->mi.freespace_initialized); in bch2_bucket_alloc_trans()
527 if (usage->buckets[BCH_DATA_need_discard] > avail) in bch2_bucket_alloc_trans()
530 if (usage->buckets[BCH_DATA_need_gc_gens] > avail) in bch2_bucket_alloc_trans()
538 c->curr_recovery_pass <= BCH_RECOVERY_PASS_check_allocations) in bch2_bucket_alloc_trans()
542 closure_wait(&c->freelist_wait, cl); in bch2_bucket_alloc_trans()
547 track_event_change(&c->times[BCH_TIME_blocked_allocate], true); in bch2_bucket_alloc_trans()
549 ob = ERR_PTR(-BCH_ERR_freelist_empty); in bch2_bucket_alloc_trans()
554 closure_wake_up(&c->freelist_wait); in bch2_bucket_alloc_trans()
561 bch2_journal_flush_async(&c->journal, NULL); in bch2_bucket_alloc_trans()
568 if (!ob && freespace && c->curr_recovery_pass <= BCH_RECOVERY_PASS_check_alloc_info) { in bch2_bucket_alloc_trans()
574 ob = ERR_PTR(-BCH_ERR_no_buckets_found); in bch2_bucket_alloc_trans()
577 ob->data_type = data_type; in bch2_bucket_alloc_trans()
609 return cmp_int(stripe->next_alloc[l], stripe->next_alloc[r]); in __dev_stripe_cmp()
621 for_each_set_bit(i, devs->d, BCH_SB_MEMBERS_MAX) in bch2_dev_alloc_list()
636 * to 0 then we lose information - clock hands can be in a wide range if in bch2_stripe_state_rescale()
644 for (u64 *v = stripe->next_alloc; in bch2_stripe_state_rescale() local
645 v < stripe->next_alloc + ARRAY_SIZE(stripe->next_alloc); v++) { in bch2_stripe_state_rescale()
646 if (*v) in bch2_stripe_state_rescale()
647 scale_max = min(scale_max, *v); in bch2_stripe_state_rescale()
648 if (*v > stripe_clock_hand_max) in bch2_stripe_state_rescale()
649 scale_min = max(scale_min, *v - stripe_clock_hand_max); in bch2_stripe_state_rescale()
654 for (u64 *v = stripe->next_alloc; in bch2_stripe_state_rescale() local
655 v < stripe->next_alloc + ARRAY_SIZE(stripe->next_alloc); v++) in bch2_stripe_state_rescale()
656 *v = *v < scale ? 0 : *v - scale; in bch2_stripe_state_rescale()
672 u64 *v = stripe->next_alloc + ca->dev_idx; in bch2_dev_stripe_increment_inlined() local
679 u64 sum = *v + free_space_inv; in bch2_dev_stripe_increment_inlined()
680 *v = sum >= *v ? sum : U64_MAX; in bch2_dev_stripe_increment_inlined()
682 if (unlikely(*v > stripe_clock_hand_rescale)) in bch2_dev_stripe_increment_inlined()
703 unsigned durability = ob_dev(c, ob)->mi.durability; in add_new_bucket()
707 __clear_bit(ob->dev, devs_may_alloc->d); in add_new_bucket()
715 if (ob->ec) in add_new_bucket()
732 struct bch_fs *c = trans->c; in bch2_bucket_alloc_set_trans()
733 int ret = -BCH_ERR_insufficient_devices; in bch2_bucket_alloc_set_trans()
743 if (!ca->mi.durability && *have_cache) { in bch2_bucket_alloc_set_trans()
793 struct bch_fs *c = trans->c; in bucket_alloc_from_stripe()
803 bch2_ec_stripe_head_get(trans, target, 0, nr_replicas - 1, watermark, cl); in bucket_alloc_from_stripe()
809 struct dev_alloc_list devs_sorted = bch2_dev_alloc_list(c, &wp->stripe, devs_may_alloc); in bucket_alloc_from_stripe()
811 for (unsigned ec_idx = 0; ec_idx < h->s->nr_data; ec_idx++) { in bucket_alloc_from_stripe()
812 if (!h->s->blocks[ec_idx]) in bucket_alloc_from_stripe()
815 struct open_bucket *ob = c->open_buckets + h->s->blocks[ec_idx]; in bucket_alloc_from_stripe()
816 if (ob->dev == *i && !test_and_set_bit(ec_idx, h->s->blocks_allocated)) { in bucket_alloc_from_stripe()
817 ob->ec_idx = ec_idx; in bucket_alloc_from_stripe()
818 ob->ec = h->s; in bucket_alloc_from_stripe()
819 ec_stripe_new_get(h->s, STRIPE_REF_io); in bucket_alloc_from_stripe()
842 if (!test_bit(ob->dev, devs_may_alloc->d)) in want_bucket()
845 if (ob->data_type != wp->data_type) in want_bucket()
848 if (!ca->mi.durability && in want_bucket()
849 (wp->data_type == BCH_DATA_btree || ec || *have_cache)) in want_bucket()
852 if (ec != (ob->ec != NULL)) in want_bucket()
872 open_bucket_for_each(c, &wp->ptrs, ob, i) { in bucket_alloc_set_writepoint()
881 wp->ptrs = ptrs_skip; in bucket_alloc_set_writepoint()
897 if (!c->open_buckets_partial_nr) in bucket_alloc_set_partial()
900 spin_lock(&c->freelist_lock); in bucket_alloc_set_partial()
902 if (!c->open_buckets_partial_nr) in bucket_alloc_set_partial()
905 for (i = c->open_buckets_partial_nr - 1; i >= 0; --i) { in bucket_alloc_set_partial()
906 struct open_bucket *ob = c->open_buckets + c->open_buckets_partial[i]; in bucket_alloc_set_partial()
914 avail = dev_buckets_free(ca, usage, watermark) + ca->nr_partial_buckets; in bucket_alloc_set_partial()
918 array_remove_item(c->open_buckets_partial, in bucket_alloc_set_partial()
919 c->open_buckets_partial_nr, in bucket_alloc_set_partial()
921 ob->on_partial_list = false; in bucket_alloc_set_partial()
924 bch2_dev_rcu(c, ob->dev)->nr_partial_buckets--; in bucket_alloc_set_partial()
935 spin_unlock(&c->freelist_lock); in bucket_alloc_set_partial()
952 struct bch_fs *c = trans->c; in __open_bucket_add_buckets()
959 devs = target_rw_devs(c, wp->data_type, target); in __open_bucket_add_buckets()
966 __clear_bit(ob->dev, devs.d); in __open_bucket_add_buckets()
992 ret = bch2_bucket_alloc_set_trans(trans, ptrs, &wp->stripe, &devs, in __open_bucket_add_buckets()
994 flags, wp->data_type, watermark, cl); in __open_bucket_add_buckets()
1022 if (erasure_code && !ec_open_bucket(trans->c, ptrs)) { in open_bucket_add_buckets()
1044 * should_drop_bucket - check if this is open_bucket should go away
1054 * coding, or the entire filesystem - check if this open_bucket matches:
1060 return ob->ec != NULL; in should_drop_bucket()
1062 bool drop = ob->dev == ca->dev_idx; in should_drop_bucket()
1066 if (!drop && ob->ec) { in should_drop_bucket()
1069 mutex_lock(&ob->ec->lock); in should_drop_bucket()
1070 nr_blocks = bkey_i_to_stripe(&ob->ec->new_stripe.key)->v.nr_blocks; in should_drop_bucket()
1073 if (!ob->ec->blocks[i]) in should_drop_bucket()
1076 ob2 = c->open_buckets + ob->ec->blocks[i]; in should_drop_bucket()
1077 drop |= ob2->dev == ca->dev_idx; in should_drop_bucket()
1079 mutex_unlock(&ob->ec->lock); in should_drop_bucket()
1095 mutex_lock(&wp->lock); in bch2_writepoint_stop()
1096 open_bucket_for_each(c, &wp->ptrs, ob, i) in bch2_writepoint_stop()
1101 wp->ptrs = ptrs; in bch2_writepoint_stop()
1102 mutex_unlock(&wp->lock); in bch2_writepoint_stop()
1111 for (i = 0; i < ARRAY_SIZE(c->write_points); i++) in bch2_open_buckets_stop()
1112 bch2_writepoint_stop(c, ca, ec, &c->write_points[i]); in bch2_open_buckets_stop()
1114 bch2_writepoint_stop(c, ca, ec, &c->copygc_write_point); in bch2_open_buckets_stop()
1115 bch2_writepoint_stop(c, ca, ec, &c->rebalance_write_point); in bch2_open_buckets_stop()
1116 bch2_writepoint_stop(c, ca, ec, &c->btree_write_point); in bch2_open_buckets_stop()
1118 mutex_lock(&c->btree_reserve_cache_lock); in bch2_open_buckets_stop()
1119 while (c->btree_reserve_cache_nr) { in bch2_open_buckets_stop()
1121 &c->btree_reserve_cache[--c->btree_reserve_cache_nr]; in bch2_open_buckets_stop()
1123 bch2_open_buckets_put(c, &a->ob); in bch2_open_buckets_stop()
1125 mutex_unlock(&c->btree_reserve_cache_lock); in bch2_open_buckets_stop()
1127 spin_lock(&c->freelist_lock); in bch2_open_buckets_stop()
1129 while (i < c->open_buckets_partial_nr) { in bch2_open_buckets_stop()
1131 c->open_buckets + c->open_buckets_partial[i]; in bch2_open_buckets_stop()
1134 --c->open_buckets_partial_nr; in bch2_open_buckets_stop()
1135 swap(c->open_buckets_partial[i], in bch2_open_buckets_stop()
1136 c->open_buckets_partial[c->open_buckets_partial_nr]); in bch2_open_buckets_stop()
1138 ob->on_partial_list = false; in bch2_open_buckets_stop()
1141 bch2_dev_rcu(c, ob->dev)->nr_partial_buckets--; in bch2_open_buckets_stop()
1144 spin_unlock(&c->freelist_lock); in bch2_open_buckets_stop()
1146 spin_lock(&c->freelist_lock); in bch2_open_buckets_stop()
1151 spin_unlock(&c->freelist_lock); in bch2_open_buckets_stop()
1160 hash_long(write_point, ilog2(ARRAY_SIZE(c->write_points_hash))); in writepoint_hash()
1162 return &c->write_points_hash[hash]; in writepoint_hash()
1172 if (wp->write_point == write_point) in __writepoint_find()
1182 u64 stranded = c->write_points_nr * c->bucket_size_max; in too_many_writepoints()
1192 if (c->write_points_nr == ARRAY_SIZE(c->write_points) || in try_increase_writepoints()
1196 wp = c->write_points + c->write_points_nr++; in try_increase_writepoints()
1197 hlist_add_head_rcu(&wp->node, writepoint_hash(c, wp->write_point)); in try_increase_writepoints()
1203 struct bch_fs *c = trans->c; in try_decrease_writepoints()
1208 mutex_lock(&c->write_points_hash_lock); in try_decrease_writepoints()
1209 if (c->write_points_nr < old_nr) { in try_decrease_writepoints()
1210 mutex_unlock(&c->write_points_hash_lock); in try_decrease_writepoints()
1214 if (c->write_points_nr == 1 || in try_decrease_writepoints()
1216 mutex_unlock(&c->write_points_hash_lock); in try_decrease_writepoints()
1220 wp = c->write_points + --c->write_points_nr; in try_decrease_writepoints()
1222 hlist_del_rcu(&wp->node); in try_decrease_writepoints()
1223 mutex_unlock(&c->write_points_hash_lock); in try_decrease_writepoints()
1225 bch2_trans_mutex_lock_norelock(trans, &wp->lock); in try_decrease_writepoints()
1226 open_bucket_for_each(c, &wp->ptrs, ob, i) in try_decrease_writepoints()
1228 wp->ptrs.nr = 0; in try_decrease_writepoints()
1229 mutex_unlock(&wp->lock); in try_decrease_writepoints()
1236 struct bch_fs *c = trans->c; in writepoint_find()
1242 bch2_trans_mutex_lock_norelock(trans, &wp->lock); in writepoint_find()
1251 bch2_trans_mutex_lock_norelock(trans, &wp->lock); in writepoint_find()
1252 if (wp->write_point == write_point) in writepoint_find()
1254 mutex_unlock(&wp->lock); in writepoint_find()
1259 for (wp = c->write_points; in writepoint_find()
1260 wp < c->write_points + c->write_points_nr; wp++) in writepoint_find()
1261 if (!oldest || time_before64(wp->last_used, oldest->last_used)) in writepoint_find()
1264 bch2_trans_mutex_lock_norelock(trans, &oldest->lock); in writepoint_find()
1265 bch2_trans_mutex_lock_norelock(trans, &c->write_points_hash_lock); in writepoint_find()
1266 if (oldest >= c->write_points + c->write_points_nr || in writepoint_find()
1268 mutex_unlock(&c->write_points_hash_lock); in writepoint_find()
1269 mutex_unlock(&oldest->lock); in writepoint_find()
1275 mutex_unlock(&c->write_points_hash_lock); in writepoint_find()
1276 mutex_unlock(&oldest->lock); in writepoint_find()
1281 hlist_del_rcu(&wp->node); in writepoint_find()
1282 wp->write_point = write_point; in writepoint_find()
1283 hlist_add_head_rcu(&wp->node, head); in writepoint_find()
1284 mutex_unlock(&c->write_points_hash_lock); in writepoint_find()
1286 wp->last_used = local_clock(); in writepoint_find()
1301 unsigned d = ob_dev(c, ob)->mi.durability; in deallocate_extra_replicas()
1304 extra_replicas -= d; in deallocate_extra_replicas()
1329 struct bch_fs *c = trans->c; in bch2_alloc_sectors_start_trans()
1345 write_points_nr = c->write_points_nr; in bch2_alloc_sectors_start_trans()
1348 *wp_ret = wp = writepoint_find(trans, write_point.v); in bch2_alloc_sectors_start_trans()
1355 if (wp->data_type != BCH_DATA_user) in bch2_alloc_sectors_start_trans()
1407 if (ret == -BCH_ERR_insufficient_devices && in bch2_alloc_sectors_start_trans()
1415 deallocate_extra_replicas(c, &ptrs, &wp->ptrs, nr_effective - nr_replicas); in bch2_alloc_sectors_start_trans()
1418 open_bucket_for_each(c, &wp->ptrs, ob, i) in bch2_alloc_sectors_start_trans()
1421 wp->ptrs = ptrs; in bch2_alloc_sectors_start_trans()
1423 wp->sectors_free = UINT_MAX; in bch2_alloc_sectors_start_trans()
1425 open_bucket_for_each(c, &wp->ptrs, ob, i) { in bch2_alloc_sectors_start_trans()
1427 * Ensure proper write alignment - either due to misaligned in bch2_alloc_sectors_start_trans()
1428 * bucket sizes (from buggy bcachefs-tools), or writes that mix in bch2_alloc_sectors_start_trans()
1432 u64 offset = bucket_to_sector(ca, ob->bucket) + in bch2_alloc_sectors_start_trans()
1433 ca->mi.bucket_size - in bch2_alloc_sectors_start_trans()
1434 ob->sectors_free; in bch2_alloc_sectors_start_trans()
1435 unsigned align = round_up(offset, block_sectors(c)) - offset; in bch2_alloc_sectors_start_trans()
1437 ob->sectors_free = max_t(int, 0, ob->sectors_free - align); in bch2_alloc_sectors_start_trans()
1439 wp->sectors_free = min(wp->sectors_free, ob->sectors_free); in bch2_alloc_sectors_start_trans()
1442 wp->sectors_free = rounddown(wp->sectors_free, block_sectors(c)); in bch2_alloc_sectors_start_trans()
1445 if (unlikely(!wp->sectors_free)) { in bch2_alloc_sectors_start_trans()
1450 BUG_ON(!wp->sectors_free || wp->sectors_free == UINT_MAX); in bch2_alloc_sectors_start_trans()
1454 open_bucket_for_each(c, &wp->ptrs, ob, i) in bch2_alloc_sectors_start_trans()
1455 if (ptrs.nr < ARRAY_SIZE(ptrs.v)) in bch2_alloc_sectors_start_trans()
1459 wp->ptrs = ptrs; in bch2_alloc_sectors_start_trans()
1461 mutex_unlock(&wp->lock); in bch2_alloc_sectors_start_trans()
1468 ret = -BCH_ERR_bucket_alloc_blocked; in bch2_alloc_sectors_start_trans()
1472 ret = -BCH_ERR_bucket_alloc_blocked; in bch2_alloc_sectors_start_trans()
1483 .gen = ob->gen, in bch2_ob_ptr()
1484 .dev = ob->dev, in bch2_ob_ptr()
1485 .offset = bucket_to_sector(ca, ob->bucket) + in bch2_ob_ptr()
1486 ca->mi.bucket_size - in bch2_ob_ptr()
1487 ob->sectors_free, in bch2_ob_ptr()
1510 mutex_init(&wp->lock); in writepoint_init()
1511 wp->data_type = type; in writepoint_init()
1513 INIT_WORK(&wp->index_update_work, bch2_write_point_do_index_updates); in writepoint_init()
1514 INIT_LIST_HEAD(&wp->writes); in writepoint_init()
1515 spin_lock_init(&wp->writes_lock); in writepoint_init()
1523 mutex_init(&c->write_points_hash_lock); in bch2_fs_allocator_foreground_init()
1524 c->write_points_nr = ARRAY_SIZE(c->write_points); in bch2_fs_allocator_foreground_init()
1527 spin_lock_init(&c->open_buckets[0].lock); in bch2_fs_allocator_foreground_init()
1529 for (ob = c->open_buckets + 1; in bch2_fs_allocator_foreground_init()
1530 ob < c->open_buckets + ARRAY_SIZE(c->open_buckets); ob++) { in bch2_fs_allocator_foreground_init()
1531 spin_lock_init(&ob->lock); in bch2_fs_allocator_foreground_init()
1532 c->open_buckets_nr_free++; in bch2_fs_allocator_foreground_init()
1534 ob->freelist = c->open_buckets_freelist; in bch2_fs_allocator_foreground_init()
1535 c->open_buckets_freelist = ob - c->open_buckets; in bch2_fs_allocator_foreground_init()
1538 writepoint_init(&c->btree_write_point, BCH_DATA_btree); in bch2_fs_allocator_foreground_init()
1539 writepoint_init(&c->rebalance_write_point, BCH_DATA_user); in bch2_fs_allocator_foreground_init()
1540 writepoint_init(&c->copygc_write_point, BCH_DATA_user); in bch2_fs_allocator_foreground_init()
1542 for (wp = c->write_points; in bch2_fs_allocator_foreground_init()
1543 wp < c->write_points + c->write_points_nr; wp++) { in bch2_fs_allocator_foreground_init()
1546 wp->last_used = local_clock(); in bch2_fs_allocator_foreground_init()
1547 wp->write_point = (unsigned long) wp; in bch2_fs_allocator_foreground_init()
1548 hlist_add_head_rcu(&wp->node, in bch2_fs_allocator_foreground_init()
1549 writepoint_hash(c, wp->write_point)); in bch2_fs_allocator_foreground_init()
1556 unsigned data_type = ob->data_type; in bch2_open_bucket_to_text()
1560 ob - c->open_buckets, in bch2_open_bucket_to_text()
1561 atomic_read(&ob->pin)); in bch2_open_bucket_to_text()
1564 ob->dev, ob->bucket, ob->gen, in bch2_open_bucket_to_text()
1565 ca->mi.bucket_size - ob->sectors_free, ca->mi.bucket_size); in bch2_open_bucket_to_text()
1566 if (ob->ec) in bch2_open_bucket_to_text()
1567 prt_printf(out, " ec idx %llu", ob->ec->idx); in bch2_open_bucket_to_text()
1568 if (ob->on_partial_list) in bch2_open_bucket_to_text()
1578 out->atomic++; in bch2_open_buckets_to_text()
1580 for (ob = c->open_buckets; in bch2_open_buckets_to_text()
1581 ob < c->open_buckets + ARRAY_SIZE(c->open_buckets); in bch2_open_buckets_to_text()
1583 spin_lock(&ob->lock); in bch2_open_buckets_to_text()
1584 if (ob->valid && (!ca || ob->dev == ca->dev_idx)) in bch2_open_buckets_to_text()
1586 spin_unlock(&ob->lock); in bch2_open_buckets_to_text()
1589 --out->atomic; in bch2_open_buckets_to_text()
1596 out->atomic++; in bch2_open_buckets_partial_to_text()
1597 spin_lock(&c->freelist_lock); in bch2_open_buckets_partial_to_text()
1599 for (i = 0; i < c->open_buckets_partial_nr; i++) in bch2_open_buckets_partial_to_text()
1601 c->open_buckets + c->open_buckets_partial[i]); in bch2_open_buckets_partial_to_text()
1603 spin_unlock(&c->freelist_lock); in bch2_open_buckets_partial_to_text()
1604 --out->atomic; in bch2_open_buckets_partial_to_text()
1620 prt_printf(out, "%lu: ", wp->write_point); in bch2_write_point_to_text()
1621 prt_human_readable_u64(out, wp->sectors_allocated << 9); in bch2_write_point_to_text()
1624 bch2_pr_time_units(out, sched_clock() - wp->last_used); in bch2_write_point_to_text()
1628 bch2_pr_time_units(out, wp->time[i]); in bch2_write_point_to_text()
1634 open_bucket_for_each(c, &wp->ptrs, ob, i) in bch2_write_point_to_text()
1644 for (wp = c->write_points; in bch2_write_points_to_text()
1645 wp < c->write_points + ARRAY_SIZE(c->write_points); in bch2_write_points_to_text()
1650 bch2_write_point_to_text(out, c, &c->copygc_write_point); in bch2_write_points_to_text()
1653 bch2_write_point_to_text(out, c, &c->rebalance_write_point); in bch2_write_points_to_text()
1656 bch2_write_point_to_text(out, c, &c->btree_write_point); in bch2_write_points_to_text()
1665 for (unsigned i = 0; i < ARRAY_SIZE(c->open_buckets); i++) in bch2_fs_alloc_debug_to_text()
1666 nr[c->open_buckets[i].data_type]++; in bch2_fs_alloc_debug_to_text()
1671 prt_printf(out, "capacity\t%llu\n", c->capacity); in bch2_fs_alloc_debug_to_text()
1672 prt_printf(out, "reserved\t%llu\n", c->reserved); in bch2_fs_alloc_debug_to_text()
1673 prt_printf(out, "hidden\t%llu\n", percpu_u64_get(&c->usage->hidden)); in bch2_fs_alloc_debug_to_text()
1674 prt_printf(out, "btree\t%llu\n", percpu_u64_get(&c->usage->btree)); in bch2_fs_alloc_debug_to_text()
1675 prt_printf(out, "data\t%llu\n", percpu_u64_get(&c->usage->data)); in bch2_fs_alloc_debug_to_text()
1676 prt_printf(out, "cached\t%llu\n", percpu_u64_get(&c->usage->cached)); in bch2_fs_alloc_debug_to_text()
1677 prt_printf(out, "reserved\t%llu\n", percpu_u64_get(&c->usage->reserved)); in bch2_fs_alloc_debug_to_text()
1678 prt_printf(out, "online_reserved\t%llu\n", percpu_u64_get(c->online_reserved)); in bch2_fs_alloc_debug_to_text()
1679 prt_printf(out, "nr_inodes\t%llu\n", percpu_u64_get(&c->usage->nr_inodes)); in bch2_fs_alloc_debug_to_text()
1682 prt_printf(out, "freelist_wait\t%s\n", c->freelist_wait.list.first ? "waiting" : "empty"); in bch2_fs_alloc_debug_to_text()
1683 prt_printf(out, "open buckets allocated\t%i\n", OPEN_BUCKETS_COUNT - c->open_buckets_nr_free); in bch2_fs_alloc_debug_to_text()
1685 prt_printf(out, "open_buckets_wait\t%s\n", c->open_buckets_wait.list.first ? "waiting" : "empty"); in bch2_fs_alloc_debug_to_text()
1688 prt_printf(out, "btree reserve cache\t%u\n", c->btree_reserve_cache_nr); in bch2_fs_alloc_debug_to_text()
1693 struct bch_fs *c = ca->fs; in bch2_dev_alloc_debug_to_text()
1699 for (unsigned i = 0; i < ARRAY_SIZE(c->open_buckets); i++) in bch2_dev_alloc_debug_to_text()
1700 nr[c->open_buckets[i].data_type]++; in bch2_dev_alloc_debug_to_text()
1716 prt_printf(out, "open buckets\t%i\r\n", ca->nr_open_buckets); in bch2_dev_alloc_debug_to_text()
1726 c->opts.allocator_stuck_timeout); in bch2_print_allocator_stuck()
1735 prt_printf(&buf, "Dev %u:\n", ca->dev_idx); in bch2_print_allocator_stuck()
1750 bch2_journal_debug_to_text(&buf, &c->journal); in bch2_print_allocator_stuck()
1759 if (c->allocator_last_stuck && in allocator_wait_timeout()
1760 time_after(c->allocator_last_stuck + HZ * 60 * 2, jiffies)) in allocator_wait_timeout()
1763 return c->opts.allocator_stuck_timeout * HZ; in allocator_wait_timeout()
1771 c->allocator_last_stuck = jiffies; in __bch2_wait_on_allocator()