fs/bcachefs/clock.c

3 #include "clock.h"
22 void bch2_io_timer_add(struct io_clock *clock, struct io_timer *timer)
24 	spin_lock(&clock->timer_lock);
26 	if (time_after_eq64((u64) atomic64_read(&clock->now), timer->expire)) {
27 		spin_unlock(&clock->timer_lock);
32 	for (size_t i = 0; i < clock->timers.nr; i++)
33 		if (clock->timers.data[i] == timer)
36 	BUG_ON(!min_heap_push(&clock->timers, &timer, &callbacks, NULL));
38 	spin_unlock(&clock->timer_lock);
41 void bch2_io_timer_del(struct io_clock *clock, struct io_timer *timer)
43 	spin_lock(&clock->timer_lock);
45 	for (size_t i = 0; i < clock->timers.nr; i++)
46 		if (clock->timers.data[i] == timer) {
47 			min_heap_del(&clock->timers, i, &callbacks, NULL);
51 	spin_unlock(&clock->timer_lock);
69 void bch2_io_clock_schedule_timeout(struct io_clock *clock, u64 until)
78 	bch2_io_timer_add(clock, &wait.io_timer);
80 	bch2_io_timer_del(clock, &wait.io_timer);
83 unsigned long bch2_kthread_io_clock_wait_once(struct io_clock *clock,
94 	bch2_io_timer_add(clock, &wait.io_timer);
103 	bch2_io_timer_del(clock, &wait.io_timer);
107 void bch2_kthread_io_clock_wait(struct io_clock *clock,
114 	       atomic64_read(&clock->now) < io_until)
115 		cpu_timeout = bch2_kthread_io_clock_wait_once(clock, io_until, cpu_timeout);
118 static struct io_timer *get_expired_timer(struct io_clock *clock, u64 now)
122 	if (clock->timers.nr &&
123 	    time_after_eq64(now, clock->timers.data[0]->expire)) {
124 		ret = *min_heap_peek(&clock->timers);
125 		min_heap_pop(&clock->timers, &callbacks, NULL);
131 void __bch2_increment_clock(struct io_clock *clock, u64 sectors)
134 	u64 now = atomic64_add_return(sectors, &clock->now);
136 	spin_lock(&clock->timer_lock);
137 	while ((timer = get_expired_timer(clock, now)))
139 	spin_unlock(&clock->timer_lock);
142 void bch2_io_timers_to_text(struct printbuf *out, struct io_clock *clock)
145 	spin_lock(&clock->timer_lock);
146 	u64 now = atomic64_read(&clock->now);
151 	for (unsigned i = 0; i < clock->timers.nr; i++)
153 		       clock->timers.data[i]->fn,
154 		       clock->timers.data[i]->fn2,
155 		       clock->timers.data[i]->expire);
156 	spin_unlock(&clock->timer_lock);
160 void bch2_io_clock_exit(struct io_clock *clock)
162 	free_heap(&clock->timers);
163 	free_percpu(clock->pcpu_buf);
166 int bch2_io_clock_init(struct io_clock *clock)
168 	atomic64_set(&clock->now, 0);
169 	spin_lock_init(&clock->timer_lock);
171 	clock->max_slop = IO_CLOCK_PCPU_SECTORS * num_possible_cpus();
173 	clock->pcpu_buf = alloc_percpu(*clock->pcpu_buf);
174 	if (!clock->pcpu_buf)
177 	if (!init_heap(&clock->timers, NR_IO_TIMERS, GFP_KERNEL))