95  * limiter and the submitting thread will then put itself to sleep. (note that this mechanism will
123 struct limiter;
124 typedef void (*assigner_fn)(struct limiter *limiter);
127 struct limiter {  struct
128 	/* The data_vio_pool to which this limiter belongs */
165 	/* The main limiter controlling the total data_vios in the pool. */  argument
166 	struct limiter limiter;  member
167 	/* The limiter controlling data_vios for discard */
168 	struct limiter discard_limiter;
232 	if (pool->limiter.busy > 0)  in check_for_drain_complete_locked()
238 	return (bio_list_empty(&pool->limiter.new_waiters) &&  in check_for_drain_complete_locked()
586 static void assign_data_vio(struct limiter *limiter, struct data_vio *data_vio)  in assign_data_vio()  argument
588 	struct bio *bio = bio_list_pop(limiter->permitted_waiters);  in assign_data_vio()
590 	launch_bio(limiter->pool->completion.vdo, data_vio, bio);  in assign_data_vio()
591 	limiter->wake_count++;  in assign_data_vio()
593 	bio = bio_list_peek(limiter->permitted_waiters);  in assign_data_vio()
594 	limiter->arrival = ((bio == NULL) ? U64_MAX : get_arrival_time(bio));  in assign_data_vio()
597 static void assign_discard_permit(struct limiter *limiter)  in assign_discard_permit()  argument
599 	struct bio *bio = bio_list_pop(&limiter->waiters);  in assign_discard_permit()
601 	if (limiter->arrival == U64_MAX)  in assign_discard_permit()
602 		limiter->arrival = get_arrival_time(bio);  in assign_discard_permit()
604 	bio_list_add(limiter->permitted_waiters, bio);  in assign_discard_permit()
607 static void get_waiters(struct limiter *limiter)  in get_waiters()  argument
609 	bio_list_merge_init(&limiter->waiters, &limiter->new_waiters);  in get_waiters()
621 static void assign_data_vio_to_waiter(struct limiter *limiter)  in assign_data_vio_to_waiter()  argument
623 	assign_data_vio(limiter, get_available_data_vio(limiter->pool));  in assign_data_vio_to_waiter()
626 static void update_limiter(struct limiter *limiter)  in update_limiter()  argument
628 	struct bio_list *waiters = &limiter->waiters;  in update_limiter()
629 	data_vio_count_t available = limiter->limit - limiter->busy;  in update_limiter()
631 	VDO_ASSERT_LOG_ONLY((limiter->release_count <= limiter->busy),  in update_limiter()
633 			    limiter->release_count, limiter->busy);  in update_limiter()
635 	get_waiters(limiter);  in update_limiter()
636 	for (; (limiter->release_count > 0) && !bio_list_empty(waiters); limiter->release_count--)  in update_limiter()
637 		limiter->assigner(limiter);  in update_limiter()
639 	if (limiter->release_count > 0) {  in update_limiter()
640 		WRITE_ONCE(limiter->busy, limiter->busy - limiter->release_count);  in update_limiter()
641 		limiter->release_count = 0;  in update_limiter()
646 		limiter->assigner(limiter);  in update_limiter()
648 	WRITE_ONCE(limiter->busy, limiter->limit - available);  in update_limiter()
649 	if (limiter->max_busy < limiter->busy)  in update_limiter()
650 		WRITE_ONCE(limiter->max_busy, limiter->busy);  in update_limiter()
684 	if (pool->limiter.arrival < pool->discard_limiter.arrival) {  in reuse_or_release_resources()
685 		assign_data_vio(&pool->limiter, data_vio);  in reuse_or_release_resources()
690 		pool->limiter.release_count++;  in reuse_or_release_resources()
710 	get_waiters(&pool->limiter);  in process_release_callback()
713 	if (pool->limiter.arrival == U64_MAX) {  in process_release_callback()
714 		struct bio *bio = bio_list_peek(&pool->limiter.waiters);  in process_release_callback()
717 			pool->limiter.arrival = get_arrival_time(bio);  in process_release_callback()
742 	update_limiter(&pool->limiter);  in process_release_callback()
743 	to_wake = pool->limiter.wake_count;  in process_release_callback()
744 	pool->limiter.wake_count = 0;  in process_release_callback()
759 		wake_up_nr(&pool->limiter.blocked_threads, to_wake);  in process_release_callback()
770 static void initialize_limiter(struct limiter *limiter, struct data_vio_pool *pool,  in initialize_limiter()  argument
773 	limiter->pool = pool;  in initialize_limiter()
774 	limiter->assigner = assigner;  in initialize_limiter()
775 	limiter->limit = limit;  in initialize_limiter()
776 	limiter->arrival = U64_MAX;  in initialize_limiter()
777 	init_waitqueue_head(&limiter->blocked_threads);  in initialize_limiter()
859 	initialize_limiter(&pool->limiter, pool, assign_data_vio_to_waiter, pool_size);  in make_data_vio_pool()
860 	pool->limiter.permitted_waiters = &pool->limiter.waiters;  in make_data_vio_pool()
912 	VDO_ASSERT_LOG_ONLY((pool->limiter.busy == 0),  in free_data_vio_pool()
914 			    pool->limiter.busy);  in free_data_vio_pool()
915 	VDO_ASSERT_LOG_ONLY((bio_list_empty(&pool->limiter.waiters) &&  in free_data_vio_pool()
916 			     bio_list_empty(&pool->limiter.new_waiters)),  in free_data_vio_pool()
932 static bool acquire_permit(struct limiter *limiter)  in acquire_permit()  argument
934 	if (limiter->busy >= limiter->limit)  in acquire_permit()
937 	WRITE_ONCE(limiter->busy, limiter->busy + 1);  in acquire_permit()
938 	if (limiter->max_busy < limiter->busy)  in acquire_permit()
939 		WRITE_ONCE(limiter->max_busy, limiter->busy);  in acquire_permit()
943 static void wait_permit(struct limiter *limiter, struct bio *bio)  in wait_permit()  argument
944 	__releases(&limiter->pool->lock)  in wait_permit()
948 	bio_list_add(&limiter->new_waiters, bio);  in wait_permit()
949 	prepare_to_wait_exclusive(&limiter->blocked_threads, &wait,  in wait_permit()
951 	spin_unlock(&limiter->pool->lock);  in wait_permit()
953 	finish_wait(&limiter->blocked_threads, &wait);  in wait_permit()
976 	if (!acquire_permit(&pool->limiter)) {  in vdo_launch_bio()
977 		wait_permit(&pool->limiter, bio);  in vdo_launch_bio()
1027 static void dump_limiter(const char *name, struct limiter *limiter)  in dump_limiter()  argument
1029 	vdo_log_info("%s: %u of %u busy (max %u), %s", name, limiter->busy,  in dump_limiter()
1030 		     limiter->limit, limiter->max_busy,  in dump_limiter()
1031 		     ((bio_list_empty(&limiter->waiters) &&  in dump_limiter()
1032 		       bio_list_empty(&limiter->new_waiters)) ?  in dump_limiter()
1053 	dump_limiter("data_vios", &pool->limiter);  in dump_data_vio_pool()
1059 		for (i = 0; i < pool->limiter.limit; i++) {  in dump_data_vio_pool()
1080 	return READ_ONCE(pool->limiter.busy);  in get_data_vio_pool_active_requests()
1085 	return READ_ONCE(pool->limiter.limit);  in get_data_vio_pool_request_limit()
1090 	return READ_ONCE(pool->limiter.max_busy);  in get_data_vio_pool_maximum_requests()