50 static void fuse_uring_flush_bg(struct fuse_ring_queue *queue)  in fuse_uring_flush_bg()  argument
52 	struct fuse_ring *ring = queue->ring;  in fuse_uring_flush_bg()
55 	lockdep_assert_held(&queue->lock);  in fuse_uring_flush_bg()
59 	 * Allow one bg request per queue, ignoring global fc limits.  in fuse_uring_flush_bg()
60 	 * This prevents a single queue from consuming all resources and  in fuse_uring_flush_bg()
61 	 * eliminates the need for remote queue wake-ups when global  in fuse_uring_flush_bg()
62 	 * limits are met but this queue has no more waiting requests.  in fuse_uring_flush_bg()
65 		!queue->active_background) &&  in fuse_uring_flush_bg()
66 	       (!list_empty(&queue->fuse_req_bg_queue))) {  in fuse_uring_flush_bg()
69 		req = list_first_entry(&queue->fuse_req_bg_queue,  in fuse_uring_flush_bg()
72 		queue->active_background++;  in fuse_uring_flush_bg()
74 		list_move_tail(&req->list, &queue->fuse_req_queue);  in fuse_uring_flush_bg()
81 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_req_end()  local
82 	struct fuse_ring *ring = queue->ring;  in fuse_uring_req_end()
85 	lockdep_assert_not_held(&queue->lock);  in fuse_uring_req_end()
86 	spin_lock(&queue->lock);  in fuse_uring_req_end()
89 		queue->active_background--;  in fuse_uring_req_end()
91 		fuse_uring_flush_bg(queue);  in fuse_uring_req_end()
95 	spin_unlock(&queue->lock);  in fuse_uring_req_end()
104 /* Abort all list queued request on the given ring queue */
105 static void fuse_uring_abort_end_queue_requests(struct fuse_ring_queue *queue)  in fuse_uring_abort_end_queue_requests()  argument
110 	spin_lock(&queue->lock);  in fuse_uring_abort_end_queue_requests()
111 	list_for_each_entry(req, &queue->fuse_req_queue, list)  in fuse_uring_abort_end_queue_requests()
113 	list_splice_init(&queue->fuse_req_queue, &req_list);  in fuse_uring_abort_end_queue_requests()
114 	spin_unlock(&queue->lock);  in fuse_uring_abort_end_queue_requests()
116 	/* must not hold queue lock to avoid order issues with fi->lock */  in fuse_uring_abort_end_queue_requests()
123 	struct fuse_ring_queue *queue;  in fuse_uring_abort_end_requests()  local
127 		queue = READ_ONCE(ring->queues[qid]);  in fuse_uring_abort_end_requests()
128 		if (!queue)  in fuse_uring_abort_end_requests()
131 		queue->stopped = true;  in fuse_uring_abort_end_requests()
134 		spin_lock(&queue->lock);  in fuse_uring_abort_end_requests()
136 		fuse_uring_flush_bg(queue);  in fuse_uring_abort_end_requests()
138 		spin_unlock(&queue->lock);  in fuse_uring_abort_end_requests()
139 		fuse_uring_abort_end_queue_requests(queue);  in fuse_uring_abort_end_requests()
146 	struct fuse_ring_queue *queue;  in fuse_uring_request_expired()  local
153 		queue = READ_ONCE(ring->queues[qid]);  in fuse_uring_request_expired()
154 		if (!queue)  in fuse_uring_request_expired()
157 		spin_lock(&queue->lock);  in fuse_uring_request_expired()
158 		if (fuse_request_expired(fc, &queue->fuse_req_queue) ||  in fuse_uring_request_expired()
159 		    fuse_request_expired(fc, &queue->fuse_req_bg_queue) ||  in fuse_uring_request_expired()
160 		    fuse_fpq_processing_expired(fc, queue->fpq.processing)) {  in fuse_uring_request_expired()
161 			spin_unlock(&queue->lock);  in fuse_uring_request_expired()
164 		spin_unlock(&queue->lock);  in fuse_uring_request_expired()
179 		struct fuse_ring_queue *queue = ring->queues[qid];  in fuse_uring_destruct()  local
182 		if (!queue)  in fuse_uring_destruct()
185 		WARN_ON(!list_empty(&queue->ent_avail_queue));  in fuse_uring_destruct()
186 		WARN_ON(!list_empty(&queue->ent_w_req_queue));  in fuse_uring_destruct()
187 		WARN_ON(!list_empty(&queue->ent_commit_queue));  in fuse_uring_destruct()
188 		WARN_ON(!list_empty(&queue->ent_in_userspace));  in fuse_uring_destruct()
190 		list_for_each_entry_safe(ent, next, &queue->ent_released,  in fuse_uring_destruct()
196 		kfree(queue->fpq.processing);  in fuse_uring_destruct()
197 		kfree(queue);  in fuse_uring_destruct()
256 	struct fuse_ring_queue *queue;  in fuse_uring_create_queue()  local
259 	queue = kzalloc(sizeof(*queue), GFP_KERNEL_ACCOUNT);  in fuse_uring_create_queue()
260 	if (!queue)  in fuse_uring_create_queue()
264 		kfree(queue);  in fuse_uring_create_queue()
268 	queue->qid = qid;  in fuse_uring_create_queue()
269 	queue->ring = ring;  in fuse_uring_create_queue()
270 	spin_lock_init(&queue->lock);  in fuse_uring_create_queue()
272 	INIT_LIST_HEAD(&queue->ent_avail_queue);  in fuse_uring_create_queue()
273 	INIT_LIST_HEAD(&queue->ent_commit_queue);  in fuse_uring_create_queue()
274 	INIT_LIST_HEAD(&queue->ent_w_req_queue);  in fuse_uring_create_queue()
275 	INIT_LIST_HEAD(&queue->ent_in_userspace);  in fuse_uring_create_queue()
276 	INIT_LIST_HEAD(&queue->fuse_req_queue);  in fuse_uring_create_queue()
277 	INIT_LIST_HEAD(&queue->fuse_req_bg_queue);  in fuse_uring_create_queue()
278 	INIT_LIST_HEAD(&queue->ent_released);  in fuse_uring_create_queue()
280 	queue->fpq.processing = pq;  in fuse_uring_create_queue()
281 	fuse_pqueue_init(&queue->fpq);  in fuse_uring_create_queue()
286 		kfree(queue->fpq.processing);  in fuse_uring_create_queue()
287 		kfree(queue);  in fuse_uring_create_queue()
294 	WRITE_ONCE(ring->queues[qid], queue);  in fuse_uring_create_queue()
297 	return queue;  in fuse_uring_create_queue()
315 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_entry_teardown()  local
317 	spin_lock(&queue->lock);  in fuse_uring_entry_teardown()
323 		/* remove entry from queue->fpq->processing */  in fuse_uring_entry_teardown()
333 	list_move(&ent->list, &queue->ent_released);  in fuse_uring_entry_teardown()
335 	spin_unlock(&queue->lock);  in fuse_uring_entry_teardown()
345 					 struct fuse_ring_queue *queue,  in fuse_uring_stop_list_entries()  argument
348 	struct fuse_ring *ring = queue->ring;  in fuse_uring_stop_list_entries()
353 	spin_lock(&queue->lock);  in fuse_uring_stop_list_entries()
357 				queue->qid, ent->state, exp_state);  in fuse_uring_stop_list_entries()
364 	spin_unlock(&queue->lock);  in fuse_uring_stop_list_entries()
366 	/* no queue lock to avoid lock order issues */  in fuse_uring_stop_list_entries()
374 static void fuse_uring_teardown_entries(struct fuse_ring_queue *queue)  in fuse_uring_teardown_entries()  argument
376 	fuse_uring_stop_list_entries(&queue->ent_in_userspace, queue,  in fuse_uring_teardown_entries()
378 	fuse_uring_stop_list_entries(&queue->ent_avail_queue, queue,  in fuse_uring_teardown_entries()
391 		struct fuse_ring_queue *queue = ring->queues[qid];  in fuse_uring_log_ent_state()  local
393 		if (!queue)  in fuse_uring_log_ent_state()
396 		spin_lock(&queue->lock);  in fuse_uring_log_ent_state()
398 		 * Log entries from the intermediate queue, the other queues  in fuse_uring_log_ent_state()
401 		list_for_each_entry(ent, &queue->ent_w_req_queue, list) {  in fuse_uring_log_ent_state()
402 			pr_info(" ent-req-queue ring=%p qid=%d ent=%p state=%d\n",  in fuse_uring_log_ent_state()
405 		list_for_each_entry(ent, &queue->ent_commit_queue, list) {  in fuse_uring_log_ent_state()
406 			pr_info(" ent-commit-queue ring=%p qid=%d ent=%p state=%d\n",  in fuse_uring_log_ent_state()
409 		spin_unlock(&queue->lock);  in fuse_uring_log_ent_state()
422 		struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]);  in fuse_uring_async_stop_queues()  local
424 		if (!queue)  in fuse_uring_async_stop_queues()
427 		fuse_uring_teardown_entries(queue);  in fuse_uring_async_stop_queues()
435 	 * If there are still queue references left  in fuse_uring_async_stop_queues()
457 		struct fuse_ring_queue *queue = READ_ONCE(ring->queues[qid]);  in fuse_uring_stop_queues()  local
459 		if (!queue)  in fuse_uring_stop_queues()
462 		fuse_uring_teardown_entries(queue);  in fuse_uring_stop_queues()
486 	struct fuse_ring_queue *queue;  in fuse_uring_cancel()  local
493 	queue = ent->queue;  in fuse_uring_cancel()
494 	spin_lock(&queue->lock);  in fuse_uring_cancel()
497 		list_move(&ent->list, &queue->ent_in_userspace);  in fuse_uring_cancel()
501 	spin_unlock(&queue->lock);  in fuse_uring_cancel()
504 		/* no queue lock to avoid lock order issues */  in fuse_uring_cancel()
649 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_copy_to_ring()  local
650 	struct fuse_ring *ring = queue->ring;  in fuse_uring_copy_to_ring()
656 		       queue->qid, ent, ent->state);  in fuse_uring_copy_to_ring()
705 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_send_next_to_ring()  local
713 	spin_lock(&queue->lock);  in fuse_uring_send_next_to_ring()
717 	list_move(&ent->list, &queue->ent_in_userspace);  in fuse_uring_send_next_to_ring()
718 	spin_unlock(&queue->lock);  in fuse_uring_send_next_to_ring()
728 				 struct fuse_ring_queue *queue)  in fuse_uring_ent_avail()  argument
731 	list_move(&ent->list, &queue->ent_avail_queue);  in fuse_uring_ent_avail()
739 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_add_to_pq()  local
740 	struct fuse_pqueue *fpq = &queue->fpq;  in fuse_uring_add_to_pq()
749  * Assign a fuse queue entry to the given entry
754 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_add_req_to_ring_ent()  local
756 	lockdep_assert_held(&queue->lock);  in fuse_uring_add_req_to_ring_ent()
760 		pr_warn("%s qid=%d state=%d\n", __func__, ent->queue->qid,  in fuse_uring_add_req_to_ring_ent()
767 	list_move(&ent->list, &queue->ent_w_req_queue);  in fuse_uring_add_req_to_ring_ent()
773 	__must_hold(&queue->lock)  in fuse_uring_ent_assign_req()
776 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_ent_assign_req()  local
777 	struct list_head *req_queue = &queue->fuse_req_queue;  in fuse_uring_ent_assign_req()
779 	lockdep_assert_held(&queue->lock);  in fuse_uring_ent_assign_req()
797 	struct fuse_ring *ring = ent->queue->ring;  in fuse_uring_commit()
823 				     struct fuse_ring_queue *queue,  in fuse_uring_next_fuse_req()  argument
830 	spin_lock(&queue->lock);  in fuse_uring_next_fuse_req()
831 	fuse_uring_ent_avail(ent, queue);  in fuse_uring_next_fuse_req()
833 	spin_unlock(&queue->lock);  in fuse_uring_next_fuse_req()
844 	struct fuse_ring_queue *queue = ent->queue;  in fuse_ring_ent_set_commit()  local
846 	lockdep_assert_held(&queue->lock);  in fuse_ring_ent_set_commit()
852 	list_move(&ent->list, &queue->ent_commit_queue);  in fuse_ring_ent_set_commit()
865 	struct fuse_ring_queue *queue;  in fuse_uring_commit_fetch()  local
878 	queue = ring->queues[qid];  in fuse_uring_commit_fetch()
879 	if (!queue)  in fuse_uring_commit_fetch()
881 	fpq = &queue->fpq;  in fuse_uring_commit_fetch()
883 	if (!READ_ONCE(fc->connected) || READ_ONCE(queue->stopped))  in fuse_uring_commit_fetch()
886 	spin_lock(&queue->lock);  in fuse_uring_commit_fetch()
895 		pr_info("qid=%d commit_id %llu not found\n", queue->qid,  in fuse_uring_commit_fetch()
897 		spin_unlock(&queue->lock);  in fuse_uring_commit_fetch()
907 				    queue->qid, commit_id, ent->state);  in fuse_uring_commit_fetch()
908 		spin_unlock(&queue->lock);  in fuse_uring_commit_fetch()
916 	spin_unlock(&queue->lock);  in fuse_uring_commit_fetch()
918 	/* without the queue lock, as other locks are taken */  in fuse_uring_commit_fetch()
928 	fuse_uring_next_fuse_req(ent, queue, issue_flags);  in fuse_uring_commit_fetch()
935 	struct fuse_ring_queue *queue;  in is_ring_ready()  local
942 		queue = ring->queues[qid];  in is_ring_ready()
943 		if (!queue) {  in is_ring_ready()
948 		spin_lock(&queue->lock);  in is_ring_ready()
949 		if (list_empty(&queue->ent_avail_queue))  in is_ring_ready()
951 		spin_unlock(&queue->lock);  in is_ring_ready()
964 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_do_register()  local
965 	struct fuse_ring *ring = queue->ring;  in fuse_uring_do_register()
971 	spin_lock(&queue->lock);  in fuse_uring_do_register()
973 	fuse_uring_ent_avail(ent, queue);  in fuse_uring_do_register()
974 	spin_unlock(&queue->lock);  in fuse_uring_do_register()
977 		bool ready = is_ring_ready(ring, queue->qid);  in fuse_uring_do_register()
1015 			   struct fuse_ring_queue *queue)  in fuse_uring_create_ring_ent()  argument
1017 	struct fuse_ring *ring = queue->ring;  in fuse_uring_create_ring_ent()
1050 	ent->queue = queue;  in fuse_uring_create_ring_ent()
1060  * entry as "ready to get fuse requests" on the queue
1067 	struct fuse_ring_queue *queue;  in fuse_uring_register()  local
1084 	queue = ring->queues[qid];  in fuse_uring_register()
1085 	if (!queue) {  in fuse_uring_register()
1086 		queue = fuse_uring_create_queue(ring, qid);  in fuse_uring_register()
1087 		if (!queue)  in fuse_uring_register()
1092 	 * The created queue above does not need to be destructed in  in fuse_uring_register()
1096 	ent = fuse_uring_create_ring_ent(cmd, queue);  in fuse_uring_register()
1179 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_send()  local
1181 	spin_lock(&queue->lock);  in fuse_uring_send()
1183 	list_move(&ent->list, &queue->ent_in_userspace);  in fuse_uring_send()
1185 	spin_unlock(&queue->lock);  in fuse_uring_send()
1199 	struct fuse_ring_queue *queue = ent->queue;  in fuse_uring_send_in_task()  local
1205 			fuse_uring_next_fuse_req(ent, queue, issue_flags);  in fuse_uring_send_in_task()
1218 	struct fuse_ring_queue *queue;  in fuse_uring_task_to_queue()  local
1227 	queue = ring->queues[qid];  in fuse_uring_task_to_queue()
1228 	WARN_ONCE(!queue, "Missing queue for qid %d\n", qid);  in fuse_uring_task_to_queue()
1230 	return queue;  in fuse_uring_task_to_queue()
1241 /* queue a fuse request and send it if a ring entry is available */
1246 	struct fuse_ring_queue *queue;  in fuse_uring_queue_fuse_req()  local
1251 	queue = fuse_uring_task_to_queue(ring);  in fuse_uring_queue_fuse_req()
1252 	if (!queue)  in fuse_uring_queue_fuse_req()
1258 	spin_lock(&queue->lock);  in fuse_uring_queue_fuse_req()
1260 	if (unlikely(queue->stopped))  in fuse_uring_queue_fuse_req()
1264 	req->ring_queue = queue;  in fuse_uring_queue_fuse_req()
1265 	ent = list_first_entry_or_null(&queue->ent_avail_queue,  in fuse_uring_queue_fuse_req()
1270 		list_add_tail(&req->list, &queue->fuse_req_queue);  in fuse_uring_queue_fuse_req()
1271 	spin_unlock(&queue->lock);  in fuse_uring_queue_fuse_req()
1279 	spin_unlock(&queue->lock);  in fuse_uring_queue_fuse_req()
1290 	struct fuse_ring_queue *queue;  in fuse_uring_queue_bq_req()  local
1293 	queue = fuse_uring_task_to_queue(ring);  in fuse_uring_queue_bq_req()
1294 	if (!queue)  in fuse_uring_queue_bq_req()
1297 	spin_lock(&queue->lock);  in fuse_uring_queue_bq_req()
1298 	if (unlikely(queue->stopped)) {  in fuse_uring_queue_bq_req()
1299 		spin_unlock(&queue->lock);  in fuse_uring_queue_bq_req()
1304 	req->ring_queue = queue;  in fuse_uring_queue_bq_req()
1305 	list_add_tail(&req->list, &queue->fuse_req_bg_queue);  in fuse_uring_queue_bq_req()
1307 	ent = list_first_entry_or_null(&queue->ent_avail_queue,  in fuse_uring_queue_bq_req()
1313 	fuse_uring_flush_bg(queue);  in fuse_uring_queue_bq_req()
1318 	 * in the queue that need to be handled first. Or no further req  in fuse_uring_queue_bq_req()
1321 	req = list_first_entry_or_null(&queue->fuse_req_queue, struct fuse_req,  in fuse_uring_queue_bq_req()
1325 		spin_unlock(&queue->lock);  in fuse_uring_queue_bq_req()
1329 		spin_unlock(&queue->lock);  in fuse_uring_queue_bq_req()
1337 	struct fuse_ring_queue *queue = req->ring_queue;  in fuse_uring_remove_pending_req()  local
1339 	return fuse_remove_pending_req(req, &queue->lock);  in fuse_uring_remove_pending_req()