6101 static bool scx_dsq_move(struct bpf_iter_scx_dsq_kern *kit,  in scx_dsq_move()  argument
6105 	struct scx_dispatch_q *src_dsq = kit->dsq, *dst_dsq;  in scx_dsq_move()
6150 		     u32_before(kit->cursor.priv, p->scx.dsq_seq) ||  in scx_dsq_move()
6165 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_VTIME)  in scx_dsq_move()
6166 		p->scx.dsq_vtime = kit->vtime;  in scx_dsq_move()
6167 	if (kit->cursor.flags & __SCX_DSQ_ITER_HAS_SLICE)  in scx_dsq_move()
6168 		p->scx.slice = kit->slice;  in scx_dsq_move()
6183 	kit->cursor.flags &= ~(__SCX_DSQ_ITER_HAS_SLICE |  in scx_dsq_move()
6300 	struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;  in scx_bpf_dsq_move_set_slice()  local
6302 	kit->slice = slice;  in scx_bpf_dsq_move_set_slice()
6303 	kit->cursor.flags |= __SCX_DSQ_ITER_HAS_SLICE;  in scx_bpf_dsq_move_set_slice()
6319 	struct bpf_iter_scx_dsq_kern *kit = (void *)it__iter;  in scx_bpf_dsq_move_set_vtime()  local
6321 	kit->vtime = vtime;  in scx_bpf_dsq_move_set_vtime()
6322 	kit->cursor.flags |= __SCX_DSQ_ITER_HAS_VTIME;  in scx_bpf_dsq_move_set_vtime()
6723 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;  in bpf_iter_scx_dsq_new()  local
6737 	kit->dsq = NULL;  in bpf_iter_scx_dsq_new()
6746 	kit->dsq = find_user_dsq(sch, dsq_id);  in bpf_iter_scx_dsq_new()
6747 	if (!kit->dsq)  in bpf_iter_scx_dsq_new()
6750 	kit->cursor = INIT_DSQ_LIST_CURSOR(kit->cursor, flags,  in bpf_iter_scx_dsq_new()
6751 					   READ_ONCE(kit->dsq->seq));  in bpf_iter_scx_dsq_new()
6764 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;  in bpf_iter_scx_dsq_next()  local
6765 	bool rev = kit->cursor.flags & SCX_DSQ_ITER_REV;  in bpf_iter_scx_dsq_next()
6769 	if (!kit->dsq)  in bpf_iter_scx_dsq_next()
6772 	raw_spin_lock_irqsave(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_next()
6774 	if (list_empty(&kit->cursor.node))  in bpf_iter_scx_dsq_next()
6777 		p = container_of(&kit->cursor, struct task_struct, scx.dsq_list);  in bpf_iter_scx_dsq_next()
6785 		p = nldsq_next_task(kit->dsq, p, rev);  in bpf_iter_scx_dsq_next()
6786 	} while (p && unlikely(u32_before(kit->cursor.priv, p->scx.dsq_seq)));  in bpf_iter_scx_dsq_next()
6790 			list_move_tail(&kit->cursor.node, &p->scx.dsq_list.node);  in bpf_iter_scx_dsq_next()
6792 			list_move(&kit->cursor.node, &p->scx.dsq_list.node);  in bpf_iter_scx_dsq_next()
6794 		list_del_init(&kit->cursor.node);  in bpf_iter_scx_dsq_next()
6797 	raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_next()
6810 	struct bpf_iter_scx_dsq_kern *kit = (void *)it;  in bpf_iter_scx_dsq_destroy()  local
6812 	if (!kit->dsq)  in bpf_iter_scx_dsq_destroy()
6815 	if (!list_empty(&kit->cursor.node)) {  in bpf_iter_scx_dsq_destroy()
6818 		raw_spin_lock_irqsave(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_destroy()
6819 		list_del_init(&kit->cursor.node);  in bpf_iter_scx_dsq_destroy()
6820 		raw_spin_unlock_irqrestore(&kit->dsq->lock, flags);  in bpf_iter_scx_dsq_destroy()
6822 	kit->dsq = NULL;  in bpf_iter_scx_dsq_destroy()