75 static unsigned int fq_pie_hash(const struct fq_pie_sched_data *q,  in fq_pie_hash()  argument
78 	return reciprocal_scale(skb_get_hash(skb), q->flows_cnt);  in fq_pie_hash()
84 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_classify()  local
91 	    TC_H_MIN(skb->priority) <= q->flows_cnt)  in fq_pie_classify()
94 	filter = rcu_dereference_bh(q->filter_list);  in fq_pie_classify()
96 		return fq_pie_hash(q, skb) + 1;  in fq_pie_classify()
112 		if (TC_H_MIN(res.classid) <= q->flows_cnt)  in fq_pie_classify()
134 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_qdisc_enqueue()  local
152 	sel_flow = &q->flows[idx];  in fq_pie_qdisc_enqueue()
155 	memory_limited = q->memory_usage > q->memory_limit + skb->truesize;  in fq_pie_qdisc_enqueue()
159 		q->stats.overlimit++;  in fq_pie_qdisc_enqueue()
162 		q->overmemory++;  in fq_pie_qdisc_enqueue()
167 	if (!pie_drop_early(sch, &q->p_params, &sel_flow->vars,  in fq_pie_qdisc_enqueue()
170 	} else if (q->p_params.ecn &&  in fq_pie_qdisc_enqueue()
171 		   sel_flow->vars.prob <= (MAX_PROB / 100) * q->ecn_prob &&  in fq_pie_qdisc_enqueue()
176 		q->stats.ecn_mark++;  in fq_pie_qdisc_enqueue()
181 		if (!q->p_params.dq_rate_estimator)  in fq_pie_qdisc_enqueue()
185 		q->stats.packets_in++;  in fq_pie_qdisc_enqueue()
186 		q->memory_usage += skb->truesize;  in fq_pie_qdisc_enqueue()
188 		sch->q.qlen++;  in fq_pie_qdisc_enqueue()
191 			list_add_tail(&sel_flow->flowchain, &q->new_flows);  in fq_pie_qdisc_enqueue()
192 			q->new_flow_count++;  in fq_pie_qdisc_enqueue()
193 			sel_flow->deficit = q->quantum;  in fq_pie_qdisc_enqueue()
202 	q->stats.dropped++;  in fq_pie_qdisc_enqueue()
240 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_qdisc_dequeue()  local
247 	head = &q->new_flows;  in fq_pie_qdisc_dequeue()
249 		head = &q->old_flows;  in fq_pie_qdisc_dequeue()
257 		flow->deficit += q->quantum;  in fq_pie_qdisc_dequeue()
258 		list_move_tail(&flow->flowchain, &q->old_flows);  in fq_pie_qdisc_dequeue()
266 		sch->q.qlen--;  in fq_pie_qdisc_dequeue()
272 		if (head == &q->new_flows && !list_empty(&q->old_flows))  in fq_pie_qdisc_dequeue()
273 			list_move_tail(&flow->flowchain, &q->old_flows);  in fq_pie_qdisc_dequeue()
282 	q->memory_usage -= get_pie_cb(skb)->mem_usage;  in fq_pie_qdisc_dequeue()
283 	pie_process_dequeue(skb, &q->p_params, &flow->vars, flow->backlog);  in fq_pie_qdisc_dequeue()
291 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_change()  local
303 		WRITE_ONCE(q->p_params.limit, limit);  in fq_pie_change()
307 		if (q->flows) {  in fq_pie_change()
312 		q->flows_cnt = nla_get_u32(tb[TCA_FQ_PIE_FLOWS]);  in fq_pie_change()
313 		if (!q->flows_cnt || q->flows_cnt > 65536) {  in fq_pie_change()
326 		WRITE_ONCE(q->p_params.target,  in fq_pie_change()
332 		WRITE_ONCE(q->p_params.tupdate,  in fq_pie_change()
336 		WRITE_ONCE(q->p_params.alpha,  in fq_pie_change()
340 		WRITE_ONCE(q->p_params.beta,  in fq_pie_change()
344 		WRITE_ONCE(q->quantum, nla_get_u32(tb[TCA_FQ_PIE_QUANTUM]));  in fq_pie_change()
347 		WRITE_ONCE(q->memory_limit,  in fq_pie_change()
351 		WRITE_ONCE(q->ecn_prob,  in fq_pie_change()
355 		WRITE_ONCE(q->p_params.ecn,  in fq_pie_change()
359 		WRITE_ONCE(q->p_params.bytemode,  in fq_pie_change()
363 		WRITE_ONCE(q->p_params.dq_rate_estimator,  in fq_pie_change()
367 	while (sch->q.qlen > sch->limit) {  in fq_pie_change()
389 	struct fq_pie_sched_data *q = timer_container_of(q, t, adapt_timer);  in fq_pie_timer()  local
391 	struct Qdisc *sch = q->sch;  in fq_pie_timer()
400 	max_cnt = min_t(int, q->flows_cnt - q->flows_cursor, 2048);  in fq_pie_timer()
402 		pie_calculate_probability(&q->p_params,  in fq_pie_timer()
403 					  &q->flows[q->flows_cursor].vars,  in fq_pie_timer()
404 					  q->flows[q->flows_cursor].backlog);  in fq_pie_timer()
405 		q->flows_cursor++;  in fq_pie_timer()
408 	tupdate = q->p_params.tupdate;  in fq_pie_timer()
410 	if (q->flows_cursor >= q->flows_cnt) {  in fq_pie_timer()
411 		q->flows_cursor = 0;  in fq_pie_timer()
415 		mod_timer(&q->adapt_timer, jiffies + next);  in fq_pie_timer()
423 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_init()  local
427 	pie_params_init(&q->p_params);  in fq_pie_init()
429 	q->p_params.limit = sch->limit;  in fq_pie_init()
430 	q->quantum = psched_mtu(qdisc_dev(sch));  in fq_pie_init()
431 	q->sch = sch;  in fq_pie_init()
432 	q->ecn_prob = 10;  in fq_pie_init()
433 	q->flows_cnt = 1024;  in fq_pie_init()
434 	q->memory_limit = SZ_32M;  in fq_pie_init()
436 	INIT_LIST_HEAD(&q->new_flows);  in fq_pie_init()
437 	INIT_LIST_HEAD(&q->old_flows);  in fq_pie_init()
438 	timer_setup(&q->adapt_timer, fq_pie_timer, 0);  in fq_pie_init()
447 	err = tcf_block_get(&q->block, &q->filter_list, sch, extack);  in fq_pie_init()
451 	q->flows = kvcalloc(q->flows_cnt, sizeof(struct fq_pie_flow),  in fq_pie_init()
453 	if (!q->flows) {  in fq_pie_init()
457 	for (idx = 0; idx < q->flows_cnt; idx++) {  in fq_pie_init()
458 		struct fq_pie_flow *flow = q->flows + idx;  in fq_pie_init()
464 	mod_timer(&q->adapt_timer, jiffies + HZ / 2);  in fq_pie_init()
469 	q->flows_cnt = 0;  in fq_pie_init()
476 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_dump()  local
485 	    nla_put_u32(skb, TCA_FQ_PIE_FLOWS, READ_ONCE(q->flows_cnt)) ||  in fq_pie_dump()
487 			((u32)PSCHED_TICKS2NS(READ_ONCE(q->p_params.target))) /  in fq_pie_dump()
490 			jiffies_to_usecs(READ_ONCE(q->p_params.tupdate))) ||  in fq_pie_dump()
491 	    nla_put_u32(skb, TCA_FQ_PIE_ALPHA, READ_ONCE(q->p_params.alpha)) ||  in fq_pie_dump()
492 	    nla_put_u32(skb, TCA_FQ_PIE_BETA, READ_ONCE(q->p_params.beta)) ||  in fq_pie_dump()
493 	    nla_put_u32(skb, TCA_FQ_PIE_QUANTUM, READ_ONCE(q->quantum)) ||  in fq_pie_dump()
495 			READ_ONCE(q->memory_limit)) ||  in fq_pie_dump()
496 	    nla_put_u32(skb, TCA_FQ_PIE_ECN_PROB, READ_ONCE(q->ecn_prob)) ||  in fq_pie_dump()
497 	    nla_put_u32(skb, TCA_FQ_PIE_ECN, READ_ONCE(q->p_params.ecn)) ||  in fq_pie_dump()
498 	    nla_put_u32(skb, TCA_FQ_PIE_BYTEMODE, READ_ONCE(q->p_params.bytemode)) ||  in fq_pie_dump()
500 			READ_ONCE(q->p_params.dq_rate_estimator)))  in fq_pie_dump()
512 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_dump_stats()  local
514 		.packets_in	= q->stats.packets_in,  in fq_pie_dump_stats()
515 		.overlimit	= q->stats.overlimit,  in fq_pie_dump_stats()
516 		.overmemory	= q->overmemory,  in fq_pie_dump_stats()
517 		.dropped	= q->stats.dropped,  in fq_pie_dump_stats()
518 		.ecn_mark	= q->stats.ecn_mark,  in fq_pie_dump_stats()
519 		.new_flow_count = q->new_flow_count,  in fq_pie_dump_stats()
520 		.memory_usage   = q->memory_usage,  in fq_pie_dump_stats()
525 	list_for_each(pos, &q->new_flows)  in fq_pie_dump_stats()
528 	list_for_each(pos, &q->old_flows)  in fq_pie_dump_stats()
537 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_reset()  local
540 	INIT_LIST_HEAD(&q->new_flows);  in fq_pie_reset()
541 	INIT_LIST_HEAD(&q->old_flows);  in fq_pie_reset()
542 	for (idx = 0; idx < q->flows_cnt; idx++) {  in fq_pie_reset()
543 		struct fq_pie_flow *flow = q->flows + idx;  in fq_pie_reset()
556 	struct fq_pie_sched_data *q = qdisc_priv(sch);  in fq_pie_destroy()  local
558 	tcf_block_put(q->block);  in fq_pie_destroy()
559 	q->p_params.tupdate = 0;  in fq_pie_destroy()
560 	timer_delete_sync(&q->adapt_timer);  in fq_pie_destroy()
561 	kvfree(q->flows);  in fq_pie_destroy()