30 #include <trace/events/qdisc.h>
34 /* Qdisc to use by default */
38 static void qdisc_maybe_clear_missed(struct Qdisc *q,  in qdisc_maybe_clear_missed()
62  * qdisc_lock(qdisc) spinlock.
65  * - enqueue, dequeue are serialized via qdisc root lock
66  * - ingress filtering is also serialized via qdisc root lock
72 static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)  in __skb_dequeue_bad_txq()
108 static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q)  in qdisc_dequeue_skb_bad_txq()
118 static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,  in qdisc_enqueue_skb_bad_txq()
142 static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)  in dev_requeue_skb()
178 static void try_bulk_dequeue_skb(struct Qdisc *q,  in try_bulk_dequeue_skb()
202 static void try_bulk_dequeue_skb_slow(struct Qdisc *q,  in try_bulk_dequeue_skb_slow()
228 static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,  in dequeue_skb()
307  * required. Owning qdisc running bit guarantees that only one CPU
314 bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,  in sch_direct_xmit()
321 	/* And release qdisc */  in sch_direct_xmit()
373  * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
388 static inline bool qdisc_restart(struct Qdisc *q, int *packets)  in qdisc_restart()
410 void __qdisc_run(struct Qdisc *q)  in __qdisc_run()
629 static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,  in noop_enqueue()  argument
636 static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)  in noop_dequeue()  argument
651 	RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
655 struct Qdisc noop_qdisc = {
678 static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,  in noqueue_init()  argument
684 	qdisc->enqueue = NULL;  in noqueue_init()
723 static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,  in pfifo_fast_enqueue()  argument
727 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_enqueue()
735 		if (qdisc_is_percpu_stats(qdisc))  in pfifo_fast_enqueue()
736 			return qdisc_drop_cpu(skb, qdisc, to_free);  in pfifo_fast_enqueue()
738 			return qdisc_drop(skb, qdisc, to_free);  in pfifo_fast_enqueue()
741 	qdisc_update_stats_at_enqueue(qdisc, pkt_len);  in pfifo_fast_enqueue()
745 static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)  in pfifo_fast_dequeue()  argument
747 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_dequeue()
762 		qdisc_update_stats_at_dequeue(qdisc, skb);  in pfifo_fast_dequeue()
764 		   READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY) {  in pfifo_fast_dequeue()
770 		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);  in pfifo_fast_dequeue()
771 		clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);  in pfifo_fast_dequeue()
786 static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)  in pfifo_fast_peek()  argument
788 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_peek()
801 static void pfifo_fast_reset(struct Qdisc *qdisc)  in pfifo_fast_reset()  argument
804 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_reset()
820 	if (qdisc_is_percpu_stats(qdisc)) {  in pfifo_fast_reset()
824 			q = per_cpu_ptr(qdisc->cpu_qstats, i);  in pfifo_fast_reset()
831 static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)  in pfifo_fast_dump()  argument
844 static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,  in pfifo_fast_init()  argument
847 	unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;  in pfifo_fast_init()
848 	struct pfifo_fast_priv *priv = qdisc_priv(qdisc);  in pfifo_fast_init()
865 	qdisc->flags |= TCQ_F_CAN_BYPASS;  in pfifo_fast_init()
869 static void pfifo_fast_destroy(struct Qdisc *sch)  in pfifo_fast_destroy()
889 static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch,  in pfifo_fast_change_tx_queue_len()
924 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,  in qdisc_alloc()
928 	struct Qdisc *sch;  in qdisc_alloc()
966 	/* seqlock has the same scope of busylock, for NOLOCK qdisc */  in qdisc_alloc()
986 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,  in qdisc_create_dflt()
991 	struct Qdisc *sch;  in qdisc_create_dflt()
1015 /* Under qdisc_lock(qdisc) and BH! */
1017 void qdisc_reset(struct Qdisc *qdisc)  in qdisc_reset()  argument
1019 	const struct Qdisc_ops *ops = qdisc->ops;  in qdisc_reset()
1021 	trace_qdisc_reset(qdisc);  in qdisc_reset()
1024 		ops->reset(qdisc);  in qdisc_reset()
1026 	__skb_queue_purge(&qdisc->gso_skb);  in qdisc_reset()
1027 	__skb_queue_purge(&qdisc->skb_bad_txq);  in qdisc_reset()
1029 	qdisc->q.qlen = 0;  in qdisc_reset()
1030 	qdisc->qstats.backlog = 0;  in qdisc_reset()
1034 void qdisc_free(struct Qdisc *qdisc)  in qdisc_free()  argument
1036 	if (qdisc_is_percpu_stats(qdisc)) {  in qdisc_free()
1037 		free_percpu(qdisc->cpu_bstats);  in qdisc_free()
1038 		free_percpu(qdisc->cpu_qstats);  in qdisc_free()
1041 	kfree(qdisc);  in qdisc_free()
1046 	struct Qdisc *q = container_of(head, struct Qdisc, rcu);  in qdisc_free_cb()
1051 static void __qdisc_destroy(struct Qdisc *qdisc)  in __qdisc_destroy()  argument
1053 	const struct Qdisc_ops  *ops = qdisc->ops;  in __qdisc_destroy()
1054 	struct net_device *dev = qdisc_dev(qdisc);  in __qdisc_destroy()
1057 	qdisc_hash_del(qdisc);  in __qdisc_destroy()
1059 	qdisc_put_stab(rtnl_dereference(qdisc->stab));  in __qdisc_destroy()
1061 	gen_kill_estimator(&qdisc->rate_est);  in __qdisc_destroy()
1063 	qdisc_reset(qdisc);  in __qdisc_destroy()
1067 		ops->destroy(qdisc);  in __qdisc_destroy()
1070 	netdev_put(dev, &qdisc->dev_tracker);  in __qdisc_destroy()
1072 	trace_qdisc_destroy(qdisc);  in __qdisc_destroy()
1074 	call_rcu(&qdisc->rcu, qdisc_free_cb);  in __qdisc_destroy()
1077 void qdisc_destroy(struct Qdisc *qdisc)  in qdisc_destroy()  argument
1079 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_destroy()
1082 	__qdisc_destroy(qdisc);  in qdisc_destroy()
1085 void qdisc_put(struct Qdisc *qdisc)  in qdisc_put()  argument
1087 	if (!qdisc)  in qdisc_put()
1090 	if (qdisc->flags & TCQ_F_BUILTIN ||  in qdisc_put()
1091 	    !refcount_dec_and_test(&qdisc->refcnt))  in qdisc_put()
1094 	__qdisc_destroy(qdisc);  in qdisc_put()
1100  * qdisc reference counter reached zero.
1103 void qdisc_put_unlocked(struct Qdisc *qdisc)  in qdisc_put_unlocked()  argument
1105 	if (qdisc->flags & TCQ_F_BUILTIN ||  in qdisc_put_unlocked()
1106 	    !refcount_dec_and_rtnl_lock(&qdisc->refcnt))  in qdisc_put_unlocked()
1109 	__qdisc_destroy(qdisc);  in qdisc_put_unlocked()
1114 /* Attach toplevel qdisc to device queue. */
1115 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,  in dev_graft_qdisc()
1116 			      struct Qdisc *qdisc)  in dev_graft_qdisc()  argument
1118 	struct Qdisc *oqdisc = rtnl_dereference(dev_queue->qdisc_sleeping);  in dev_graft_qdisc()
1125 	if (qdisc == NULL)  in dev_graft_qdisc()
1126 		qdisc = &noop_qdisc;  in dev_graft_qdisc()
1127 	rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);  in dev_graft_qdisc()
1128 	rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);  in dev_graft_qdisc()
1140 	struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);  in shutdown_scheduler_queue()  local
1141 	struct Qdisc *qdisc_default = _qdisc_default;  in shutdown_scheduler_queue()
1143 	if (qdisc) {  in shutdown_scheduler_queue()
1144 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);  in shutdown_scheduler_queue()
1147 		qdisc_put(qdisc);  in shutdown_scheduler_queue()
1155 	struct Qdisc *qdisc;  in attach_one_default_qdisc()  local
1163 	qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);  in attach_one_default_qdisc()
1164 	if (!qdisc)  in attach_one_default_qdisc()
1168 		qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;  in attach_one_default_qdisc()
1169 	rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);  in attach_one_default_qdisc()
1175 	struct Qdisc *qdisc;  in attach_default_qdiscs()  local
1182 		qdisc = rtnl_dereference(txq->qdisc_sleeping);  in attach_default_qdiscs()
1183 		rcu_assign_pointer(dev->qdisc, qdisc);  in attach_default_qdiscs()
1184 		qdisc_refcount_inc(qdisc);  in attach_default_qdiscs()
1186 		qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);  in attach_default_qdiscs()
1187 		if (qdisc) {  in attach_default_qdiscs()
1188 			rcu_assign_pointer(dev->qdisc, qdisc);  in attach_default_qdiscs()
1189 			qdisc->ops->attach(qdisc);  in attach_default_qdiscs()
1192 	qdisc = rtnl_dereference(dev->qdisc);  in attach_default_qdiscs()
1194 	/* Detect default qdisc setup/init failed and fallback to "noqueue" */  in attach_default_qdiscs()
1195 	if (qdisc == &noop_qdisc) {  in attach_default_qdiscs()
1196 		netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",  in attach_default_qdiscs()
1201 		qdisc = rtnl_dereference(txq->qdisc_sleeping);  in attach_default_qdiscs()
1202 		rcu_assign_pointer(dev->qdisc, qdisc);  in attach_default_qdiscs()
1203 		qdisc_refcount_inc(qdisc);  in attach_default_qdiscs()
1208 	if (qdisc != &noop_qdisc)  in attach_default_qdiscs()
1209 		qdisc_hash_add(qdisc, false);  in attach_default_qdiscs()
1217 	struct Qdisc *new_qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);  in transition_one_qdisc()
1223 	rcu_assign_pointer(dev_queue->qdisc, new_qdisc);  in transition_one_qdisc()
1239 	if (rtnl_dereference(dev->qdisc) == &noop_qdisc)  in dev_activate()
1258 static void qdisc_deactivate(struct Qdisc *qdisc)  in qdisc_deactivate()  argument
1260 	if (qdisc->flags & TCQ_F_BUILTIN)  in qdisc_deactivate()
1263 	set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);  in qdisc_deactivate()
1270 	struct Qdisc *qdisc_default = _qdisc_default;  in dev_deactivate_queue()
1271 	struct Qdisc *qdisc;  in dev_deactivate_queue()  local
1273 	qdisc = rtnl_dereference(dev_queue->qdisc);  in dev_deactivate_queue()
1274 	if (qdisc) {  in dev_deactivate_queue()
1275 		qdisc_deactivate(qdisc);  in dev_deactivate_queue()
1276 		rcu_assign_pointer(dev_queue->qdisc, qdisc_default);  in dev_deactivate_queue()
1284 	struct Qdisc *qdisc;  in dev_reset_queue()  local
1287 	qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);  in dev_reset_queue()
1288 	if (!qdisc)  in dev_reset_queue()
1291 	nolock = qdisc->flags & TCQ_F_NOLOCK;  in dev_reset_queue()
1294 		spin_lock_bh(&qdisc->seqlock);  in dev_reset_queue()
1295 	spin_lock_bh(qdisc_lock(qdisc));  in dev_reset_queue()
1297 	qdisc_reset(qdisc);  in dev_reset_queue()
1299 	spin_unlock_bh(qdisc_lock(qdisc));  in dev_reset_queue()
1301 		clear_bit(__QDISC_STATE_MISSED, &qdisc->state);  in dev_reset_queue()
1302 		clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);  in dev_reset_queue()
1303 		spin_unlock_bh(&qdisc->seqlock);  in dev_reset_queue()
1314 		struct Qdisc *q;  in some_qdisc_is_busy()
1355 	/* Wait for outstanding qdisc-less dev_queue_xmit calls or  in dev_deactivate_many()
1356 	 * outstanding qdisc enqueuing calls.  in dev_deactivate_many()
1394 	struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);  in qdisc_change_tx_queue_len()  local
1395 	const struct Qdisc_ops *ops = qdisc->ops;  in qdisc_change_tx_queue_len()
1398 		return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);  in qdisc_change_tx_queue_len()
1405 	struct Qdisc *qdisc = rtnl_dereference(dev->qdisc);  in dev_qdisc_change_real_num_tx()  local
1407 	if (qdisc->ops->change_real_num_tx)  in dev_qdisc_change_real_num_tx()
1408 		qdisc->ops->change_real_num_tx(qdisc, new_real_tx);  in dev_qdisc_change_real_num_tx()
1411 void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx)  in mq_change_real_num_tx()
1415 	struct Qdisc *qdisc;  in mq_change_real_num_tx()  local
1419 		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);  in mq_change_real_num_tx()
1423 		if (qdisc != &noop_qdisc && !qdisc->handle)  in mq_change_real_num_tx()
1424 			qdisc_hash_del(qdisc);  in mq_change_real_num_tx()
1427 		qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);  in mq_change_real_num_tx()
1428 		if (qdisc != &noop_qdisc && !qdisc->handle)  in mq_change_real_num_tx()
1429 			qdisc_hash_add(qdisc, false);  in mq_change_real_num_tx()
1461 	struct Qdisc *qdisc = _qdisc;  in dev_init_scheduler_queue()  local
1463 	rcu_assign_pointer(dev_queue->qdisc, qdisc);  in dev_init_scheduler_queue()
1464 	rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);  in dev_init_scheduler_queue()
1469 	rcu_assign_pointer(dev->qdisc, &noop_qdisc);  in dev_init_scheduler()
1482 	qdisc_put(rtnl_dereference(dev->qdisc));  in dev_shutdown()
1483 	rcu_assign_pointer(dev->qdisc, &noop_qdisc);  in dev_shutdown()
1597 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,  in mini_qdisc_pair_init()  argument
1600 	miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;  in mini_qdisc_pair_init()
1601 	miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;  in mini_qdisc_pair_init()
1602 	miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;  in mini_qdisc_pair_init()
1603 	miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;  in mini_qdisc_pair_init()