Lines Matching full:qdisc
45 /* Only for !TCQ_F_NOLOCK qdisc. Never access it directly.
73 struct Qdisc { struct
75 struct Qdisc *sch, argument
77 struct sk_buff * (*dequeue)(struct Qdisc *sch); argument
96 #define TCQ_F_NOLOCK 0x100 /* qdisc does not require locking */
97 #define TCQ_F_OFFLOADED 0x200 /* qdisc is offloaded to HW */
121 unsigned long state2; /* must be written under qdisc spinlock */
122 struct Qdisc *next_sched; argument
134 static inline void qdisc_refcount_inc(struct Qdisc *qdisc) in qdisc_refcount_inc() argument
136 if (qdisc->flags & TCQ_F_BUILTIN) in qdisc_refcount_inc()
138 refcount_inc(&qdisc->refcnt); in qdisc_refcount_inc()
141 static inline bool qdisc_refcount_dec_if_one(struct Qdisc *qdisc) in qdisc_refcount_dec_if_one() argument
143 if (qdisc->flags & TCQ_F_BUILTIN) in qdisc_refcount_dec_if_one()
145 return refcount_dec_if_one(&qdisc->refcnt); in qdisc_refcount_dec_if_one()
148 /* Intended to be used by unlocked users, when concurrent qdisc release is
152 static inline struct Qdisc *qdisc_refcount_inc_nz(struct Qdisc *qdisc) in qdisc_refcount_inc_nz() argument
154 if (qdisc->flags & TCQ_F_BUILTIN) in qdisc_refcount_inc_nz()
155 return qdisc; in qdisc_refcount_inc_nz()
156 if (refcount_inc_not_zero(&qdisc->refcnt)) in qdisc_refcount_inc_nz()
157 return qdisc; in qdisc_refcount_inc_nz()
161 /* For !TCQ_F_NOLOCK qdisc: callers must either call this within a qdisc
165 static inline bool qdisc_is_running(struct Qdisc *qdisc) in qdisc_is_running() argument
167 if (qdisc->flags & TCQ_F_NOLOCK) in qdisc_is_running()
168 return spin_is_locked(&qdisc->seqlock); in qdisc_is_running()
169 return test_bit(__QDISC_STATE2_RUNNING, &qdisc->state2); in qdisc_is_running()
172 static inline bool nolock_qdisc_is_empty(const struct Qdisc *qdisc) in nolock_qdisc_is_empty() argument
174 return !(READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY); in nolock_qdisc_is_empty()
177 static inline bool qdisc_is_percpu_stats(const struct Qdisc *q) in qdisc_is_percpu_stats()
182 static inline bool qdisc_is_empty(const struct Qdisc *qdisc) in qdisc_is_empty() argument
184 if (qdisc_is_percpu_stats(qdisc)) in qdisc_is_empty()
185 return nolock_qdisc_is_empty(qdisc); in qdisc_is_empty()
186 return !READ_ONCE(qdisc->q.qlen); in qdisc_is_empty()
189 /* For !TCQ_F_NOLOCK qdisc, qdisc_run_begin/end() must be invoked with
190 * the qdisc root lock acquired.
192 static inline bool qdisc_run_begin(struct Qdisc *qdisc) in qdisc_run_begin() argument
194 if (qdisc->flags & TCQ_F_NOLOCK) { in qdisc_run_begin()
195 if (spin_trylock(&qdisc->seqlock)) in qdisc_run_begin()
203 if (test_and_set_bit(__QDISC_STATE_MISSED, &qdisc->state)) in qdisc_run_begin()
210 return spin_trylock(&qdisc->seqlock); in qdisc_run_begin()
212 return !__test_and_set_bit(__QDISC_STATE2_RUNNING, &qdisc->state2); in qdisc_run_begin()
215 static inline void qdisc_run_end(struct Qdisc *qdisc) in qdisc_run_end() argument
217 if (qdisc->flags & TCQ_F_NOLOCK) { in qdisc_run_end()
218 spin_unlock(&qdisc->seqlock); in qdisc_run_end()
227 &qdisc->state))) in qdisc_run_end()
228 __netif_schedule(qdisc); in qdisc_run_end()
230 __clear_bit(__QDISC_STATE2_RUNNING, &qdisc->state2); in qdisc_run_end()
234 static inline bool qdisc_may_bulk(const struct Qdisc *qdisc) in qdisc_may_bulk() argument
236 return qdisc->flags & TCQ_F_ONETXQUEUE; in qdisc_may_bulk()
246 /* Child qdisc manipulation */
247 struct netdev_queue * (*select_queue)(struct Qdisc *, struct tcmsg *);
248 int (*graft)(struct Qdisc *, unsigned long cl,
249 struct Qdisc *, struct Qdisc **,
251 struct Qdisc * (*leaf)(struct Qdisc *, unsigned long cl);
252 void (*qlen_notify)(struct Qdisc *, unsigned long);
255 unsigned long (*find)(struct Qdisc *, u32 classid);
256 int (*change)(struct Qdisc *, u32, u32,
259 int (*delete)(struct Qdisc *, unsigned long,
261 void (*walk)(struct Qdisc *, struct qdisc_walker * arg);
264 struct tcf_block * (*tcf_block)(struct Qdisc *sch,
267 unsigned long (*bind_tcf)(struct Qdisc *, unsigned long,
269 void (*unbind_tcf)(struct Qdisc *, unsigned long);
272 int (*dump)(struct Qdisc *, unsigned long,
274 int (*dump_stats)(struct Qdisc *, unsigned long,
293 struct Qdisc *sch,
295 struct sk_buff * (*dequeue)(struct Qdisc *);
296 struct sk_buff * (*peek)(struct Qdisc *);
298 int (*init)(struct Qdisc *sch, struct nlattr *arg,
300 void (*reset)(struct Qdisc *);
301 void (*destroy)(struct Qdisc *);
302 int (*change)(struct Qdisc *sch,
305 void (*attach)(struct Qdisc *sch);
306 int (*change_tx_queue_len)(struct Qdisc *, unsigned int);
307 void (*change_real_num_tx)(struct Qdisc *sch,
310 int (*dump)(struct Qdisc *, struct sk_buff *);
311 int (*dump_stats)(struct Qdisc *, struct gnet_dump *);
313 void (*ingress_block_set)(struct Qdisc *sch,
315 void (*egress_block_set)(struct Qdisc *sch,
317 u32 (*ingress_block_get)(struct Qdisc *sch);
318 u32 (*egress_block_get)(struct Qdisc *sch);
469 struct Qdisc *q;
512 static inline int qdisc_qlen(const struct Qdisc *q) in qdisc_qlen()
517 static inline int qdisc_qlen_sum(const struct Qdisc *q) in qdisc_qlen_sum()
537 static inline spinlock_t *qdisc_lock(struct Qdisc *qdisc) in qdisc_lock() argument
539 return &qdisc->q.lock; in qdisc_lock()
542 static inline struct Qdisc *qdisc_root(const struct Qdisc *qdisc) in qdisc_root() argument
544 struct Qdisc *q = rcu_dereference_rtnl(qdisc->dev_queue->qdisc); in qdisc_root()
549 static inline struct Qdisc *qdisc_root_bh(const struct Qdisc *qdisc) in qdisc_root_bh() argument
551 return rcu_dereference_bh(qdisc->dev_queue->qdisc); in qdisc_root_bh()
554 static inline struct Qdisc *qdisc_root_sleeping(const struct Qdisc *qdisc) in qdisc_root_sleeping() argument
556 return rcu_dereference_rtnl(qdisc->dev_queue->qdisc_sleeping); in qdisc_root_sleeping()
559 static inline spinlock_t *qdisc_root_sleeping_lock(const struct Qdisc *qdisc) in qdisc_root_sleeping_lock() argument
561 struct Qdisc *root = qdisc_root_sleeping(qdisc); in qdisc_root_sleeping_lock()
567 static inline struct net_device *qdisc_dev(const struct Qdisc *qdisc) in qdisc_dev() argument
569 return qdisc->dev_queue->dev; in qdisc_dev()
572 static inline void sch_tree_lock(struct Qdisc *q) in sch_tree_lock()
580 static inline void sch_tree_unlock(struct Qdisc *q) in sch_tree_unlock()
588 extern struct Qdisc noop_qdisc;
649 WARN(1, "Qdisc class overflow"); in qdisc_class_get()
659 WARN(1, "Qdisc class underflow"); in qdisc_class_put()
676 void qdisc_class_hash_grow(struct Qdisc *, struct Qdisc_class_hash *);
687 struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
688 struct Qdisc *qdisc);
689 void qdisc_reset(struct Qdisc *qdisc);
690 void qdisc_destroy(struct Qdisc *qdisc);
691 void qdisc_put(struct Qdisc *qdisc);
692 void qdisc_put_unlocked(struct Qdisc *qdisc);
693 void qdisc_tree_reduce_backlog(struct Qdisc *qdisc, int n, int len);
695 int qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type,
697 void qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch,
698 struct Qdisc *new, struct Qdisc *old,
703 qdisc_offload_dump_helper(struct Qdisc *q, enum tc_setup_type type, in qdisc_offload_dump_helper()
711 qdisc_offload_graft_helper(struct net_device *dev, struct Qdisc *sch, in qdisc_offload_graft_helper()
712 struct Qdisc *new, struct Qdisc *old, in qdisc_offload_graft_helper()
721 struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
724 void qdisc_free(struct Qdisc *qdisc);
725 struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
755 struct Qdisc *qdisc; in qdisc_reset_all_tx_gt() local
758 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc); in qdisc_reset_all_tx_gt()
759 if (qdisc) { in qdisc_reset_all_tx_gt()
760 spin_lock_bh(qdisc_lock(qdisc)); in qdisc_reset_all_tx_gt()
761 qdisc_reset(qdisc); in qdisc_reset_all_tx_gt()
762 spin_unlock_bh(qdisc_lock(qdisc)); in qdisc_reset_all_tx_gt()
775 const struct Qdisc *q = rcu_dereference(txq->qdisc); in qdisc_all_tx_empty()
794 if (rcu_access_pointer(txq->qdisc) != in qdisc_tx_changing()
801 /* Is the device using the noop qdisc on all queues? */
808 if (rcu_access_pointer(txq->qdisc) != &noop_qdisc) in qdisc_tx_is_noop()
819 /* additional qdisc xmit flags (NET_XMIT_MASK in linux/netdevice.h) */
832 const struct Qdisc *sch) in qdisc_calculate_pkt_len()
842 static inline int qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch, in qdisc_enqueue()
866 static inline void qdisc_bstats_cpu_update(struct Qdisc *sch, in qdisc_bstats_cpu_update()
872 static inline void qdisc_bstats_update(struct Qdisc *sch, in qdisc_bstats_update()
878 static inline void qdisc_qstats_backlog_dec(struct Qdisc *sch, in qdisc_qstats_backlog_dec()
884 static inline void qdisc_qstats_cpu_backlog_dec(struct Qdisc *sch, in qdisc_qstats_cpu_backlog_dec()
890 static inline void qdisc_qstats_backlog_inc(struct Qdisc *sch, in qdisc_qstats_backlog_inc()
896 static inline void qdisc_qstats_cpu_backlog_inc(struct Qdisc *sch, in qdisc_qstats_cpu_backlog_inc()
902 static inline void qdisc_qstats_cpu_qlen_inc(struct Qdisc *sch) in qdisc_qstats_cpu_qlen_inc()
907 static inline void qdisc_qstats_cpu_qlen_dec(struct Qdisc *sch) in qdisc_qstats_cpu_qlen_dec()
912 static inline void qdisc_qstats_cpu_requeues_inc(struct Qdisc *sch) in qdisc_qstats_cpu_requeues_inc()
917 static inline void __qdisc_qstats_drop(struct Qdisc *sch, int count) in __qdisc_qstats_drop()
932 static inline void qdisc_qstats_drop(struct Qdisc *sch) in qdisc_qstats_drop()
937 static inline void qdisc_qstats_cpu_drop(struct Qdisc *sch) in qdisc_qstats_cpu_drop()
942 static inline void qdisc_qstats_overlimit(struct Qdisc *sch) in qdisc_qstats_overlimit()
947 static inline int qdisc_qstats_copy(struct gnet_dump *d, struct Qdisc *sch) in qdisc_qstats_copy()
954 static inline void qdisc_qstats_qlen_backlog(struct Qdisc *sch, __u32 *qlen, in qdisc_qstats_qlen_backlog()
964 static inline void qdisc_tree_flush_backlog(struct Qdisc *sch) in qdisc_tree_flush_backlog()
972 static inline void qdisc_purge_queue(struct Qdisc *sch) in qdisc_purge_queue()
997 static inline int qdisc_enqueue_tail(struct sk_buff *skb, struct Qdisc *sch) in qdisc_enqueue_tail()
1030 static inline struct sk_buff *qdisc_dequeue_head(struct Qdisc *sch) in qdisc_dequeue_head()
1073 /* Instead of calling kfree_skb() while root qdisc lock is held,
1092 static inline unsigned int __qdisc_queue_drop_head(struct Qdisc *sch, in __qdisc_queue_drop_head()
1109 static inline struct sk_buff *qdisc_peek_head(struct Qdisc *sch) in qdisc_peek_head()
1116 /* generic pseudo peek method for non-work-conserving qdisc */
1117 static inline struct sk_buff *qdisc_peek_dequeued(struct Qdisc *sch) in qdisc_peek_dequeued()
1136 static inline void qdisc_update_stats_at_dequeue(struct Qdisc *sch, in qdisc_update_stats_at_dequeue()
1150 static inline void qdisc_update_stats_at_enqueue(struct Qdisc *sch, in qdisc_update_stats_at_enqueue()
1162 /* use instead of qdisc->dequeue() for all qdiscs queried with ->peek() */
1163 static inline struct sk_buff *qdisc_dequeue_peeked(struct Qdisc *sch) in qdisc_dequeue_peeked()
1199 static inline void qdisc_reset_queue(struct Qdisc *sch) in qdisc_reset_queue()
1204 static inline struct Qdisc *qdisc_replace(struct Qdisc *sch, struct Qdisc *new, in qdisc_replace()
1205 struct Qdisc **pold) in qdisc_replace()
1207 struct Qdisc *old; in qdisc_replace()
1219 static inline void rtnl_qdisc_drop(struct sk_buff *skb, struct Qdisc *sch) in rtnl_qdisc_drop()
1225 static inline int qdisc_drop_cpu(struct sk_buff *skb, struct Qdisc *sch, in qdisc_drop_cpu()
1234 static inline int qdisc_drop(struct sk_buff *skb, struct Qdisc *sch, in qdisc_drop()
1243 static inline int qdisc_drop_all(struct sk_buff *skb, struct Qdisc *sch, in qdisc_drop_all()
1285 * Qdisc using 64bit rate should add new attributes in psched_ratecfg_getrate()
1309 /* Mini Qdisc serves for specific needs of ingress/clsact Qdisc.
1339 void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1344 void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx);
1348 /* Make sure qdisc is no longer in SCHED state. */
1349 static inline void qdisc_synchronize(const struct Qdisc *q) in qdisc_synchronize()