LCOV - code coverage report
Current view: top level - net/sched - sch_generic.c (source / functions) Hit Total Coverage
Test: landlock.info Lines: 212 659 32.2 %
Date: 2021-04-22 12:43:58 Functions: 21 57 36.8 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0-or-later
       2             : /*
       3             :  * net/sched/sch_generic.c      Generic packet scheduler routines.
       4             :  *
       5             :  * Authors:     Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
       6             :  *              Jamal Hadi Salim, <hadi@cyberus.ca> 990601
       7             :  *              - Ingress support
       8             :  */
       9             : 
      10             : #include <linux/bitops.h>
      11             : #include <linux/module.h>
      12             : #include <linux/types.h>
      13             : #include <linux/kernel.h>
      14             : #include <linux/sched.h>
      15             : #include <linux/string.h>
      16             : #include <linux/errno.h>
      17             : #include <linux/netdevice.h>
      18             : #include <linux/skbuff.h>
      19             : #include <linux/rtnetlink.h>
      20             : #include <linux/init.h>
      21             : #include <linux/rcupdate.h>
      22             : #include <linux/list.h>
      23             : #include <linux/slab.h>
      24             : #include <linux/if_vlan.h>
      25             : #include <linux/skb_array.h>
      26             : #include <linux/if_macvlan.h>
      27             : #include <net/sch_generic.h>
      28             : #include <net/pkt_sched.h>
      29             : #include <net/dst.h>
      30             : #include <trace/events/qdisc.h>
      31             : #include <trace/events/net.h>
      32             : #include <net/xfrm.h>
      33             : 
      34             : /* Qdisc to use by default */
      35             : const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops;
      36             : EXPORT_SYMBOL(default_qdisc_ops);
      37             : 
      38             : /* Main transmission queue. */
      39             : 
      40             : /* Modifications to data participating in scheduling must be protected with
      41             :  * qdisc_lock(qdisc) spinlock.
      42             :  *
      43             :  * The idea is the following:
      44             :  * - enqueue, dequeue are serialized via qdisc root lock
      45             :  * - ingress filtering is also serialized via qdisc root lock
      46             :  * - updates to tree and tree walking are only done under the rtnl mutex.
      47             :  */
      48             : 
      49             : #define SKB_XOFF_MAGIC ((struct sk_buff *)1UL)
      50             : 
      51           0 : static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
      52             : {
      53           0 :         const struct netdev_queue *txq = q->dev_queue;
      54           0 :         spinlock_t *lock = NULL;
      55           0 :         struct sk_buff *skb;
      56             : 
      57           0 :         if (q->flags & TCQ_F_NOLOCK) {
      58           0 :                 lock = qdisc_lock(q);
      59           0 :                 spin_lock(lock);
      60             :         }
      61             : 
      62           0 :         skb = skb_peek(&q->skb_bad_txq);
      63           0 :         if (skb) {
      64             :                 /* check the reason of requeuing without tx lock first */
      65           0 :                 txq = skb_get_tx_queue(txq->dev, skb);
      66           0 :                 if (!netif_xmit_frozen_or_stopped(txq)) {
      67           0 :                         skb = __skb_dequeue(&q->skb_bad_txq);
      68           0 :                         if (qdisc_is_percpu_stats(q)) {
      69           0 :                                 qdisc_qstats_cpu_backlog_dec(q, skb);
      70           0 :                                 qdisc_qstats_cpu_qlen_dec(q);
      71             :                         } else {
      72           0 :                                 qdisc_qstats_backlog_dec(q, skb);
      73           0 :                                 q->q.qlen--;
      74             :                         }
      75             :                 } else {
      76             :                         skb = SKB_XOFF_MAGIC;
      77             :                 }
      78             :         }
      79             : 
      80           0 :         if (lock)
      81           0 :                 spin_unlock(lock);
      82             : 
      83           0 :         return skb;
      84             : }
      85             : 
      86         896 : static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q)
      87             : {
      88         896 :         struct sk_buff *skb = skb_peek(&q->skb_bad_txq);
      89             : 
      90         896 :         if (unlikely(skb))
      91           0 :                 skb = __skb_dequeue_bad_txq(q);
      92             : 
      93         896 :         return skb;
      94             : }
      95             : 
      96           0 : static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
      97             :                                              struct sk_buff *skb)
      98             : {
      99           0 :         spinlock_t *lock = NULL;
     100             : 
     101           0 :         if (q->flags & TCQ_F_NOLOCK) {
     102           0 :                 lock = qdisc_lock(q);
     103           0 :                 spin_lock(lock);
     104             :         }
     105             : 
     106           0 :         __skb_queue_tail(&q->skb_bad_txq, skb);
     107             : 
     108           0 :         if (qdisc_is_percpu_stats(q)) {
     109           0 :                 qdisc_qstats_cpu_backlog_inc(q, skb);
     110           0 :                 qdisc_qstats_cpu_qlen_inc(q);
     111             :         } else {
     112           0 :                 qdisc_qstats_backlog_inc(q, skb);
     113           0 :                 q->q.qlen++;
     114             :         }
     115             : 
     116           0 :         if (lock)
     117           0 :                 spin_unlock(lock);
     118           0 : }
     119             : 
     120           0 : static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
     121             : {
     122           0 :         spinlock_t *lock = NULL;
     123             : 
     124           0 :         if (q->flags & TCQ_F_NOLOCK) {
     125           0 :                 lock = qdisc_lock(q);
     126           0 :                 spin_lock(lock);
     127             :         }
     128             : 
     129           0 :         while (skb) {
     130           0 :                 struct sk_buff *next = skb->next;
     131             : 
     132           0 :                 __skb_queue_tail(&q->gso_skb, skb);
     133             : 
     134             :                 /* it's still part of the queue */
     135           0 :                 if (qdisc_is_percpu_stats(q)) {
     136           0 :                         qdisc_qstats_cpu_requeues_inc(q);
     137           0 :                         qdisc_qstats_cpu_backlog_inc(q, skb);
     138           0 :                         qdisc_qstats_cpu_qlen_inc(q);
     139             :                 } else {
     140           0 :                         q->qstats.requeues++;
     141           0 :                         qdisc_qstats_backlog_inc(q, skb);
     142           0 :                         q->q.qlen++;
     143             :                 }
     144             : 
     145             :                 skb = next;
     146             :         }
     147           0 :         if (lock)
     148           0 :                 spin_unlock(lock);
     149           0 :         __netif_schedule(q);
     150           0 : }
     151             : 
     152         448 : static void try_bulk_dequeue_skb(struct Qdisc *q,
     153             :                                  struct sk_buff *skb,
     154             :                                  const struct netdev_queue *txq,
     155             :                                  int *packets)
     156             : {
     157         448 :         int bytelimit = qdisc_avail_bulklimit(txq) - skb->len;
     158             : 
     159         448 :         while (bytelimit > 0) {
     160           0 :                 struct sk_buff *nskb = q->dequeue(q);
     161             : 
     162           0 :                 if (!nskb)
     163             :                         break;
     164             : 
     165           0 :                 bytelimit -= nskb->len; /* covers GSO len */
     166           0 :                 skb->next = nskb;
     167           0 :                 skb = nskb;
     168           0 :                 (*packets)++; /* GSO counts as one pkt */
     169             :         }
     170         448 :         skb_mark_not_on_list(skb);
     171         448 : }
     172             : 
     173             : /* This variant of try_bulk_dequeue_skb() makes sure
     174             :  * all skbs in the chain are for the same txq
     175             :  */
     176           0 : static void try_bulk_dequeue_skb_slow(struct Qdisc *q,
     177             :                                       struct sk_buff *skb,
     178             :                                       int *packets)
     179             : {
     180           0 :         int mapping = skb_get_queue_mapping(skb);
     181           0 :         struct sk_buff *nskb;
     182           0 :         int cnt = 0;
     183             : 
     184           0 :         do {
     185           0 :                 nskb = q->dequeue(q);
     186           0 :                 if (!nskb)
     187             :                         break;
     188           0 :                 if (unlikely(skb_get_queue_mapping(nskb) != mapping)) {
     189           0 :                         qdisc_enqueue_skb_bad_txq(q, nskb);
     190           0 :                         break;
     191             :                 }
     192           0 :                 skb->next = nskb;
     193           0 :                 skb = nskb;
     194           0 :         } while (++cnt < 8);
     195           0 :         (*packets) += cnt;
     196           0 :         skb_mark_not_on_list(skb);
     197           0 : }
     198             : 
     199             : /* Note that dequeue_skb can possibly return a SKB list (via skb->next).
     200             :  * A requeued skb (via q->gso_skb) can also be a SKB list.
     201             :  */
     202         896 : static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
     203             :                                    int *packets)
     204             : {
     205         896 :         const struct netdev_queue *txq = q->dev_queue;
     206         896 :         struct sk_buff *skb = NULL;
     207             : 
     208         896 :         *packets = 1;
     209         896 :         if (unlikely(!skb_queue_empty(&q->gso_skb))) {
     210           0 :                 spinlock_t *lock = NULL;
     211             : 
     212           0 :                 if (q->flags & TCQ_F_NOLOCK) {
     213           0 :                         lock = qdisc_lock(q);
     214           0 :                         spin_lock(lock);
     215             :                 }
     216             : 
     217           0 :                 skb = skb_peek(&q->gso_skb);
     218             : 
     219             :                 /* skb may be null if another cpu pulls gso_skb off in between
     220             :                  * empty check and lock.
     221             :                  */
     222           0 :                 if (!skb) {
     223           0 :                         if (lock)
     224           0 :                                 spin_unlock(lock);
     225           0 :                         goto validate;
     226             :                 }
     227             : 
     228             :                 /* skb in gso_skb were already validated */
     229           0 :                 *validate = false;
     230           0 :                 if (xfrm_offload(skb))
     231             :                         *validate = true;
     232             :                 /* check the reason of requeuing without tx lock first */
     233           0 :                 txq = skb_get_tx_queue(txq->dev, skb);
     234           0 :                 if (!netif_xmit_frozen_or_stopped(txq)) {
     235           0 :                         skb = __skb_dequeue(&q->gso_skb);
     236           0 :                         if (qdisc_is_percpu_stats(q)) {
     237           0 :                                 qdisc_qstats_cpu_backlog_dec(q, skb);
     238           0 :                                 qdisc_qstats_cpu_qlen_dec(q);
     239             :                         } else {
     240           0 :                                 qdisc_qstats_backlog_dec(q, skb);
     241           0 :                                 q->q.qlen--;
     242             :                         }
     243             :                 } else {
     244             :                         skb = NULL;
     245             :                 }
     246           0 :                 if (lock)
     247           0 :                         spin_unlock(lock);
     248           0 :                 goto trace;
     249             :         }
     250         896 : validate:
     251         896 :         *validate = true;
     252             : 
     253         896 :         if ((q->flags & TCQ_F_ONETXQUEUE) &&
     254         896 :             netif_xmit_frozen_or_stopped(txq))
     255             :                 return skb;
     256             : 
     257         896 :         skb = qdisc_dequeue_skb_bad_txq(q);
     258         896 :         if (unlikely(skb)) {
     259           0 :                 if (skb == SKB_XOFF_MAGIC)
     260             :                         return NULL;
     261           0 :                 goto bulk;
     262             :         }
     263         896 :         skb = q->dequeue(q);
     264         896 :         if (skb) {
     265         448 : bulk:
     266         448 :                 if (qdisc_may_bulk(q))
     267         448 :                         try_bulk_dequeue_skb(q, skb, txq, packets);
     268             :                 else
     269           0 :                         try_bulk_dequeue_skb_slow(q, skb, packets);
     270             :         }
     271         448 : trace:
     272         896 :         trace_qdisc_dequeue(q, txq, *packets, skb);
     273         896 :         return skb;
     274             : }
     275             : 
     276             : /*
     277             :  * Transmit possibly several skbs, and handle the return status as
     278             :  * required. Owning running seqcount bit guarantees that
     279             :  * only one CPU can execute this function.
     280             :  *
     281             :  * Returns to the caller:
     282             :  *                              false  - hardware queue frozen backoff
     283             :  *                              true   - feel free to send more pkts
     284             :  */
     285         448 : bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
     286             :                      struct net_device *dev, struct netdev_queue *txq,
     287             :                      spinlock_t *root_lock, bool validate)
     288             : {
     289         448 :         int ret = NETDEV_TX_BUSY;
     290         448 :         bool again = false;
     291             : 
     292             :         /* And release qdisc */
     293         448 :         if (root_lock)
     294           0 :                 spin_unlock(root_lock);
     295             : 
     296             :         /* Note that we validate skb (GSO, checksum, ...) outside of locks */
     297         448 :         if (validate)
     298         448 :                 skb = validate_xmit_skb_list(skb, dev, &again);
     299             : 
     300             : #ifdef CONFIG_XFRM_OFFLOAD
     301             :         if (unlikely(again)) {
     302             :                 if (root_lock)
     303             :                         spin_lock(root_lock);
     304             : 
     305             :                 dev_requeue_skb(skb, q);
     306             :                 return false;
     307             :         }
     308             : #endif
     309             : 
     310         448 :         if (likely(skb)) {
     311         448 :                 HARD_TX_LOCK(dev, txq, smp_processor_id());
     312         448 :                 if (!netif_xmit_frozen_or_stopped(txq))
     313         448 :                         skb = dev_hard_start_xmit(skb, dev, txq, &ret);
     314             : 
     315         448 :                 HARD_TX_UNLOCK(dev, txq);
     316             :         } else {
     317           0 :                 if (root_lock)
     318           0 :                         spin_lock(root_lock);
     319           0 :                 return true;
     320             :         }
     321             : 
     322         448 :         if (root_lock)
     323           0 :                 spin_lock(root_lock);
     324             : 
     325         448 :         if (!dev_xmit_complete(ret)) {
     326             :                 /* Driver returned NETDEV_TX_BUSY - requeue skb */
     327           0 :                 if (unlikely(ret != NETDEV_TX_BUSY))
     328           0 :                         net_warn_ratelimited("BUG %s code %d qlen %d\n",
     329             :                                              dev->name, ret, q->q.qlen);
     330             : 
     331           0 :                 dev_requeue_skb(skb, q);
     332           0 :                 return false;
     333             :         }
     334             : 
     335             :         return true;
     336             : }
     337             : 
     338             : /*
     339             :  * NOTE: Called under qdisc_lock(q) with locally disabled BH.
     340             :  *
     341             :  * running seqcount guarantees only one CPU can process
     342             :  * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
     343             :  * this queue.
     344             :  *
     345             :  *  netif_tx_lock serializes accesses to device driver.
     346             :  *
     347             :  *  qdisc_lock(q) and netif_tx_lock are mutually exclusive,
     348             :  *  if one is grabbed, another must be free.
     349             :  *
     350             :  * Note, that this procedure can be called by a watchdog timer
     351             :  *
     352             :  * Returns to the caller:
     353             :  *                              0  - queue is empty or throttled.
     354             :  *                              >0 - queue is not empty.
     355             :  *
     356             :  */
     357         896 : static inline bool qdisc_restart(struct Qdisc *q, int *packets)
     358             : {
     359         896 :         spinlock_t *root_lock = NULL;
     360         896 :         struct netdev_queue *txq;
     361         896 :         struct net_device *dev;
     362         896 :         struct sk_buff *skb;
     363         896 :         bool validate;
     364             : 
     365             :         /* Dequeue packet */
     366         896 :         skb = dequeue_skb(q, &validate, packets);
     367         896 :         if (unlikely(!skb))
     368             :                 return false;
     369             : 
     370         448 :         if (!(q->flags & TCQ_F_NOLOCK))
     371           0 :                 root_lock = qdisc_lock(q);
     372             : 
     373         448 :         dev = qdisc_dev(q);
     374         448 :         txq = skb_get_tx_queue(dev, skb);
     375             : 
     376         448 :         return sch_direct_xmit(skb, q, dev, txq, root_lock, validate);
     377             : }
     378             : 
     379         448 : void __qdisc_run(struct Qdisc *q)
     380             : {
     381         448 :         int quota = dev_tx_weight;
     382         448 :         int packets;
     383             : 
     384         896 :         while (qdisc_restart(q, &packets)) {
     385         448 :                 quota -= packets;
     386         448 :                 if (quota <= 0) {
     387           0 :                         __netif_schedule(q);
     388           0 :                         break;
     389             :                 }
     390             :         }
     391         448 : }
     392             : 
     393           0 : unsigned long dev_trans_start(struct net_device *dev)
     394             : {
     395           0 :         unsigned long val, res;
     396           0 :         unsigned int i;
     397             : 
     398           0 :         if (is_vlan_dev(dev))
     399           0 :                 dev = vlan_dev_real_dev(dev);
     400           0 :         else if (netif_is_macvlan(dev))
     401           0 :                 dev = macvlan_dev_real_dev(dev);
     402           0 :         res = netdev_get_tx_queue(dev, 0)->trans_start;
     403           0 :         for (i = 1; i < dev->num_tx_queues; i++) {
     404           0 :                 val = netdev_get_tx_queue(dev, i)->trans_start;
     405           0 :                 if (val && time_after(val, res))
     406           0 :                         res = val;
     407             :         }
     408             : 
     409           0 :         return res;
     410             : }
     411             : EXPORT_SYMBOL(dev_trans_start);
     412             : 
     413           0 : static void dev_watchdog(struct timer_list *t)
     414             : {
     415           0 :         struct net_device *dev = from_timer(dev, t, watchdog_timer);
     416             : 
     417           0 :         netif_tx_lock(dev);
     418           0 :         if (!qdisc_tx_is_noop(dev)) {
     419           0 :                 if (netif_device_present(dev) &&
     420           0 :                     netif_running(dev) &&
     421           0 :                     netif_carrier_ok(dev)) {
     422           0 :                         int some_queue_timedout = 0;
     423             :                         unsigned int i;
     424             :                         unsigned long trans_start;
     425             : 
     426           0 :                         for (i = 0; i < dev->num_tx_queues; i++) {
     427           0 :                                 struct netdev_queue *txq;
     428             : 
     429           0 :                                 txq = netdev_get_tx_queue(dev, i);
     430           0 :                                 trans_start = txq->trans_start;
     431           0 :                                 if (netif_xmit_stopped(txq) &&
     432           0 :                                     time_after(jiffies, (trans_start +
     433             :                                                          dev->watchdog_timeo))) {
     434           0 :                                         some_queue_timedout = 1;
     435           0 :                                         txq->trans_timeout++;
     436           0 :                                         break;
     437             :                                 }
     438             :                         }
     439             : 
     440           0 :                         if (some_queue_timedout) {
     441           0 :                                 trace_net_dev_xmit_timeout(dev, i);
     442           0 :                                 WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n",
     443             :                                        dev->name, netdev_drivername(dev), i);
     444           0 :                                 dev->netdev_ops->ndo_tx_timeout(dev, i);
     445             :                         }
     446           0 :                         if (!mod_timer(&dev->watchdog_timer,
     447             :                                        round_jiffies(jiffies +
     448           0 :                                                      dev->watchdog_timeo)))
     449           0 :                                 dev_hold(dev);
     450             :                 }
     451             :         }
     452           0 :         netif_tx_unlock(dev);
     453             : 
     454           0 :         dev_put(dev);
     455           0 : }
     456             : 
     457           3 : void __netdev_watchdog_up(struct net_device *dev)
     458             : {
     459           3 :         if (dev->netdev_ops->ndo_tx_timeout) {
     460           0 :                 if (dev->watchdog_timeo <= 0)
     461           0 :                         dev->watchdog_timeo = 5*HZ;
     462           0 :                 if (!mod_timer(&dev->watchdog_timer,
     463           0 :                                round_jiffies(jiffies + dev->watchdog_timeo)))
     464           0 :                         dev_hold(dev);
     465             :         }
     466           3 : }
     467             : EXPORT_SYMBOL_GPL(__netdev_watchdog_up);
     468             : 
     469           3 : static void dev_watchdog_up(struct net_device *dev)
     470             : {
     471           3 :         __netdev_watchdog_up(dev);
     472           3 : }
     473             : 
     474           0 : static void dev_watchdog_down(struct net_device *dev)
     475             : {
     476           0 :         netif_tx_lock_bh(dev);
     477           0 :         if (del_timer(&dev->watchdog_timer))
     478           0 :                 dev_put(dev);
     479           0 :         netif_tx_unlock_bh(dev);
     480           0 : }
     481             : 
     482             : /**
     483             :  *      netif_carrier_on - set carrier
     484             :  *      @dev: network device
     485             :  *
     486             :  * Device has detected acquisition of carrier.
     487             :  */
     488           1 : void netif_carrier_on(struct net_device *dev)
     489             : {
     490           1 :         if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
     491           1 :                 if (dev->reg_state == NETREG_UNINITIALIZED)
     492             :                         return;
     493           1 :                 atomic_inc(&dev->carrier_up_count);
     494           1 :                 linkwatch_fire_event(dev);
     495           1 :                 if (netif_running(dev))
     496           0 :                         __netdev_watchdog_up(dev);
     497             :         }
     498             : }
     499             : EXPORT_SYMBOL(netif_carrier_on);
     500             : 
     501             : /**
     502             :  *      netif_carrier_off - clear carrier
     503             :  *      @dev: network device
     504             :  *
     505             :  * Device has detected loss of carrier.
     506             :  */
     507           1 : void netif_carrier_off(struct net_device *dev)
     508             : {
     509           1 :         if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
     510           1 :                 if (dev->reg_state == NETREG_UNINITIALIZED)
     511             :                         return;
     512           1 :                 atomic_inc(&dev->carrier_down_count);
     513           1 :                 linkwatch_fire_event(dev);
     514             :         }
     515             : }
     516             : EXPORT_SYMBOL(netif_carrier_off);
     517             : 
     518             : /* "NOOP" scheduler: the best scheduler, recommended for all interfaces
     519             :    under all circumstances. It is difficult to invent anything faster or
     520             :    cheaper.
     521             :  */
     522             : 
     523           0 : static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
     524             :                         struct sk_buff **to_free)
     525             : {
     526           0 :         __qdisc_drop(skb, to_free);
     527           0 :         return NET_XMIT_CN;
     528             : }
     529             : 
     530           0 : static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
     531             : {
     532           0 :         return NULL;
     533             : }
     534             : 
     535             : struct Qdisc_ops noop_qdisc_ops __read_mostly = {
     536             :         .id             =       "noop",
     537             :         .priv_size      =       0,
     538             :         .enqueue        =       noop_enqueue,
     539             :         .dequeue        =       noop_dequeue,
     540             :         .peek           =       noop_dequeue,
     541             :         .owner          =       THIS_MODULE,
     542             : };
     543             : 
     544             : static struct netdev_queue noop_netdev_queue = {
     545             :         RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
     546             :         .qdisc_sleeping =       &noop_qdisc,
     547             : };
     548             : 
     549             : struct Qdisc noop_qdisc = {
     550             :         .enqueue        =       noop_enqueue,
     551             :         .dequeue        =       noop_dequeue,
     552             :         .flags          =       TCQ_F_BUILTIN,
     553             :         .ops            =       &noop_qdisc_ops,
     554             :         .q.lock         =       __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
     555             :         .dev_queue      =       &noop_netdev_queue,
     556             :         .running        =       SEQCNT_ZERO(noop_qdisc.running),
     557             :         .busylock       =       __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
     558             :         .gso_skb = {
     559             :                 .next = (struct sk_buff *)&noop_qdisc.gso_skb,
     560             :                 .prev = (struct sk_buff *)&noop_qdisc.gso_skb,
     561             :                 .qlen = 0,
     562             :                 .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.gso_skb.lock),
     563             :         },
     564             :         .skb_bad_txq = {
     565             :                 .next = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
     566             :                 .prev = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
     567             :                 .qlen = 0,
     568             :                 .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.skb_bad_txq.lock),
     569             :         },
     570             : };
     571             : EXPORT_SYMBOL(noop_qdisc);
     572             : 
     573           2 : static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
     574             :                         struct netlink_ext_ack *extack)
     575             : {
     576             :         /* register_qdisc() assigns a default of noop_enqueue if unset,
     577             :          * but __dev_queue_xmit() treats noqueue only as such
     578             :          * if this is NULL - so clear it here. */
     579           2 :         qdisc->enqueue = NULL;
     580           2 :         return 0;
     581             : }
     582             : 
     583             : struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
     584             :         .id             =       "noqueue",
     585             :         .priv_size      =       0,
     586             :         .init           =       noqueue_init,
     587             :         .enqueue        =       noop_enqueue,
     588             :         .dequeue        =       noop_dequeue,
     589             :         .peek           =       noop_dequeue,
     590             :         .owner          =       THIS_MODULE,
     591             : };
     592             : 
     593             : static const u8 prio2band[TC_PRIO_MAX + 1] = {
     594             :         1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1
     595             : };
     596             : 
     597             : /* 3-band FIFO queue: old style, but should be a bit faster than
     598             :    generic prio+fifo combination.
     599             :  */
     600             : 
     601             : #define PFIFO_FAST_BANDS 3
     602             : 
     603             : /*
     604             :  * Private data for a pfifo_fast scheduler containing:
     605             :  *      - rings for priority bands
     606             :  */
     607             : struct pfifo_fast_priv {
     608             :         struct skb_array q[PFIFO_FAST_BANDS];
     609             : };
     610             : 
     611        3042 : static inline struct skb_array *band2list(struct pfifo_fast_priv *priv,
     612             :                                           int band)
     613             : {
     614        3042 :         return &priv->q[band];
     615             : }
     616             : 
     617         448 : static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
     618             :                               struct sk_buff **to_free)
     619             : {
     620         448 :         int band = prio2band[skb->priority & TC_PRIO_MAX];
     621         448 :         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
     622         448 :         struct skb_array *q = band2list(priv, band);
     623         448 :         unsigned int pkt_len = qdisc_pkt_len(skb);
     624         448 :         int err;
     625             : 
     626         448 :         err = skb_array_produce(q, skb);
     627             : 
     628         448 :         if (unlikely(err)) {
     629           0 :                 if (qdisc_is_percpu_stats(qdisc))
     630           0 :                         return qdisc_drop_cpu(skb, qdisc, to_free);
     631             :                 else
     632           0 :                         return qdisc_drop(skb, qdisc, to_free);
     633             :         }
     634             : 
     635         448 :         qdisc_update_stats_at_enqueue(qdisc, pkt_len);
     636         448 :         return NET_XMIT_SUCCESS;
     637             : }
     638             : 
     639         896 : static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
     640             : {
     641         896 :         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
     642         896 :         struct sk_buff *skb = NULL;
     643         896 :         int band;
     644             : 
     645        3487 :         for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
     646        2591 :                 struct skb_array *q = band2list(priv, band);
     647             : 
     648        2591 :                 if (__skb_array_empty(q))
     649        2143 :                         continue;
     650             : 
     651         448 :                 skb = __skb_array_consume(q);
     652             :         }
     653         896 :         if (likely(skb)) {
     654         448 :                 qdisc_update_stats_at_dequeue(qdisc, skb);
     655             :         } else {
     656         448 :                 WRITE_ONCE(qdisc->empty, true);
     657             :         }
     658             : 
     659         896 :         return skb;
     660             : }
     661             : 
     662           0 : static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
     663             : {
     664           0 :         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
     665           0 :         struct sk_buff *skb = NULL;
     666           0 :         int band;
     667             : 
     668           0 :         for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
     669           0 :                 struct skb_array *q = band2list(priv, band);
     670             : 
     671           0 :                 skb = __skb_array_peek(q);
     672             :         }
     673             : 
     674           0 :         return skb;
     675             : }
     676             : 
     677           0 : static void pfifo_fast_reset(struct Qdisc *qdisc)
     678             : {
     679           0 :         int i, band;
     680           0 :         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
     681             : 
     682           0 :         for (band = 0; band < PFIFO_FAST_BANDS; band++) {
     683           0 :                 struct skb_array *q = band2list(priv, band);
     684           0 :                 struct sk_buff *skb;
     685             : 
     686             :                 /* NULL ring is possible if destroy path is due to a failed
     687             :                  * skb_array_init() in pfifo_fast_init() case.
     688             :                  */
     689           0 :                 if (!q->ring.queue)
     690           0 :                         continue;
     691             : 
     692           0 :                 while ((skb = __skb_array_consume(q)) != NULL)
     693           0 :                         kfree_skb(skb);
     694             :         }
     695             : 
     696           0 :         if (qdisc_is_percpu_stats(qdisc)) {
     697           0 :                 for_each_possible_cpu(i) {
     698           0 :                         struct gnet_stats_queue *q;
     699             : 
     700           0 :                         q = per_cpu_ptr(qdisc->cpu_qstats, i);
     701           0 :                         q->backlog = 0;
     702           0 :                         q->qlen = 0;
     703             :                 }
     704             :         }
     705           0 : }
     706             : 
     707           0 : static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
     708             : {
     709           0 :         struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
     710             : 
     711           0 :         memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1);
     712           0 :         if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
     713           0 :                 goto nla_put_failure;
     714           0 :         return skb->len;
     715             : 
     716           0 : nla_put_failure:
     717           0 :         return -1;
     718             : }
     719             : 
     720           1 : static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
     721             :                            struct netlink_ext_ack *extack)
     722             : {
     723           1 :         unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
     724           1 :         struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
     725           1 :         int prio;
     726             : 
     727             :         /* guard against zero length rings */
     728           1 :         if (!qlen)
     729             :                 return -EINVAL;
     730             : 
     731           4 :         for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
     732           3 :                 struct skb_array *q = band2list(priv, prio);
     733           3 :                 int err;
     734             : 
     735           3 :                 err = skb_array_init(q, qlen, GFP_KERNEL);
     736           3 :                 if (err)
     737             :                         return -ENOMEM;
     738             :         }
     739             : 
     740             :         /* Can by-pass the queue discipline */
     741           1 :         qdisc->flags |= TCQ_F_CAN_BYPASS;
     742           1 :         return 0;
     743             : }
     744             : 
     745           0 : static void pfifo_fast_destroy(struct Qdisc *sch)
     746             : {
     747           0 :         struct pfifo_fast_priv *priv = qdisc_priv(sch);
     748           0 :         int prio;
     749             : 
     750           0 :         for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
     751           0 :                 struct skb_array *q = band2list(priv, prio);
     752             : 
     753             :                 /* NULL ring is possible if destroy path is due to a failed
     754             :                  * skb_array_init() in pfifo_fast_init() case.
     755             :                  */
     756           0 :                 if (!q->ring.queue)
     757           0 :                         continue;
     758             :                 /* Destroy ring but no need to kfree_skb because a call to
     759             :                  * pfifo_fast_reset() has already done that work.
     760             :                  */
     761           0 :                 ptr_ring_cleanup(&q->ring, NULL);
     762             :         }
     763           0 : }
     764             : 
     765           0 : static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch,
     766             :                                           unsigned int new_len)
     767             : {
     768           0 :         struct pfifo_fast_priv *priv = qdisc_priv(sch);
     769           0 :         struct skb_array *bands[PFIFO_FAST_BANDS];
     770           0 :         int prio;
     771             : 
     772           0 :         for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
     773           0 :                 struct skb_array *q = band2list(priv, prio);
     774             : 
     775           0 :                 bands[prio] = q;
     776             :         }
     777             : 
     778           0 :         return skb_array_resize_multiple(bands, PFIFO_FAST_BANDS, new_len,
     779             :                                          GFP_KERNEL);
     780             : }
     781             : 
     782             : struct Qdisc_ops pfifo_fast_ops __read_mostly = {
     783             :         .id             =       "pfifo_fast",
     784             :         .priv_size      =       sizeof(struct pfifo_fast_priv),
     785             :         .enqueue        =       pfifo_fast_enqueue,
     786             :         .dequeue        =       pfifo_fast_dequeue,
     787             :         .peek           =       pfifo_fast_peek,
     788             :         .init           =       pfifo_fast_init,
     789             :         .destroy        =       pfifo_fast_destroy,
     790             :         .reset          =       pfifo_fast_reset,
     791             :         .dump           =       pfifo_fast_dump,
     792             :         .change_tx_queue_len =  pfifo_fast_change_tx_queue_len,
     793             :         .owner          =       THIS_MODULE,
     794             :         .static_flags   =       TCQ_F_NOLOCK | TCQ_F_CPUSTATS,
     795             : };
     796             : EXPORT_SYMBOL(pfifo_fast_ops);
     797             : 
     798             : static struct lock_class_key qdisc_tx_busylock;
     799             : static struct lock_class_key qdisc_running_key;
     800             : 
     801           3 : struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
     802             :                           const struct Qdisc_ops *ops,
     803             :                           struct netlink_ext_ack *extack)
     804             : {
     805           3 :         struct Qdisc *sch;
     806           3 :         unsigned int size = sizeof(*sch) + ops->priv_size;
     807           3 :         int err = -ENOBUFS;
     808           3 :         struct net_device *dev;
     809             : 
     810           3 :         if (!dev_queue) {
     811           0 :                 NL_SET_ERR_MSG(extack, "No device queue given");
     812           0 :                 err = -EINVAL;
     813           0 :                 goto errout;
     814             :         }
     815             : 
     816           3 :         dev = dev_queue->dev;
     817           3 :         sch = kzalloc_node(size, GFP_KERNEL, netdev_queue_numa_node_read(dev_queue));
     818             : 
     819           3 :         if (!sch)
     820           0 :                 goto errout;
     821           3 :         __skb_queue_head_init(&sch->gso_skb);
     822           3 :         __skb_queue_head_init(&sch->skb_bad_txq);
     823           3 :         qdisc_skb_head_init(&sch->q);
     824           3 :         spin_lock_init(&sch->q.lock);
     825             : 
     826           3 :         if (ops->static_flags & TCQ_F_CPUSTATS) {
     827           2 :                 sch->cpu_bstats =
     828           6 :                         netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu);
     829           1 :                 if (!sch->cpu_bstats)
     830           0 :                         goto errout1;
     831             : 
     832           1 :                 sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
     833           1 :                 if (!sch->cpu_qstats) {
     834           0 :                         free_percpu(sch->cpu_bstats);
     835           0 :                         goto errout1;
     836             :                 }
     837             :         }
     838             : 
     839           3 :         spin_lock_init(&sch->busylock);
     840           6 :         lockdep_set_class(&sch->busylock,
     841             :                           dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
     842             : 
     843             :         /* seqlock has the same scope of busylock, for NOLOCK qdisc */
     844           3 :         spin_lock_init(&sch->seqlock);
     845           6 :         lockdep_set_class(&sch->busylock,
     846             :                           dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
     847             : 
     848           3 :         seqcount_init(&sch->running);
     849           6 :         lockdep_set_class(&sch->running,
     850             :                           dev->qdisc_running_key ?: &qdisc_running_key);
     851             : 
     852           3 :         sch->ops = ops;
     853           3 :         sch->flags = ops->static_flags;
     854           3 :         sch->enqueue = ops->enqueue;
     855           3 :         sch->dequeue = ops->dequeue;
     856           3 :         sch->dev_queue = dev_queue;
     857           3 :         sch->empty = true;
     858           3 :         dev_hold(dev);
     859           3 :         refcount_set(&sch->refcnt, 1);
     860             : 
     861           3 :         return sch;
     862           0 : errout1:
     863           0 :         kfree(sch);
     864           0 : errout:
     865           0 :         return ERR_PTR(err);
     866             : }
     867             : 
     868           3 : struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
     869             :                                 const struct Qdisc_ops *ops,
     870             :                                 unsigned int parentid,
     871             :                                 struct netlink_ext_ack *extack)
     872             : {
     873           3 :         struct Qdisc *sch;
     874             : 
     875           3 :         if (!try_module_get(ops->owner)) {
     876             :                 NL_SET_ERR_MSG(extack, "Failed to increase module reference counter");
     877             :                 return NULL;
     878             :         }
     879             : 
     880           3 :         sch = qdisc_alloc(dev_queue, ops, extack);
     881           3 :         if (IS_ERR(sch)) {
     882           3 :                 module_put(ops->owner);
     883             :                 return NULL;
     884             :         }
     885           3 :         sch->parent = parentid;
     886             : 
     887           3 :         if (!ops->init || ops->init(sch, NULL, extack) == 0) {
     888           3 :                 trace_qdisc_create(ops, dev_queue->dev, parentid);
     889           3 :                 return sch;
     890             :         }
     891             : 
     892           0 :         qdisc_put(sch);
     893           0 :         return NULL;
     894             : }
     895             : EXPORT_SYMBOL(qdisc_create_dflt);
     896             : 
     897             : /* Under qdisc_lock(qdisc) and BH! */
     898             : 
     899           0 : void qdisc_reset(struct Qdisc *qdisc)
     900             : {
     901           0 :         const struct Qdisc_ops *ops = qdisc->ops;
     902           0 :         struct sk_buff *skb, *tmp;
     903             : 
     904           0 :         trace_qdisc_reset(qdisc);
     905             : 
     906           0 :         if (ops->reset)
     907           0 :                 ops->reset(qdisc);
     908             : 
     909           0 :         skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {
     910           0 :                 __skb_unlink(skb, &qdisc->gso_skb);
     911           0 :                 kfree_skb_list(skb);
     912             :         }
     913             : 
     914           0 :         skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {
     915           0 :                 __skb_unlink(skb, &qdisc->skb_bad_txq);
     916           0 :                 kfree_skb_list(skb);
     917             :         }
     918             : 
     919           0 :         qdisc->q.qlen = 0;
     920           0 :         qdisc->qstats.backlog = 0;
     921           0 : }
     922             : EXPORT_SYMBOL(qdisc_reset);
     923             : 
     924           0 : void qdisc_free(struct Qdisc *qdisc)
     925             : {
     926           0 :         if (qdisc_is_percpu_stats(qdisc)) {
     927           0 :                 free_percpu(qdisc->cpu_bstats);
     928           0 :                 free_percpu(qdisc->cpu_qstats);
     929             :         }
     930             : 
     931           0 :         kfree(qdisc);
     932           0 : }
     933             : 
     934           0 : static void qdisc_free_cb(struct rcu_head *head)
     935             : {
     936           0 :         struct Qdisc *q = container_of(head, struct Qdisc, rcu);
     937             : 
     938           0 :         qdisc_free(q);
     939           0 : }
     940             : 
     941           0 : static void qdisc_destroy(struct Qdisc *qdisc)
     942             : {
     943           0 :         const struct Qdisc_ops  *ops = qdisc->ops;
     944             : 
     945             : #ifdef CONFIG_NET_SCHED
     946             :         qdisc_hash_del(qdisc);
     947             : 
     948             :         qdisc_put_stab(rtnl_dereference(qdisc->stab));
     949             : #endif
     950           0 :         gen_kill_estimator(&qdisc->rate_est);
     951             : 
     952           0 :         qdisc_reset(qdisc);
     953             : 
     954           0 :         if (ops->destroy)
     955           0 :                 ops->destroy(qdisc);
     956             : 
     957           0 :         module_put(ops->owner);
     958           0 :         dev_put(qdisc_dev(qdisc));
     959             : 
     960           0 :         trace_qdisc_destroy(qdisc);
     961             : 
     962           0 :         call_rcu(&qdisc->rcu, qdisc_free_cb);
     963           0 : }
     964             : 
     965           0 : void qdisc_put(struct Qdisc *qdisc)
     966             : {
     967           0 :         if (!qdisc)
     968             :                 return;
     969             : 
     970           0 :         if (qdisc->flags & TCQ_F_BUILTIN ||
     971           0 :             !refcount_dec_and_test(&qdisc->refcnt))
     972           0 :                 return;
     973             : 
     974           0 :         qdisc_destroy(qdisc);
     975             : }
     976             : EXPORT_SYMBOL(qdisc_put);
     977             : 
     978             : /* Version of qdisc_put() that is called with rtnl mutex unlocked.
     979             :  * Intended to be used as optimization, this function only takes rtnl lock if
     980             :  * qdisc reference counter reached zero.
     981             :  */
     982             : 
     983           0 : void qdisc_put_unlocked(struct Qdisc *qdisc)
     984             : {
     985           0 :         if (qdisc->flags & TCQ_F_BUILTIN ||
     986           0 :             !refcount_dec_and_rtnl_lock(&qdisc->refcnt))
     987           0 :                 return;
     988             : 
     989           0 :         qdisc_destroy(qdisc);
     990           0 :         rtnl_unlock();
     991             : }
     992             : EXPORT_SYMBOL(qdisc_put_unlocked);
     993             : 
     994             : /* Attach toplevel qdisc to device queue. */
     995           0 : struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
     996             :                               struct Qdisc *qdisc)
     997             : {
     998           0 :         struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
     999           0 :         spinlock_t *root_lock;
    1000             : 
    1001           0 :         root_lock = qdisc_lock(oqdisc);
    1002           0 :         spin_lock_bh(root_lock);
    1003             : 
    1004             :         /* ... and graft new one */
    1005           0 :         if (qdisc == NULL)
    1006           0 :                 qdisc = &noop_qdisc;
    1007           0 :         dev_queue->qdisc_sleeping = qdisc;
    1008           0 :         rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
    1009             : 
    1010           0 :         spin_unlock_bh(root_lock);
    1011             : 
    1012           0 :         return oqdisc;
    1013             : }
    1014             : EXPORT_SYMBOL(dev_graft_qdisc);
    1015             : 
    1016           3 : static void attach_one_default_qdisc(struct net_device *dev,
    1017             :                                      struct netdev_queue *dev_queue,
    1018             :                                      void *_unused)
    1019             : {
    1020           3 :         struct Qdisc *qdisc;
    1021           3 :         const struct Qdisc_ops *ops = default_qdisc_ops;
    1022             : 
    1023           3 :         if (dev->priv_flags & IFF_NO_QUEUE)
    1024             :                 ops = &noqueue_qdisc_ops;
    1025           1 :         else if(dev->type == ARPHRD_CAN)
    1026           0 :                 ops = &pfifo_fast_ops;
    1027             : 
    1028           3 :         qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
    1029           3 :         if (!qdisc)
    1030             :                 return;
    1031             : 
    1032           3 :         if (!netif_is_multiqueue(dev))
    1033           3 :                 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
    1034           3 :         dev_queue->qdisc_sleeping = qdisc;
    1035             : }
    1036             : 
    1037           3 : static void attach_default_qdiscs(struct net_device *dev)
    1038             : {
    1039           3 :         struct netdev_queue *txq;
    1040           3 :         struct Qdisc *qdisc;
    1041             : 
    1042           3 :         txq = netdev_get_tx_queue(dev, 0);
    1043             : 
    1044           3 :         if (!netif_is_multiqueue(dev) ||
    1045           0 :             dev->priv_flags & IFF_NO_QUEUE) {
    1046           3 :                 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
    1047           3 :                 dev->qdisc = txq->qdisc_sleeping;
    1048           3 :                 qdisc_refcount_inc(dev->qdisc);
    1049             :         } else {
    1050           0 :                 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
    1051           0 :                 if (qdisc) {
    1052           0 :                         dev->qdisc = qdisc;
    1053           0 :                         qdisc->ops->attach(qdisc);
    1054             :                 }
    1055             :         }
    1056             : 
    1057             :         /* Detect default qdisc setup/init failed and fallback to "noqueue" */
    1058           3 :         if (dev->qdisc == &noop_qdisc) {
    1059           0 :                 netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",
    1060           0 :                             default_qdisc_ops->id, noqueue_qdisc_ops.id);
    1061           0 :                 dev->priv_flags |= IFF_NO_QUEUE;
    1062           0 :                 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
    1063           0 :                 dev->qdisc = txq->qdisc_sleeping;
    1064           0 :                 qdisc_refcount_inc(dev->qdisc);
    1065           0 :                 dev->priv_flags ^= IFF_NO_QUEUE;
    1066             :         }
    1067             : 
    1068             : #ifdef CONFIG_NET_SCHED
    1069             :         if (dev->qdisc != &noop_qdisc)
    1070             :                 qdisc_hash_add(dev->qdisc, false);
    1071             : #endif
    1072           3 : }
    1073             : 
    1074           3 : static void transition_one_qdisc(struct net_device *dev,
    1075             :                                  struct netdev_queue *dev_queue,
    1076             :                                  void *_need_watchdog)
    1077             : {
    1078           3 :         struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping;
    1079           3 :         int *need_watchdog_p = _need_watchdog;
    1080             : 
    1081           3 :         if (!(new_qdisc->flags & TCQ_F_BUILTIN))
    1082           3 :                 clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state);
    1083             : 
    1084           3 :         rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
    1085           3 :         if (need_watchdog_p) {
    1086           3 :                 dev_queue->trans_start = 0;
    1087           3 :                 *need_watchdog_p = 1;
    1088             :         }
    1089           3 : }
    1090             : 
    1091           3 : void dev_activate(struct net_device *dev)
    1092             : {
    1093           3 :         int need_watchdog;
    1094             : 
    1095             :         /* No queueing discipline is attached to device;
    1096             :          * create default one for devices, which need queueing
    1097             :          * and noqueue_qdisc for virtual interfaces
    1098             :          */
    1099             : 
    1100           3 :         if (dev->qdisc == &noop_qdisc)
    1101           3 :                 attach_default_qdiscs(dev);
    1102             : 
    1103           3 :         if (!netif_carrier_ok(dev))
    1104             :                 /* Delay activation until next carrier-on event */
    1105           0 :                 return;
    1106             : 
    1107           3 :         need_watchdog = 0;
    1108           3 :         netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
    1109           3 :         if (dev_ingress_queue(dev))
    1110           0 :                 transition_one_qdisc(dev, dev_ingress_queue(dev), NULL);
    1111             : 
    1112           3 :         if (need_watchdog) {
    1113           3 :                 netif_trans_update(dev);
    1114           6 :                 dev_watchdog_up(dev);
    1115             :         }
    1116             : }
    1117             : EXPORT_SYMBOL(dev_activate);
    1118             : 
    1119           0 : static void qdisc_deactivate(struct Qdisc *qdisc)
    1120             : {
    1121           0 :         if (qdisc->flags & TCQ_F_BUILTIN)
    1122             :                 return;
    1123             : 
    1124           0 :         set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
    1125             : }
    1126             : 
    1127           0 : static void dev_deactivate_queue(struct net_device *dev,
    1128             :                                  struct netdev_queue *dev_queue,
    1129             :                                  void *_qdisc_default)
    1130             : {
    1131           0 :         struct Qdisc *qdisc_default = _qdisc_default;
    1132           0 :         struct Qdisc *qdisc;
    1133             : 
    1134           0 :         qdisc = rtnl_dereference(dev_queue->qdisc);
    1135           0 :         if (qdisc) {
    1136           0 :                 qdisc_deactivate(qdisc);
    1137           0 :                 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
    1138             :         }
    1139           0 : }
    1140             : 
    1141           0 : static void dev_reset_queue(struct net_device *dev,
    1142             :                             struct netdev_queue *dev_queue,
    1143             :                             void *_unused)
    1144             : {
    1145           0 :         struct Qdisc *qdisc;
    1146           0 :         bool nolock;
    1147             : 
    1148           0 :         qdisc = dev_queue->qdisc_sleeping;
    1149           0 :         if (!qdisc)
    1150             :                 return;
    1151             : 
    1152           0 :         nolock = qdisc->flags & TCQ_F_NOLOCK;
    1153             : 
    1154           0 :         if (nolock)
    1155           0 :                 spin_lock_bh(&qdisc->seqlock);
    1156           0 :         spin_lock_bh(qdisc_lock(qdisc));
    1157             : 
    1158           0 :         qdisc_reset(qdisc);
    1159             : 
    1160           0 :         spin_unlock_bh(qdisc_lock(qdisc));
    1161           0 :         if (nolock)
    1162           0 :                 spin_unlock_bh(&qdisc->seqlock);
    1163             : }
    1164             : 
    1165           0 : static bool some_qdisc_is_busy(struct net_device *dev)
    1166             : {
    1167           0 :         unsigned int i;
    1168             : 
    1169           0 :         for (i = 0; i < dev->num_tx_queues; i++) {
    1170           0 :                 struct netdev_queue *dev_queue;
    1171           0 :                 spinlock_t *root_lock;
    1172           0 :                 struct Qdisc *q;
    1173           0 :                 int val;
    1174             : 
    1175           0 :                 dev_queue = netdev_get_tx_queue(dev, i);
    1176           0 :                 q = dev_queue->qdisc_sleeping;
    1177             : 
    1178           0 :                 root_lock = qdisc_lock(q);
    1179           0 :                 spin_lock_bh(root_lock);
    1180             : 
    1181           0 :                 val = (qdisc_is_running(q) ||
    1182           0 :                        test_bit(__QDISC_STATE_SCHED, &q->state));
    1183             : 
    1184           0 :                 spin_unlock_bh(root_lock);
    1185             : 
    1186           0 :                 if (val)
    1187             :                         return true;
    1188             :         }
    1189             :         return false;
    1190             : }
    1191             : 
    1192             : /**
    1193             :  *      dev_deactivate_many - deactivate transmissions on several devices
    1194             :  *      @head: list of devices to deactivate
    1195             :  *
    1196             :  *      This function returns only when all outstanding transmissions
    1197             :  *      have completed, unless all devices are in dismantle phase.
    1198             :  */
    1199           0 : void dev_deactivate_many(struct list_head *head)
    1200             : {
    1201           0 :         struct net_device *dev;
    1202             : 
    1203           0 :         list_for_each_entry(dev, head, close_list) {
    1204           0 :                 netdev_for_each_tx_queue(dev, dev_deactivate_queue,
    1205             :                                          &noop_qdisc);
    1206           0 :                 if (dev_ingress_queue(dev))
    1207           0 :                         dev_deactivate_queue(dev, dev_ingress_queue(dev),
    1208             :                                              &noop_qdisc);
    1209             : 
    1210           0 :                 dev_watchdog_down(dev);
    1211             :         }
    1212             : 
    1213             :         /* Wait for outstanding qdisc-less dev_queue_xmit calls or
    1214             :          * outstanding qdisc enqueuing calls.
    1215             :          * This is avoided if all devices are in dismantle phase :
    1216             :          * Caller will call synchronize_net() for us
    1217             :          */
    1218           0 :         synchronize_net();
    1219             : 
    1220           0 :         list_for_each_entry(dev, head, close_list) {
    1221           0 :                 netdev_for_each_tx_queue(dev, dev_reset_queue, NULL);
    1222             : 
    1223           0 :                 if (dev_ingress_queue(dev))
    1224           0 :                         dev_reset_queue(dev, dev_ingress_queue(dev), NULL);
    1225             :         }
    1226             : 
    1227             :         /* Wait for outstanding qdisc_run calls. */
    1228           0 :         list_for_each_entry(dev, head, close_list) {
    1229           0 :                 while (some_qdisc_is_busy(dev)) {
    1230             :                         /* wait_event() would avoid this sleep-loop but would
    1231             :                          * require expensive checks in the fast paths of packet
    1232             :                          * processing which isn't worth it.
    1233             :                          */
    1234           0 :                         schedule_timeout_uninterruptible(1);
    1235             :                 }
    1236             :         }
    1237           0 : }
    1238             : 
    1239           0 : void dev_deactivate(struct net_device *dev)
    1240             : {
    1241           0 :         LIST_HEAD(single);
    1242             : 
    1243           0 :         list_add(&dev->close_list, &single);
    1244           0 :         dev_deactivate_many(&single);
    1245           0 :         list_del(&single);
    1246           0 : }
    1247             : EXPORT_SYMBOL(dev_deactivate);
    1248             : 
    1249           0 : static int qdisc_change_tx_queue_len(struct net_device *dev,
    1250             :                                      struct netdev_queue *dev_queue)
    1251             : {
    1252           0 :         struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
    1253           0 :         const struct Qdisc_ops *ops = qdisc->ops;
    1254             : 
    1255           0 :         if (ops->change_tx_queue_len)
    1256           0 :                 return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
    1257             :         return 0;
    1258             : }
    1259             : 
    1260           0 : int dev_qdisc_change_tx_queue_len(struct net_device *dev)
    1261             : {
    1262           0 :         bool up = dev->flags & IFF_UP;
    1263           0 :         unsigned int i;
    1264           0 :         int ret = 0;
    1265             : 
    1266           0 :         if (up)
    1267           0 :                 dev_deactivate(dev);
    1268             : 
    1269           0 :         for (i = 0; i < dev->num_tx_queues; i++) {
    1270           0 :                 ret = qdisc_change_tx_queue_len(dev, &dev->_tx[i]);
    1271             : 
    1272             :                 /* TODO: revert changes on a partial failure */
    1273           0 :                 if (ret)
    1274             :                         break;
    1275             :         }
    1276             : 
    1277           0 :         if (up)
    1278           0 :                 dev_activate(dev);
    1279           0 :         return ret;
    1280             : }
    1281             : 
    1282           3 : static void dev_init_scheduler_queue(struct net_device *dev,
    1283             :                                      struct netdev_queue *dev_queue,
    1284             :                                      void *_qdisc)
    1285             : {
    1286           3 :         struct Qdisc *qdisc = _qdisc;
    1287             : 
    1288           3 :         rcu_assign_pointer(dev_queue->qdisc, qdisc);
    1289           3 :         dev_queue->qdisc_sleeping = qdisc;
    1290           0 : }
    1291             : 
    1292           3 : void dev_init_scheduler(struct net_device *dev)
    1293             : {
    1294           3 :         dev->qdisc = &noop_qdisc;
    1295           3 :         netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
    1296           3 :         if (dev_ingress_queue(dev))
    1297           0 :                 dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
    1298             : 
    1299           3 :         timer_setup(&dev->watchdog_timer, dev_watchdog, 0);
    1300           3 : }
    1301             : 
    1302           0 : static void shutdown_scheduler_queue(struct net_device *dev,
    1303             :                                      struct netdev_queue *dev_queue,
    1304             :                                      void *_qdisc_default)
    1305             : {
    1306           0 :         struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
    1307           0 :         struct Qdisc *qdisc_default = _qdisc_default;
    1308             : 
    1309           0 :         if (qdisc) {
    1310           0 :                 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
    1311           0 :                 dev_queue->qdisc_sleeping = qdisc_default;
    1312             : 
    1313           0 :                 qdisc_put(qdisc);
    1314             :         }
    1315           0 : }
    1316             : 
    1317           0 : void dev_shutdown(struct net_device *dev)
    1318             : {
    1319           0 :         netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
    1320           0 :         if (dev_ingress_queue(dev))
    1321           0 :                 shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
    1322           0 :         qdisc_put(dev->qdisc);
    1323           0 :         dev->qdisc = &noop_qdisc;
    1324             : 
    1325           0 :         WARN_ON(timer_pending(&dev->watchdog_timer));
    1326           0 : }
    1327             : 
    1328           0 : void psched_ratecfg_precompute(struct psched_ratecfg *r,
    1329             :                                const struct tc_ratespec *conf,
    1330             :                                u64 rate64)
    1331             : {
    1332           0 :         memset(r, 0, sizeof(*r));
    1333           0 :         r->overhead = conf->overhead;
    1334           0 :         r->rate_bytes_ps = max_t(u64, conf->rate, rate64);
    1335           0 :         r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);
    1336           0 :         r->mult = 1;
    1337             :         /*
    1338             :          * The deal here is to replace a divide by a reciprocal one
    1339             :          * in fast path (a reciprocal divide is a multiply and a shift)
    1340             :          *
    1341             :          * Normal formula would be :
    1342             :          *  time_in_ns = (NSEC_PER_SEC * len) / rate_bps
    1343             :          *
    1344             :          * We compute mult/shift to use instead :
    1345             :          *  time_in_ns = (len * mult) >> shift;
    1346             :          *
    1347             :          * We try to get the highest possible mult value for accuracy,
    1348             :          * but have to make sure no overflows will ever happen.
    1349             :          */
    1350           0 :         if (r->rate_bytes_ps > 0) {
    1351             :                 u64 factor = NSEC_PER_SEC;
    1352             : 
    1353           0 :                 for (;;) {
    1354           0 :                         r->mult = div64_u64(factor, r->rate_bytes_ps);
    1355           0 :                         if (r->mult & (1U << 31) || factor & (1ULL << 63))
    1356             :                                 break;
    1357           0 :                         factor <<= 1;
    1358           0 :                         r->shift++;
    1359             :                 }
    1360             :         }
    1361           0 : }
    1362             : EXPORT_SYMBOL(psched_ratecfg_precompute);
    1363             : 
    1364           0 : static void mini_qdisc_rcu_func(struct rcu_head *head)
    1365             : {
    1366           0 : }
    1367             : 
    1368           0 : void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
    1369             :                           struct tcf_proto *tp_head)
    1370             : {
    1371             :         /* Protected with chain0->filter_chain_lock.
    1372             :          * Can't access chain directly because tp_head can be NULL.
    1373             :          */
    1374           0 :         struct mini_Qdisc *miniq_old =
    1375           0 :                 rcu_dereference_protected(*miniqp->p_miniq, 1);
    1376           0 :         struct mini_Qdisc *miniq;
    1377             : 
    1378           0 :         if (!tp_head) {
    1379           0 :                 RCU_INIT_POINTER(*miniqp->p_miniq, NULL);
    1380             :                 /* Wait for flying RCU callback before it is freed. */
    1381           0 :                 rcu_barrier();
    1382           0 :                 return;
    1383             :         }
    1384             : 
    1385           0 :         miniq = !miniq_old || miniq_old == &miniqp->miniq2 ?
    1386           0 :                 &miniqp->miniq1 : &miniqp->miniq2;
    1387             : 
    1388             :         /* We need to make sure that readers won't see the miniq
    1389             :          * we are about to modify. So wait until previous call_rcu callback
    1390             :          * is done.
    1391             :          */
    1392           0 :         rcu_barrier();
    1393           0 :         miniq->filter_list = tp_head;
    1394           0 :         rcu_assign_pointer(*miniqp->p_miniq, miniq);
    1395             : 
    1396           0 :         if (miniq_old)
    1397             :                 /* This is counterpart of the rcu barriers above. We need to
    1398             :                  * block potential new user of miniq_old until all readers
    1399             :                  * are not seeing it.
    1400             :                  */
    1401           0 :                 call_rcu(&miniq_old->rcu, mini_qdisc_rcu_func);
    1402             : }
    1403             : EXPORT_SYMBOL(mini_qdisc_pair_swap);
    1404             : 
    1405           0 : void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp,
    1406             :                                 struct tcf_block *block)
    1407             : {
    1408           0 :         miniqp->miniq1.block = block;
    1409           0 :         miniqp->miniq2.block = block;
    1410           0 : }
    1411             : EXPORT_SYMBOL(mini_qdisc_pair_block_init);
    1412             : 
    1413           0 : void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
    1414             :                           struct mini_Qdisc __rcu **p_miniq)
    1415             : {
    1416           0 :         miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
    1417           0 :         miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
    1418           0 :         miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
    1419           0 :         miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;
    1420           0 :         miniqp->p_miniq = p_miniq;
    1421           0 : }
    1422             : EXPORT_SYMBOL(mini_qdisc_pair_init);

Generated by: LCOV version 1.14