Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0-only */
       2             : /*
       3             :  * net busy poll support
       4             :  * Copyright(c) 2013 Intel Corporation.
       5             :  *
       6             :  * Author: Eliezer Tamir
       7             :  *
       8             :  * Contact Information:
       9             :  * e1000-devel Mailing List <e1000-devel@lists.sourceforge.net>
      10             :  */
      11             : 
      12             : #ifndef _LINUX_NET_BUSY_POLL_H
      13             : #define _LINUX_NET_BUSY_POLL_H
      14             : 
      15             : #include <linux/netdevice.h>
      16             : #include <linux/sched/clock.h>
      17             : #include <linux/sched/signal.h>
      18             : #include <net/ip.h>
      19             : 
      20             : /*              0 - Reserved to indicate value not set
      21             :  *     1..NR_CPUS - Reserved for sender_cpu
      22             :  *  NR_CPUS+1..~0 - Region available for NAPI IDs
      23             :  */
      24             : #define MIN_NAPI_ID ((unsigned int)(NR_CPUS + 1))
      25             : 
      26             : #define BUSY_POLL_BUDGET 8
      27             : 
      28             : #ifdef CONFIG_NET_RX_BUSY_POLL
      29             : 
      30             : struct napi_struct;
      31             : extern unsigned int sysctl_net_busy_read __read_mostly;
      32             : extern unsigned int sysctl_net_busy_poll __read_mostly;
      33             : 
      34        6682 : static inline bool net_busy_loop_on(void)
      35             : {
      36        6682 :         return sysctl_net_busy_poll;
      37             : }
      38             : 
      39        7191 : static inline bool sk_can_busy_loop(const struct sock *sk)
      40             : {
      41        7191 :         return sk->sk_ll_usec && !signal_pending(current);
      42             : }
      43             : 
      44             : bool sk_busy_loop_end(void *p, unsigned long start_time);
      45             : 
      46             : void napi_busy_loop(unsigned int napi_id,
      47             :                     bool (*loop_end)(void *, unsigned long),
      48             :                     void *loop_end_arg, bool prefer_busy_poll, u16 budget);
      49             : 
      50             : #else /* CONFIG_NET_RX_BUSY_POLL */
      51             : static inline unsigned long net_busy_loop_on(void)
      52             : {
      53             :         return 0;
      54             : }
      55             : 
      56             : static inline bool sk_can_busy_loop(struct sock *sk)
      57             : {
      58             :         return false;
      59             : }
      60             : 
      61             : #endif /* CONFIG_NET_RX_BUSY_POLL */
      62             : 
      63           0 : static inline unsigned long busy_loop_current_time(void)
      64             : {
      65             : #ifdef CONFIG_NET_RX_BUSY_POLL
      66           0 :         return (unsigned long)(local_clock() >> 10);
      67             : #else
      68             :         return 0;
      69             : #endif
      70             : }
      71             : 
      72             : /* in poll/select we use the global sysctl_net_ll_poll value */
      73           0 : static inline bool busy_loop_timeout(unsigned long start_time)
      74             : {
      75             : #ifdef CONFIG_NET_RX_BUSY_POLL
      76           0 :         unsigned long bp_usec = READ_ONCE(sysctl_net_busy_poll);
      77             : 
      78           0 :         if (bp_usec) {
      79           0 :                 unsigned long end_time = start_time + bp_usec;
      80           0 :                 unsigned long now = busy_loop_current_time();
      81             : 
      82           0 :                 return time_after(now, end_time);
      83             :         }
      84             : #endif
      85             :         return true;
      86             : }
      87             : 
      88           0 : static inline bool sk_busy_loop_timeout(struct sock *sk,
      89             :                                         unsigned long start_time)
      90             : {
      91             : #ifdef CONFIG_NET_RX_BUSY_POLL
      92           0 :         unsigned long bp_usec = READ_ONCE(sk->sk_ll_usec);
      93             : 
      94           0 :         if (bp_usec) {
      95           0 :                 unsigned long end_time = start_time + bp_usec;
      96           0 :                 unsigned long now = busy_loop_current_time();
      97             : 
      98           0 :                 return time_after(now, end_time);
      99             :         }
     100             : #endif
     101             :         return true;
     102             : }
     103             : 
     104           0 : static inline void sk_busy_loop(struct sock *sk, int nonblock)
     105             : {
     106             : #ifdef CONFIG_NET_RX_BUSY_POLL
     107           0 :         unsigned int napi_id = READ_ONCE(sk->sk_napi_id);
     108             : 
     109           0 :         if (napi_id >= MIN_NAPI_ID)
     110           0 :                 napi_busy_loop(napi_id, nonblock ? NULL : sk_busy_loop_end, sk,
     111           0 :                                READ_ONCE(sk->sk_prefer_busy_poll),
     112           0 :                                READ_ONCE(sk->sk_busy_poll_budget) ?: BUSY_POLL_BUDGET);
     113             : #endif
     114           0 : }
     115             : 
     116             : /* used in the NIC receive handler to mark the skb */
     117         723 : static inline void skb_mark_napi_id(struct sk_buff *skb,
     118             :                                     struct napi_struct *napi)
     119             : {
     120             : #ifdef CONFIG_NET_RX_BUSY_POLL
     121             :         /* If the skb was already marked with a valid NAPI ID, avoid overwriting
     122             :          * it.
     123             :          */
     124         723 :         if (skb->napi_id < MIN_NAPI_ID)
     125         723 :                 skb->napi_id = napi->napi_id;
     126             : #endif
     127             : }
     128             : 
     129             : /* used in the protocol hanlder to propagate the napi_id to the socket */
     130         416 : static inline void sk_mark_napi_id(struct sock *sk, const struct sk_buff *skb)
     131             : {
     132             : #ifdef CONFIG_NET_RX_BUSY_POLL
     133         416 :         WRITE_ONCE(sk->sk_napi_id, skb->napi_id);
     134             : #endif
     135         832 :         sk_rx_queue_set(sk, skb);
     136             : }
     137             : 
     138           2 : static inline void __sk_mark_napi_id_once(struct sock *sk, unsigned int napi_id)
     139             : {
     140             : #ifdef CONFIG_NET_RX_BUSY_POLL
     141           2 :         if (!READ_ONCE(sk->sk_napi_id))
     142           1 :                 WRITE_ONCE(sk->sk_napi_id, napi_id);
     143             : #endif
     144             : }
     145             : 
     146             : /* variant used for unconnected sockets */
     147           2 : static inline void sk_mark_napi_id_once(struct sock *sk,
     148             :                                         const struct sk_buff *skb)
     149             : {
     150             : #ifdef CONFIG_NET_RX_BUSY_POLL
     151           3 :         __sk_mark_napi_id_once(sk, skb->napi_id);
     152             : #endif
     153             : }
     154             : 
     155             : static inline void sk_mark_napi_id_once_xdp(struct sock *sk,
     156             :                                             const struct xdp_buff *xdp)
     157             : {
     158             : #ifdef CONFIG_NET_RX_BUSY_POLL
     159             :         __sk_mark_napi_id_once(sk, xdp->rxq->napi_id);
     160             : #endif
     161             : }
     162             : 
     163             : #endif /* _LINUX_NET_BUSY_POLL_H */