Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0+ */
       2             : /*
       3             :  * RCU segmented callback lists, internal-to-rcu header file
       4             :  *
       5             :  * Copyright IBM Corporation, 2017
       6             :  *
       7             :  * Authors: Paul E. McKenney <paulmck@linux.ibm.com>
       8             :  */
       9             : 
      10             : #include <linux/rcu_segcblist.h>
      11             : 
      12             : /* Return number of callbacks in the specified callback list. */
      13             : static inline long rcu_cblist_n_cbs(struct rcu_cblist *rclp)
      14             : {
      15             :         return READ_ONCE(rclp->len);
      16             : }
      17             : 
      18             : /* Return number of callbacks in segmented callback list by summing seglen. */
      19             : long rcu_segcblist_n_segment_cbs(struct rcu_segcblist *rsclp);
      20             : 
      21             : void rcu_cblist_init(struct rcu_cblist *rclp);
      22             : void rcu_cblist_enqueue(struct rcu_cblist *rclp, struct rcu_head *rhp);
      23             : void rcu_cblist_flush_enqueue(struct rcu_cblist *drclp,
      24             :                               struct rcu_cblist *srclp,
      25             :                               struct rcu_head *rhp);
      26             : struct rcu_head *rcu_cblist_dequeue(struct rcu_cblist *rclp);
      27             : 
      28             : /*
      29             :  * Is the specified rcu_segcblist structure empty?
      30             :  *
      31             :  * But careful!  The fact that the ->head field is NULL does not
      32             :  * necessarily imply that there are no callbacks associated with
      33             :  * this structure.  When callbacks are being invoked, they are
      34             :  * removed as a group.  If callback invocation must be preempted,
      35             :  * the remaining callbacks will be added back to the list.  Either
      36             :  * way, the counts are updated later.
      37             :  *
      38             :  * So it is often the case that rcu_segcblist_n_cbs() should be used
      39             :  * instead.
      40             :  */
      41       66084 : static inline bool rcu_segcblist_empty(struct rcu_segcblist *rsclp)
      42             : {
      43       66084 :         return !READ_ONCE(rsclp->head);
      44             : }
      45             : 
      46             : /* Return number of callbacks in segmented callback list. */
      47     2007780 : static inline long rcu_segcblist_n_cbs(struct rcu_segcblist *rsclp)
      48             : {
      49             : #ifdef CONFIG_RCU_NOCB_CPU
      50             :         return atomic_long_read(&rsclp->len);
      51             : #else
      52     2007778 :         return READ_ONCE(rsclp->len);
      53             : #endif
      54             : }
      55             : 
      56          16 : static inline void rcu_segcblist_set_flags(struct rcu_segcblist *rsclp,
      57             :                                            int flags)
      58             : {
      59          16 :         rsclp->flags |= flags;
      60           0 : }
      61             : 
      62           0 : static inline void rcu_segcblist_clear_flags(struct rcu_segcblist *rsclp,
      63             :                                              int flags)
      64             : {
      65           0 :         rsclp->flags &= ~flags;
      66           0 : }
      67             : 
      68      901114 : static inline bool rcu_segcblist_test_flags(struct rcu_segcblist *rsclp,
      69             :                                             int flags)
      70             : {
      71      901114 :         return READ_ONCE(rsclp->flags) & flags;
      72             : }
      73             : 
      74             : /*
      75             :  * Is the specified rcu_segcblist enabled, for example, not corresponding
      76             :  * to an offline CPU?
      77             :  */
      78      901114 : static inline bool rcu_segcblist_is_enabled(struct rcu_segcblist *rsclp)
      79             : {
      80      901114 :         return rcu_segcblist_test_flags(rsclp, SEGCBLIST_ENABLED);
      81             : }
      82             : 
      83             : /* Is the specified rcu_segcblist offloaded, or is SEGCBLIST_SOFTIRQ_ONLY set? */
      84      734111 : static inline bool rcu_segcblist_is_offloaded(struct rcu_segcblist *rsclp)
      85             : {
      86      734111 :         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) &&
      87             :             !rcu_segcblist_test_flags(rsclp, SEGCBLIST_SOFTIRQ_ONLY))
      88             :                 return true;
      89             : 
      90      734111 :         return false;
      91             : }
      92             : 
      93       54051 : static inline bool rcu_segcblist_completely_offloaded(struct rcu_segcblist *rsclp)
      94             : {
      95       54051 :         int flags = SEGCBLIST_KTHREAD_CB | SEGCBLIST_KTHREAD_GP | SEGCBLIST_OFFLOADED;
      96             : 
      97       54051 :         if (IS_ENABLED(CONFIG_RCU_NOCB_CPU) && (rsclp->flags & flags) == flags)
      98             :                 return true;
      99             : 
     100       54051 :         return false;
     101             : }
     102             : 
     103             : /*
     104             :  * Are all segments following the specified segment of the specified
     105             :  * rcu_segcblist structure empty of callbacks?  (The specified
     106             :  * segment might well contain callbacks.)
     107             :  */
     108       84035 : static inline bool rcu_segcblist_restempty(struct rcu_segcblist *rsclp, int seg)
     109             : {
     110       84035 :         return !READ_ONCE(*READ_ONCE(rsclp->tails[seg]));
     111             : }
     112             : 
     113             : /*
     114             :  * Is the specified segment of the specified rcu_segcblist structure
     115             :  * empty of callbacks?
     116             :  */
     117             : static inline bool rcu_segcblist_segempty(struct rcu_segcblist *rsclp, int seg)
     118             : {
     119             :         if (seg == RCU_DONE_TAIL)
     120             :                 return &rsclp->head == rsclp->tails[RCU_DONE_TAIL];
     121             :         return rsclp->tails[seg - 1] == rsclp->tails[seg];
     122             : }
     123             : 
     124             : void rcu_segcblist_inc_len(struct rcu_segcblist *rsclp);
     125             : void rcu_segcblist_add_len(struct rcu_segcblist *rsclp, long v);
     126             : void rcu_segcblist_init(struct rcu_segcblist *rsclp);
     127             : void rcu_segcblist_disable(struct rcu_segcblist *rsclp);
     128             : void rcu_segcblist_offload(struct rcu_segcblist *rsclp, bool offload);
     129             : bool rcu_segcblist_ready_cbs(struct rcu_segcblist *rsclp);
     130             : bool rcu_segcblist_pend_cbs(struct rcu_segcblist *rsclp);
     131             : struct rcu_head *rcu_segcblist_first_cb(struct rcu_segcblist *rsclp);
     132             : struct rcu_head *rcu_segcblist_first_pend_cb(struct rcu_segcblist *rsclp);
     133             : bool rcu_segcblist_nextgp(struct rcu_segcblist *rsclp, unsigned long *lp);
     134             : void rcu_segcblist_enqueue(struct rcu_segcblist *rsclp,
     135             :                            struct rcu_head *rhp);
     136             : bool rcu_segcblist_entrain(struct rcu_segcblist *rsclp,
     137             :                            struct rcu_head *rhp);
     138             : void rcu_segcblist_extract_done_cbs(struct rcu_segcblist *rsclp,
     139             :                                     struct rcu_cblist *rclp);
     140             : void rcu_segcblist_extract_pend_cbs(struct rcu_segcblist *rsclp,
     141             :                                     struct rcu_cblist *rclp);
     142             : void rcu_segcblist_insert_count(struct rcu_segcblist *rsclp,
     143             :                                 struct rcu_cblist *rclp);
     144             : void rcu_segcblist_insert_done_cbs(struct rcu_segcblist *rsclp,
     145             :                                    struct rcu_cblist *rclp);
     146             : void rcu_segcblist_insert_pend_cbs(struct rcu_segcblist *rsclp,
     147             :                                    struct rcu_cblist *rclp);
     148             : void rcu_segcblist_advance(struct rcu_segcblist *rsclp, unsigned long seq);
     149             : bool rcu_segcblist_accelerate(struct rcu_segcblist *rsclp, unsigned long seq);
     150             : void rcu_segcblist_merge(struct rcu_segcblist *dst_rsclp,
     151             :                          struct rcu_segcblist *src_rsclp);