Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _LINUX_SCHED_TOPOLOGY_H
       3             : #define _LINUX_SCHED_TOPOLOGY_H
       4             : 
       5             : #include <linux/topology.h>
       6             : 
       7             : #include <linux/sched/idle.h>
       8             : 
       9             : /*
      10             :  * sched-domains (multiprocessor balancing) declarations:
      11             :  */
      12             : #ifdef CONFIG_SMP
      13             : 
      14             : /* Generate SD flag indexes */
      15             : #define SD_FLAG(name, mflags) __##name,
      16             : enum {
      17             :         #include <linux/sched/sd_flags.h>
      18             :         __SD_FLAG_CNT,
      19             : };
      20             : #undef SD_FLAG
      21             : /* Generate SD flag bits */
      22             : #define SD_FLAG(name, mflags) name = 1 << __##name,
      23             : enum {
      24             :         #include <linux/sched/sd_flags.h>
      25             : };
      26             : #undef SD_FLAG
      27             : 
      28             : #ifdef CONFIG_SCHED_DEBUG
      29             : 
      30             : struct sd_flag_debug {
      31             :         unsigned int meta_flags;
      32             :         char *name;
      33             : };
      34             : extern const struct sd_flag_debug sd_flag_debug[];
      35             : 
      36             : #endif
      37             : 
      38             : #ifdef CONFIG_SCHED_SMT
      39           4 : static inline int cpu_smt_flags(void)
      40             : {
      41           4 :         return SD_SHARE_CPUCAPACITY | SD_SHARE_PKG_RESOURCES;
      42             : }
      43             : #endif
      44             : 
      45             : #ifdef CONFIG_SCHED_MC
      46           4 : static inline int cpu_core_flags(void)
      47             : {
      48           4 :         return SD_SHARE_PKG_RESOURCES;
      49             : }
      50             : #endif
      51             : 
      52             : #ifdef CONFIG_NUMA
      53           0 : static inline int cpu_numa_flags(void)
      54             : {
      55           0 :         return SD_NUMA;
      56             : }
      57             : #endif
      58             : 
      59             : extern int arch_asym_cpu_priority(int cpu);
      60             : 
      61             : struct sched_domain_attr {
      62             :         int relax_domain_level;
      63             : };
      64             : 
      65             : #define SD_ATTR_INIT    (struct sched_domain_attr) {    \
      66             :         .relax_domain_level = -1,                       \
      67             : }
      68             : 
      69             : extern int sched_domain_level_max;
      70             : 
      71             : struct sched_group;
      72             : 
      73             : struct sched_domain_shared {
      74             :         atomic_t        ref;
      75             :         atomic_t        nr_busy_cpus;
      76             :         int             has_idle_cores;
      77             : };
      78             : 
      79             : struct sched_domain {
      80             :         /* These fields must be setup */
      81             :         struct sched_domain __rcu *parent;      /* top domain must be null terminated */
      82             :         struct sched_domain __rcu *child;       /* bottom domain must be null terminated */
      83             :         struct sched_group *groups;     /* the balancing groups of the domain */
      84             :         unsigned long min_interval;     /* Minimum balance interval ms */
      85             :         unsigned long max_interval;     /* Maximum balance interval ms */
      86             :         unsigned int busy_factor;       /* less balancing by factor if busy */
      87             :         unsigned int imbalance_pct;     /* No balance until over watermark */
      88             :         unsigned int cache_nice_tries;  /* Leave cache hot tasks for # tries */
      89             : 
      90             :         int nohz_idle;                  /* NOHZ IDLE status */
      91             :         int flags;                      /* See SD_* */
      92             :         int level;
      93             : 
      94             :         /* Runtime fields. */
      95             :         unsigned long last_balance;     /* init to jiffies. units in jiffies */
      96             :         unsigned int balance_interval;  /* initialise to 1. units in ms. */
      97             :         unsigned int nr_balance_failed; /* initialise to 0 */
      98             : 
      99             :         /* idle_balance() stats */
     100             :         u64 max_newidle_lb_cost;
     101             :         unsigned long next_decay_max_lb_cost;
     102             : 
     103             :         u64 avg_scan_cost;              /* select_idle_sibling */
     104             : 
     105             : #ifdef CONFIG_SCHEDSTATS
     106             :         /* load_balance() stats */
     107             :         unsigned int lb_count[CPU_MAX_IDLE_TYPES];
     108             :         unsigned int lb_failed[CPU_MAX_IDLE_TYPES];
     109             :         unsigned int lb_balanced[CPU_MAX_IDLE_TYPES];
     110             :         unsigned int lb_imbalance[CPU_MAX_IDLE_TYPES];
     111             :         unsigned int lb_gained[CPU_MAX_IDLE_TYPES];
     112             :         unsigned int lb_hot_gained[CPU_MAX_IDLE_TYPES];
     113             :         unsigned int lb_nobusyg[CPU_MAX_IDLE_TYPES];
     114             :         unsigned int lb_nobusyq[CPU_MAX_IDLE_TYPES];
     115             : 
     116             :         /* Active load balancing */
     117             :         unsigned int alb_count;
     118             :         unsigned int alb_failed;
     119             :         unsigned int alb_pushed;
     120             : 
     121             :         /* SD_BALANCE_EXEC stats */
     122             :         unsigned int sbe_count;
     123             :         unsigned int sbe_balanced;
     124             :         unsigned int sbe_pushed;
     125             : 
     126             :         /* SD_BALANCE_FORK stats */
     127             :         unsigned int sbf_count;
     128             :         unsigned int sbf_balanced;
     129             :         unsigned int sbf_pushed;
     130             : 
     131             :         /* try_to_wake_up() stats */
     132             :         unsigned int ttwu_wake_remote;
     133             :         unsigned int ttwu_move_affine;
     134             :         unsigned int ttwu_move_balance;
     135             : #endif
     136             : #ifdef CONFIG_SCHED_DEBUG
     137             :         char *name;
     138             : #endif
     139             :         union {
     140             :                 void *private;          /* used during construction */
     141             :                 struct rcu_head rcu;    /* used during destruction */
     142             :         };
     143             :         struct sched_domain_shared *shared;
     144             : 
     145             :         unsigned int span_weight;
     146             :         /*
     147             :          * Span of all CPUs in this domain.
     148             :          *
     149             :          * NOTE: this field is variable length. (Allocated dynamically
     150             :          * by attaching extra space to the end of the structure,
     151             :          * depending on how many CPUs the kernel has booted up with)
     152             :          */
     153             :         unsigned long span[];
     154             : };
     155             : 
     156       12442 : static inline struct cpumask *sched_domain_span(struct sched_domain *sd)
     157             : {
     158       12442 :         return to_cpumask(sd->span);
     159             : }
     160             : 
     161             : extern void partition_sched_domains_locked(int ndoms_new,
     162             :                                            cpumask_var_t doms_new[],
     163             :                                            struct sched_domain_attr *dattr_new);
     164             : 
     165             : extern void partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
     166             :                                     struct sched_domain_attr *dattr_new);
     167             : 
     168             : /* Allocate an array of sched domains, for partition_sched_domains(). */
     169             : cpumask_var_t *alloc_sched_domains(unsigned int ndoms);
     170             : void free_sched_domains(cpumask_var_t doms[], unsigned int ndoms);
     171             : 
     172             : bool cpus_share_cache(int this_cpu, int that_cpu);
     173             : 
     174             : typedef const struct cpumask *(*sched_domain_mask_f)(int cpu);
     175             : typedef int (*sched_domain_flags_f)(void);
     176             : 
     177             : #define SDTL_OVERLAP    0x01
     178             : 
     179             : struct sd_data {
     180             :         struct sched_domain *__percpu *sd;
     181             :         struct sched_domain_shared *__percpu *sds;
     182             :         struct sched_group *__percpu *sg;
     183             :         struct sched_group_capacity *__percpu *sgc;
     184             : };
     185             : 
     186             : struct sched_domain_topology_level {
     187             :         sched_domain_mask_f mask;
     188             :         sched_domain_flags_f sd_flags;
     189             :         int                 flags;
     190             :         int                 numa_level;
     191             :         struct sd_data      data;
     192             : #ifdef CONFIG_SCHED_DEBUG
     193             :         char                *name;
     194             : #endif
     195             : };
     196             : 
     197             : extern void set_sched_topology(struct sched_domain_topology_level *tl);
     198             : 
     199             : #ifdef CONFIG_SCHED_DEBUG
     200             : # define SD_INIT_NAME(type)             .name = #type
     201             : #else
     202             : # define SD_INIT_NAME(type)
     203             : #endif
     204             : 
     205             : #else /* CONFIG_SMP */
     206             : 
     207             : struct sched_domain_attr;
     208             : 
     209             : static inline void
     210             : partition_sched_domains_locked(int ndoms_new, cpumask_var_t doms_new[],
     211             :                                struct sched_domain_attr *dattr_new)
     212             : {
     213             : }
     214             : 
     215             : static inline void
     216             : partition_sched_domains(int ndoms_new, cpumask_var_t doms_new[],
     217             :                         struct sched_domain_attr *dattr_new)
     218             : {
     219             : }
     220             : 
     221             : static inline bool cpus_share_cache(int this_cpu, int that_cpu)
     222             : {
     223             :         return true;
     224             : }
     225             : 
     226             : #endif  /* !CONFIG_SMP */
     227             : 
     228             : #if defined(CONFIG_ENERGY_MODEL) && defined(CONFIG_CPU_FREQ_GOV_SCHEDUTIL)
     229             : extern void rebuild_sched_domains_energy(void);
     230             : #else
     231             : static inline void rebuild_sched_domains_energy(void)
     232             : {
     233             : }
     234             : #endif
     235             : 
     236             : #ifndef arch_scale_cpu_capacity
     237             : /**
     238             :  * arch_scale_cpu_capacity - get the capacity scale factor of a given CPU.
     239             :  * @cpu: the CPU in question.
     240             :  *
     241             :  * Return: the CPU scale factor normalized against SCHED_CAPACITY_SCALE, i.e.
     242             :  *
     243             :  *             max_perf(cpu)
     244             :  *      ----------------------------- * SCHED_CAPACITY_SCALE
     245             :  *      max(max_perf(c) : c \in CPUs)
     246             :  */
     247             : static __always_inline
     248       85111 : unsigned long arch_scale_cpu_capacity(int cpu)
     249             : {
     250       85111 :         return SCHED_CAPACITY_SCALE;
     251             : }
     252             : #endif
     253             : 
     254             : #ifndef arch_scale_thermal_pressure
     255             : static __always_inline
     256       37326 : unsigned long arch_scale_thermal_pressure(int cpu)
     257             : {
     258       37326 :         return 0;
     259             : }
     260             : #endif
     261             : 
     262             : #ifndef arch_set_thermal_pressure
     263             : static __always_inline
     264             : void arch_set_thermal_pressure(const struct cpumask *cpus,
     265             :                                unsigned long th_pressure)
     266             : { }
     267             : #endif
     268             : 
     269             : static inline int task_node(const struct task_struct *p)
     270             : {
     271             :         return cpu_to_node(task_cpu(p));
     272             : }
     273             : 
     274             : #endif /* _LINUX_SCHED_TOPOLOGY_H */