Line data    Source code

       1             : /*
       2             :  * include/linux/topology.h
       3             :  *
       4             :  * Written by: Matthew Dobson, IBM Corporation
       5             :  *
       6             :  * Copyright (C) 2002, IBM Corp.
       7             :  *
       8             :  * All rights reserved.
       9             :  *
      10             :  * This program is free software; you can redistribute it and/or modify
      11             :  * it under the terms of the GNU General Public License as published by
      12             :  * the Free Software Foundation; either version 2 of the License, or
      13             :  * (at your option) any later version.
      14             :  *
      15             :  * This program is distributed in the hope that it will be useful, but
      16             :  * WITHOUT ANY WARRANTY; without even the implied warranty of
      17             :  * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or
      18             :  * NON INFRINGEMENT.  See the GNU General Public License for more
      19             :  * details.
      20             :  *
      21             :  * You should have received a copy of the GNU General Public License
      22             :  * along with this program; if not, write to the Free Software
      23             :  * Foundation, Inc., 675 Mass Ave, Cambridge, MA 02139, USA.
      24             :  *
      25             :  * Send feedback to <colpatch@us.ibm.com>
      26             :  */
      27             : #ifndef _LINUX_TOPOLOGY_H
      28             : #define _LINUX_TOPOLOGY_H
      29             : 
      30             : #include <linux/arch_topology.h>
      31             : #include <linux/cpumask.h>
      32             : #include <linux/bitops.h>
      33             : #include <linux/mmzone.h>
      34             : #include <linux/smp.h>
      35             : #include <linux/percpu.h>
      36             : #include <asm/topology.h>
      37             : 
      38             : #ifndef nr_cpus_node
      39             : #define nr_cpus_node(node) cpumask_weight(cpumask_of_node(node))
      40             : #endif
      41             : 
      42             : #define for_each_node_with_cpus(node)                   \
      43             :         for_each_online_node(node)                      \
      44             :                 if (nr_cpus_node(node))
      45             : 
      46             : int arch_update_cpu_topology(void);
      47             : 
      48             : /* Conform to ACPI 2.0 SLIT distance definitions */
      49             : #define LOCAL_DISTANCE          10
      50             : #define REMOTE_DISTANCE         20
      51             : #define DISTANCE_BITS           8
      52             : #ifndef node_distance
      53             : #define node_distance(from,to)  ((from) == (to) ? LOCAL_DISTANCE : REMOTE_DISTANCE)
      54             : #endif
      55             : #ifndef RECLAIM_DISTANCE
      56             : /*
      57             :  * If the distance between nodes in a system is larger than RECLAIM_DISTANCE
      58             :  * (in whatever arch specific measurement units returned by node_distance())
      59             :  * and node_reclaim_mode is enabled then the VM will only call node_reclaim()
      60             :  * on nodes within this distance.
      61             :  */
      62             : #define RECLAIM_DISTANCE 30
      63             : #endif
      64             : 
      65             : /*
      66             :  * The following tunable allows platforms to override the default node
      67             :  * reclaim distance (RECLAIM_DISTANCE) if remote memory accesses are
      68             :  * sufficiently fast that the default value actually hurts
      69             :  * performance.
      70             :  *
      71             :  * AMD EPYC machines use this because even though the 2-hop distance
      72             :  * is 32 (3.2x slower than a local memory access) performance actually
      73             :  * *improves* if allowed to reclaim memory and load balance tasks
      74             :  * between NUMA nodes 2-hops apart.
      75             :  */
      76             : extern int __read_mostly node_reclaim_distance;
      77             : 
      78             : #ifndef PENALTY_FOR_NODE_WITH_CPUS
      79             : #define PENALTY_FOR_NODE_WITH_CPUS      (1)
      80             : #endif
      81             : 
      82             : #ifdef CONFIG_USE_PERCPU_NUMA_NODE_ID
      83             : DECLARE_PER_CPU(int, numa_node);
      84             : 
      85             : #ifndef numa_node_id
      86             : /* Returns the number of the current Node. */
      87      825398 : static inline int numa_node_id(void)
      88             : {
      89      797915 :         return raw_cpu_read(numa_node);
      90             : }
      91             : #endif
      92             : 
      93             : #ifndef cpu_to_node
      94         452 : static inline int cpu_to_node(int cpu)
      95             : {
      96         432 :         return per_cpu(numa_node, cpu);
      97             : }
      98             : #endif
      99             : 
     100             : #ifndef set_numa_node
     101           4 : static inline void set_numa_node(int node)
     102             : {
     103           4 :         this_cpu_write(numa_node, node);
     104           4 : }
     105             : #endif
     106             : 
     107             : #ifndef set_cpu_numa_node
     108           8 : static inline void set_cpu_numa_node(int cpu, int node)
     109             : {
     110           8 :         per_cpu(numa_node, cpu) = node;
     111           4 : }
     112             : #endif
     113             : 
     114             : #else   /* !CONFIG_USE_PERCPU_NUMA_NODE_ID */
     115             : 
     116             : /* Returns the number of the current Node. */
     117             : #ifndef numa_node_id
     118             : static inline int numa_node_id(void)
     119             : {
     120             :         return cpu_to_node(raw_smp_processor_id());
     121             : }
     122             : #endif
     123             : 
     124             : #endif  /* [!]CONFIG_USE_PERCPU_NUMA_NODE_ID */
     125             : 
     126             : #ifdef CONFIG_HAVE_MEMORYLESS_NODES
     127             : 
     128             : /*
     129             :  * N.B., Do NOT reference the '_numa_mem_' per cpu variable directly.
     130             :  * It will not be defined when CONFIG_HAVE_MEMORYLESS_NODES is not defined.
     131             :  * Use the accessor functions set_numa_mem(), numa_mem_id() and cpu_to_mem().
     132             :  */
     133             : DECLARE_PER_CPU(int, _numa_mem_);
     134             : 
     135             : #ifndef set_numa_mem
     136             : static inline void set_numa_mem(int node)
     137             : {
     138             :         this_cpu_write(_numa_mem_, node);
     139             : }
     140             : #endif
     141             : 
     142             : #ifndef numa_mem_id
     143             : /* Returns the number of the nearest Node with memory */
     144             : static inline int numa_mem_id(void)
     145             : {
     146             :         return raw_cpu_read(_numa_mem_);
     147             : }
     148             : #endif
     149             : 
     150             : #ifndef cpu_to_mem
     151             : static inline int cpu_to_mem(int cpu)
     152             : {
     153             :         return per_cpu(_numa_mem_, cpu);
     154             : }
     155             : #endif
     156             : 
     157             : #ifndef set_cpu_numa_mem
     158             : static inline void set_cpu_numa_mem(int cpu, int node)
     159             : {
     160             :         per_cpu(_numa_mem_, cpu) = node;
     161             : }
     162             : #endif
     163             : 
     164             : #else   /* !CONFIG_HAVE_MEMORYLESS_NODES */
     165             : 
     166             : #ifndef numa_mem_id
     167             : /* Returns the number of the nearest Node with memory */
     168      251868 : static inline int numa_mem_id(void)
     169             : {
     170      251868 :         return numa_node_id();
     171             : }
     172             : #endif
     173             : 
     174             : #ifndef cpu_to_mem
     175             : static inline int cpu_to_mem(int cpu)
     176             : {
     177             :         return cpu_to_node(cpu);
     178             : }
     179             : #endif
     180             : 
     181             : #endif  /* [!]CONFIG_HAVE_MEMORYLESS_NODES */
     182             : 
     183             : #ifndef topology_physical_package_id
     184             : #define topology_physical_package_id(cpu)       ((void)(cpu), -1)
     185             : #endif
     186             : #ifndef topology_die_id
     187             : #define topology_die_id(cpu)                    ((void)(cpu), -1)
     188             : #endif
     189             : #ifndef topology_core_id
     190             : #define topology_core_id(cpu)                   ((void)(cpu), 0)
     191             : #endif
     192             : #ifndef topology_sibling_cpumask
     193             : #define topology_sibling_cpumask(cpu)           cpumask_of(cpu)
     194             : #endif
     195             : #ifndef topology_core_cpumask
     196             : #define topology_core_cpumask(cpu)              cpumask_of(cpu)
     197             : #endif
     198             : #ifndef topology_die_cpumask
     199             : #define topology_die_cpumask(cpu)               cpumask_of(cpu)
     200             : #endif
     201             : 
     202             : #if defined(CONFIG_SCHED_SMT) && !defined(cpu_smt_mask)
     203          59 : static inline const struct cpumask *cpu_smt_mask(int cpu)
     204             : {
     205          59 :         return topology_sibling_cpumask(cpu);
     206             : }
     207             : #endif
     208             : 
     209          32 : static inline const struct cpumask *cpu_cpu_mask(int cpu)
     210             : {
     211          32 :         return cpumask_of_node(cpu_to_node(cpu));
     212             : }
     213             : 
     214             : 
     215             : #endif /* _LINUX_TOPOLOGY_H */