Line data    Source code

       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _ASM_X86_SYNC_CORE_H
       3             : #define _ASM_X86_SYNC_CORE_H
       4             : 
       5             : #include <linux/preempt.h>
       6             : #include <asm/processor.h>
       7             : #include <asm/cpufeature.h>
       8             : #include <asm/special_insns.h>
       9             : 
      10             : #ifdef CONFIG_X86_32
      11             : static inline void iret_to_self(void)
      12             : {
      13             :         asm volatile (
      14             :                 "pushfl\n\t"
      15             :                 "pushl %%cs\n\t"
      16             :                 "pushl $1f\n\t"
      17             :                 "iret\n\t"
      18             :                 "1:"
      19             :                 : ASM_CALL_CONSTRAINT : : "memory");
      20             : }
      21             : #else
      22        1151 : static inline void iret_to_self(void)
      23             : {
      24        1151 :         unsigned int tmp;
      25             : 
      26        1151 :         asm volatile (
      27             :                 "mov %%ss, %0\n\t"
      28             :                 "pushq %q0\n\t"
      29             :                 "pushq %%rsp\n\t"
      30             :                 "addq $8, (%%rsp)\n\t"
      31             :                 "pushfq\n\t"
      32             :                 "mov %%cs, %0\n\t"
      33             :                 "pushq %q0\n\t"
      34             :                 "pushq $1f\n\t"
      35             :                 "iretq\n\t"
      36             :                 "1:"
      37             :                 : "=&r" (tmp), ASM_CALL_CONSTRAINT : : "cc", "memory");
      38        1151 : }
      39             : #endif /* CONFIG_X86_32 */
      40             : 
      41             : /*
      42             :  * This function forces the icache and prefetched instruction stream to
      43             :  * catch up with reality in two very specific cases:
      44             :  *
      45             :  *  a) Text was modified using one virtual address and is about to be executed
      46             :  *     from the same physical page at a different virtual address.
      47             :  *
      48             :  *  b) Text was modified on a different CPU, may subsequently be
      49             :  *     executed on this CPU, and you want to make sure the new version
      50             :  *     gets executed.  This generally means you're calling this in an IPI.
      51             :  *
      52             :  * If you're calling this for a different reason, you're probably doing
      53             :  * it wrong.
      54             :  *
      55             :  * Like all of Linux's memory ordering operations, this is a
      56             :  * compiler barrier as well.
      57             :  */
      58        1151 : static inline void sync_core(void)
      59             : {
      60             :         /*
      61             :          * The SERIALIZE instruction is the most straightforward way to
      62             :          * do this, but it is not universally available.
      63             :          */
      64        1151 :         if (static_cpu_has(X86_FEATURE_SERIALIZE)) {
      65           0 :                 serialize();
      66           0 :                 return;
      67             :         }
      68             : 
      69             :         /*
      70             :          * For all other processors, there are quite a few ways to do this.
      71             :          * IRET-to-self is nice because it works on every CPU, at any CPL
      72             :          * (so it's compatible with paravirtualization), and it never exits
      73             :          * to a hypervisor.  The only downsides are that it's a bit slow
      74             :          * (it seems to be a bit more than 2x slower than the fastest
      75             :          * options) and that it unmasks NMIs.  The "push %cs" is needed,
      76             :          * because in paravirtual environments __KERNEL_CS may not be a
      77             :          * valid CS value when we do IRET directly.
      78             :          *
      79             :          * In case NMI unmasking or performance ever becomes a problem,
      80             :          * the next best option appears to be MOV-to-CR2 and an
      81             :          * unconditional jump.  That sequence also works on all CPUs,
      82             :          * but it will fault at CPL3 (i.e. Xen PV).
      83             :          *
      84             :          * CPUID is the conventional way, but it's nasty: it doesn't
      85             :          * exist on some 486-like CPUs, and it usually exits to a
      86             :          * hypervisor.
      87             :          */
      88        1151 :         iret_to_self();
      89             : }
      90             : 
      91             : /*
      92             :  * Ensure that a core serializing instruction is issued before returning
      93             :  * to user-mode. x86 implements return to user-space through sysexit,
      94             :  * sysrel, and sysretq, which are not core serializing.
      95             :  */
      96             : static inline void sync_core_before_usermode(void)
      97             : {
      98             :         /* With PTI, we unconditionally serialize before running user code. */
      99             :         if (static_cpu_has(X86_FEATURE_PTI))
     100             :                 return;
     101             : 
     102             :         /*
     103             :          * Even if we're in an interrupt, we might reschedule before returning,
     104             :          * in which case we could switch to a different thread in the same mm
     105             :          * and return using SYSRET or SYSEXIT.  Instead of trying to keep
     106             :          * track of our need to sync the core, just sync right away.
     107             :          */
     108             :         sync_core();
     109             : }
     110             : 
     111             : #endif /* _ASM_X86_SYNC_CORE_H */