LCOV - code coverage report
Current view: top level - arch/x86/include/asm - desc.h (source / functions) Hit Total Coverage
Test: landlock.info Lines: 76 120 63.3 %
Date: 2021-04-22 12:43:58 Functions: 7 8 87.5 %

          Line data    Source code
       1             : /* SPDX-License-Identifier: GPL-2.0 */
       2             : #ifndef _ASM_X86_DESC_H
       3             : #define _ASM_X86_DESC_H
       4             : 
       5             : #include <asm/desc_defs.h>
       6             : #include <asm/ldt.h>
       7             : #include <asm/mmu.h>
       8             : #include <asm/fixmap.h>
       9             : #include <asm/irq_vectors.h>
      10             : #include <asm/cpu_entry_area.h>
      11             : 
      12             : #include <linux/smp.h>
      13             : #include <linux/percpu.h>
      14             : 
      15           0 : static inline void fill_ldt(struct desc_struct *desc, const struct user_desc *info)
      16             : {
      17           0 :         desc->limit0         = info->limit & 0x0ffff;
      18             : 
      19           0 :         desc->base0          = (info->base_addr & 0x0000ffff);
      20           0 :         desc->base1          = (info->base_addr & 0x00ff0000) >> 16;
      21             : 
      22           0 :         desc->type           = (info->read_exec_only ^ 1) << 1;
      23           0 :         desc->type          |= info->contents << 2;
      24             :         /* Set the ACCESS bit so it can be mapped RO */
      25           0 :         desc->type          |= 1;
      26             : 
      27           0 :         desc->s                      = 1;
      28           0 :         desc->dpl            = 0x3;
      29           0 :         desc->p                      = info->seg_not_present ^ 1;
      30           0 :         desc->limit1         = (info->limit & 0xf0000) >> 16;
      31           0 :         desc->avl            = info->useable;
      32           0 :         desc->d                      = info->seg_32bit;
      33           0 :         desc->g                      = info->limit_in_pages;
      34             : 
      35           0 :         desc->base2          = (info->base_addr & 0xff000000) >> 24;
      36             :         /*
      37             :          * Don't allow setting of the lm bit. It would confuse
      38             :          * user_64bit_mode and would get overridden by sysret anyway.
      39             :          */
      40           0 :         desc->l                      = 0;
      41           0 : }
      42             : 
      43             : struct gdt_page {
      44             :         struct desc_struct gdt[GDT_ENTRIES];
      45             : } __attribute__((aligned(PAGE_SIZE)));
      46             : 
      47             : DECLARE_PER_CPU_PAGE_ALIGNED(struct gdt_page, gdt_page);
      48             : 
      49             : /* Provide the original GDT */
      50       53492 : static inline struct desc_struct *get_cpu_gdt_rw(unsigned int cpu)
      51             : {
      52       53478 :         return per_cpu(gdt_page, cpu).gdt;
      53             : }
      54             : 
      55             : /* Provide the current original GDT */
      56             : static inline struct desc_struct *get_current_gdt_rw(void)
      57             : {
      58             :         return this_cpu_ptr(&gdt_page)->gdt;
      59             : }
      60             : 
      61             : /* Provide the fixmap address of the remapped GDT */
      62          11 : static inline struct desc_struct *get_cpu_gdt_ro(int cpu)
      63             : {
      64           4 :         return (struct desc_struct *)&get_cpu_entry_area(cpu)->gdt;
      65             : }
      66             : 
      67             : /* Provide the current read-only GDT */
      68             : static inline struct desc_struct *get_current_gdt_ro(void)
      69             : {
      70             :         return get_cpu_gdt_ro(smp_processor_id());
      71             : }
      72             : 
      73             : /* Provide the physical address of the GDT page. */
      74           4 : static inline phys_addr_t get_cpu_gdt_paddr(unsigned int cpu)
      75             : {
      76           4 :         return per_cpu_ptr_to_phys(get_cpu_gdt_rw(cpu));
      77             : }
      78             : 
      79             : static inline void pack_gate(gate_desc *gate, unsigned type, unsigned long func,
      80             :                              unsigned dpl, unsigned ist, unsigned seg)
      81             : {
      82             :         gate->offset_low     = (u16) func;
      83             :         gate->bits.p         = 1;
      84             :         gate->bits.dpl               = dpl;
      85             :         gate->bits.zero              = 0;
      86             :         gate->bits.type              = type;
      87             :         gate->offset_middle  = (u16) (func >> 16);
      88             : #ifdef CONFIG_X86_64
      89             :         gate->segment                = __KERNEL_CS;
      90             :         gate->bits.ist               = ist;
      91             :         gate->reserved               = 0;
      92             :         gate->offset_high    = (u32) (func >> 32);
      93             : #else
      94             :         gate->segment                = seg;
      95             :         gate->bits.ist               = 0;
      96             : #endif
      97             : }
      98             : 
      99           0 : static inline int desc_empty(const void *ptr)
     100             : {
     101           0 :         const u32 *desc = ptr;
     102             : 
     103           0 :         return !(desc[0] | desc[1]);
     104             : }
     105             : 
     106             : #ifdef CONFIG_PARAVIRT_XXL
     107             : #include <asm/paravirt.h>
     108             : #else
     109             : #define load_TR_desc()                          native_load_tr_desc()
     110             : #define load_gdt(dtr)                           native_load_gdt(dtr)
     111             : #define load_idt(dtr)                           native_load_idt(dtr)
     112             : #define load_tr(tr)                             asm volatile("ltr %0"::"m" (tr))
     113             : #define load_ldt(ldt)                           asm volatile("lldt %0"::"m" (ldt))
     114             : 
     115             : #define store_gdt(dtr)                          native_store_gdt(dtr)
     116             : #define store_tr(tr)                            (tr = native_store_tr())
     117             : 
     118             : #define load_TLS(t, cpu)                        native_load_tls(t, cpu)
     119             : #define set_ldt                                 native_set_ldt
     120             : 
     121             : #define write_ldt_entry(dt, entry, desc)        native_write_ldt_entry(dt, entry, desc)
     122             : #define write_gdt_entry(dt, entry, desc, type)  native_write_gdt_entry(dt, entry, desc, type)
     123             : #define write_idt_entry(dt, entry, g)           native_write_idt_entry(dt, entry, g)
     124             : 
     125             : static inline void paravirt_alloc_ldt(struct desc_struct *ldt, unsigned entries)
     126             : {
     127             : }
     128             : 
     129             : static inline void paravirt_free_ldt(struct desc_struct *ldt, unsigned entries)
     130             : {
     131             : }
     132             : #endif  /* CONFIG_PARAVIRT_XXL */
     133             : 
     134             : #define store_ldt(ldt) asm("sldt %0" : "=m"(ldt))
     135             : 
     136         279 : static inline void native_write_idt_entry(gate_desc *idt, int entry, const gate_desc *gate)
     137             : {
     138         279 :         memcpy(&idt[entry], gate, sizeof(*gate));
     139             : }
     140             : 
     141             : static inline void native_write_ldt_entry(struct desc_struct *ldt, int entry, const void *desc)
     142             : {
     143             :         memcpy(&ldt[entry], desc, 8);
     144             : }
     145             : 
     146             : static inline void
     147          14 : native_write_gdt_entry(struct desc_struct *gdt, int entry, const void *desc, int type)
     148             : {
     149          14 :         unsigned int size;
     150             : 
     151          14 :         switch (type) {
     152             :         case DESC_TSS:  size = sizeof(tss_desc);        break;
     153             :         case DESC_LDT:  size = sizeof(ldt_desc);        break;
     154           7 :         default:        size = sizeof(*gdt);            break;
     155             :         }
     156             : 
     157          14 :         memcpy(&gdt[entry], desc, size);
     158             : }
     159             : 
     160           7 : static inline void set_tssldt_descriptor(void *d, unsigned long addr,
     161             :                                          unsigned type, unsigned size)
     162             : {
     163           7 :         struct ldttss_desc *desc = d;
     164             : 
     165           7 :         memset(desc, 0, sizeof(*desc));
     166             : 
     167           7 :         desc->limit0         = (u16) size;
     168           7 :         desc->base0          = (u16) addr;
     169           7 :         desc->base1          = (addr >> 16) & 0xFF;
     170           7 :         desc->type           = type;
     171           7 :         desc->p                      = 1;
     172           7 :         desc->limit1         = (size >> 16) & 0xF;
     173           7 :         desc->base2          = (addr >> 24) & 0xFF;
     174             : #ifdef CONFIG_X86_64
     175           7 :         desc->base3          = (u32) (addr >> 32);
     176             : #endif
     177           7 : }
     178             : 
     179           7 : static inline void __set_tss_desc(unsigned cpu, unsigned int entry, struct x86_hw_tss *addr)
     180             : {
     181           7 :         struct desc_struct *d = get_cpu_gdt_rw(cpu);
     182           7 :         tss_desc tss;
     183             : 
     184           7 :         set_tssldt_descriptor(&tss, (unsigned long)addr, DESC_TSS,
     185             :                               __KERNEL_TSS_LIMIT);
     186           7 :         write_gdt_entry(d, entry, &tss, DESC_TSS);
     187           7 : }
     188             : 
     189             : #define set_tss_desc(cpu, addr) __set_tss_desc(cpu, GDT_ENTRY_TSS, addr)
     190             : 
     191           4 : static inline void native_set_ldt(const void *addr, unsigned int entries)
     192             : {
     193           4 :         if (likely(entries == 0))
     194           4 :                 asm volatile("lldt %w0"::"q" (0));
     195             :         else {
     196           0 :                 unsigned cpu = smp_processor_id();
     197           0 :                 ldt_desc ldt;
     198             : 
     199           0 :                 set_tssldt_descriptor(&ldt, (unsigned long)addr, DESC_LDT,
     200           0 :                                       entries * LDT_ENTRY_SIZE - 1);
     201           0 :                 write_gdt_entry(get_cpu_gdt_rw(cpu), GDT_ENTRY_LDT,
     202             :                                 &ldt, DESC_LDT);
     203           0 :                 asm volatile("lldt %w0"::"q" (GDT_ENTRY_LDT*8));
     204             :         }
     205           4 : }
     206             : 
     207          10 : static inline void native_load_gdt(const struct desc_ptr *dtr)
     208             : {
     209          10 :         asm volatile("lgdt %0"::"m" (*dtr));
     210             : }
     211             : 
     212          13 : static __always_inline void native_load_idt(const struct desc_ptr *dtr)
     213             : {
     214          13 :         asm volatile("lidt %0"::"m" (*dtr));
     215             : }
     216             : 
     217           7 : static inline void native_store_gdt(struct desc_ptr *dtr)
     218             : {
     219           7 :         asm volatile("sgdt %0":"=m" (*dtr));
     220             : }
     221             : 
     222           0 : static inline void store_idt(struct desc_ptr *dtr)
     223             : {
     224           0 :         asm volatile("sidt %0":"=m" (*dtr));
     225             : }
     226             : 
     227             : /*
     228             :  * The LTR instruction marks the TSS GDT entry as busy. On 64-bit, the GDT is
     229             :  * a read-only remapping. To prevent a page fault, the GDT is switched to the
     230             :  * original writeable version when needed.
     231             :  */
     232             : #ifdef CONFIG_X86_64
     233           7 : static inline void native_load_tr_desc(void)
     234             : {
     235           7 :         struct desc_ptr gdt;
     236           7 :         int cpu = raw_smp_processor_id();
     237           7 :         bool restore = 0;
     238           7 :         struct desc_struct *fixmap_gdt;
     239             : 
     240           7 :         native_store_gdt(&gdt);
     241           7 :         fixmap_gdt = get_cpu_gdt_ro(cpu);
     242             : 
     243             :         /*
     244             :          * If the current GDT is the read-only fixmap, swap to the original
     245             :          * writeable version. Swap back at the end.
     246             :          */
     247           7 :         if (gdt.address == (unsigned long)fixmap_gdt) {
     248           0 :                 load_direct_gdt(cpu);
     249           0 :                 restore = 1;
     250             :         }
     251           0 :         asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
     252           7 :         if (restore)
     253           0 :                 load_fixmap_gdt(cpu);
     254           7 : }
     255             : #else
     256             : static inline void native_load_tr_desc(void)
     257             : {
     258             :         asm volatile("ltr %w0"::"q" (GDT_ENTRY_TSS*8));
     259             : }
     260             : #endif
     261             : 
     262           0 : static inline unsigned long native_store_tr(void)
     263             : {
     264           0 :         unsigned long tr;
     265             : 
     266           0 :         asm volatile("str %0":"=r" (tr));
     267             : 
     268           0 :         return tr;
     269             : }
     270             : 
     271       53465 : static inline void native_load_tls(struct thread_struct *t, unsigned int cpu)
     272             : {
     273       53465 :         struct desc_struct *gdt = get_cpu_gdt_rw(cpu);
     274       53465 :         unsigned int i;
     275             : 
     276      213822 :         for (i = 0; i < GDT_ENTRY_TLS_ENTRIES; i++)
     277      160357 :                 gdt[GDT_ENTRY_TLS_MIN + i] = t->tls_array[i];
     278       53465 : }
     279             : 
     280             : DECLARE_PER_CPU(bool, __tss_limit_invalid);
     281             : 
     282             : static inline void force_reload_TR(void)
     283             : {
     284             :         struct desc_struct *d = get_current_gdt_rw();
     285             :         tss_desc tss;
     286             : 
     287             :         memcpy(&tss, &d[GDT_ENTRY_TSS], sizeof(tss_desc));
     288             : 
     289             :         /*
     290             :          * LTR requires an available TSS, and the TSS is currently
     291             :          * busy.  Make it be available so that LTR will work.
     292             :          */
     293             :         tss.type = DESC_TSS;
     294             :         write_gdt_entry(d, GDT_ENTRY_TSS, &tss, DESC_TSS);
     295             : 
     296             :         load_TR_desc();
     297             :         this_cpu_write(__tss_limit_invalid, false);
     298             : }
     299             : 
     300             : /*
     301             :  * Call this if you need the TSS limit to be correct, which should be the case
     302             :  * if and only if you have TIF_IO_BITMAP set or you're switching to a task
     303             :  * with TIF_IO_BITMAP set.
     304             :  */
     305             : static inline void refresh_tss_limit(void)
     306             : {
     307             :         DEBUG_LOCKS_WARN_ON(preemptible());
     308             : 
     309             :         if (unlikely(this_cpu_read(__tss_limit_invalid)))
     310             :                 force_reload_TR();
     311             : }
     312             : 
     313             : /*
     314             :  * If you do something evil that corrupts the cached TSS limit (I'm looking
     315             :  * at you, VMX exits), call this function.
     316             :  *
     317             :  * The optimization here is that the TSS limit only matters for Linux if the
     318             :  * IO bitmap is in use.  If the TSS limit gets forced to its minimum value,
     319             :  * everything works except that IO bitmap will be ignored and all CPL 3 IO
     320             :  * instructions will #GP, which is exactly what we want for normal tasks.
     321             :  */
     322             : static inline void invalidate_tss_limit(void)
     323             : {
     324             :         DEBUG_LOCKS_WARN_ON(preemptible());
     325             : 
     326             :         if (unlikely(test_thread_flag(TIF_IO_BITMAP)))
     327             :                 force_reload_TR();
     328             :         else
     329             :                 this_cpu_write(__tss_limit_invalid, true);
     330             : }
     331             : 
     332             : /* This intentionally ignores lm, since 32-bit apps don't have that field. */
     333             : #define LDT_empty(info)                                 \
     334             :         ((info)->base_addr           == 0    &&      \
     335             :          (info)->limit                       == 0    &&      \
     336             :          (info)->contents            == 0    &&      \
     337             :          (info)->read_exec_only              == 1    &&      \
     338             :          (info)->seg_32bit           == 0    &&      \
     339             :          (info)->limit_in_pages              == 0    &&      \
     340             :          (info)->seg_not_present     == 1    &&      \
     341             :          (info)->useable             == 0)
     342             : 
     343             : /* Lots of programs expect an all-zero user_desc to mean "no segment at all". */
     344           0 : static inline bool LDT_zero(const struct user_desc *info)
     345             : {
     346           0 :         return (info->base_addr              == 0 &&
     347           0 :                 info->limit          == 0 &&
     348             :                 info->contents               == 0 &&
     349             :                 info->read_exec_only == 0 &&
     350             :                 info->seg_32bit              == 0 &&
     351             :                 info->limit_in_pages == 0 &&
     352           0 :                 info->seg_not_present        == 0 &&
     353             :                 info->useable                == 0);
     354             : }
     355             : 
     356           4 : static inline void clear_LDT(void)
     357             : {
     358           4 :         set_ldt(NULL, 0);
     359             : }
     360             : 
     361           0 : static inline unsigned long get_desc_base(const struct desc_struct *desc)
     362             : {
     363           0 :         return (unsigned)(desc->base0 | ((desc->base1) << 16) | ((desc->base2) << 24));
     364             : }
     365             : 
     366             : static inline void set_desc_base(struct desc_struct *desc, unsigned long base)
     367             : {
     368             :         desc->base0 = base & 0xffff;
     369             :         desc->base1 = (base >> 16) & 0xff;
     370             :         desc->base2 = (base >> 24) & 0xff;
     371             : }
     372             : 
     373           0 : static inline unsigned long get_desc_limit(const struct desc_struct *desc)
     374             : {
     375           0 :         return desc->limit0 | (desc->limit1 << 16);
     376             : }
     377             : 
     378             : static inline void set_desc_limit(struct desc_struct *desc, unsigned long limit)
     379             : {
     380             :         desc->limit0 = limit & 0xffff;
     381             :         desc->limit1 = (limit >> 16) & 0xf;
     382             : }
     383             : 
     384             : void alloc_intr_gate(unsigned int n, const void *addr);
     385             : 
     386         242 : static inline void init_idt_data(struct idt_data *data, unsigned int n,
     387             :                                  const void *addr)
     388             : {
     389         242 :         BUG_ON(n > 0xFF);
     390             : 
     391         242 :         memset(data, 0, sizeof(*data));
     392         242 :         data->vector = n;
     393         242 :         data->addr   = addr;
     394         242 :         data->segment        = __KERNEL_CS;
     395         242 :         data->bits.type      = GATE_INTERRUPT;
     396         242 :         data->bits.p = 1;
     397         242 : }
     398             : 
     399         279 : static inline void idt_init_desc(gate_desc *gate, const struct idt_data *d)
     400             : {
     401         279 :         unsigned long addr = (unsigned long) d->addr;
     402             : 
     403         279 :         gate->offset_low     = (u16) addr;
     404         279 :         gate->segment                = (u16) d->segment;
     405         279 :         gate->bits           = d->bits;
     406         279 :         gate->offset_middle  = (u16) (addr >> 16);
     407             : #ifdef CONFIG_X86_64
     408         279 :         gate->offset_high    = (u32) (addr >> 32);
     409         279 :         gate->reserved               = 0;
     410             : #endif
     411             : }
     412             : 
     413             : extern unsigned long system_vectors[];
     414             : 
     415             : extern void load_current_idt(void);
     416             : extern void idt_setup_early_handler(void);
     417             : extern void idt_setup_early_traps(void);
     418             : extern void idt_setup_traps(void);
     419             : extern void idt_setup_apic_and_irq_gates(void);
     420             : extern bool idt_is_f00f_address(unsigned long address);
     421             : 
     422             : #ifdef CONFIG_X86_64
     423             : extern void idt_setup_early_pf(void);
     424             : extern void idt_setup_ist_traps(void);
     425             : #else
     426             : static inline void idt_setup_early_pf(void) { }
     427             : static inline void idt_setup_ist_traps(void) { }
     428             : #endif
     429             : 
     430             : extern void idt_invalidate(void *addr);
     431             : 
     432             : #endif /* _ASM_X86_DESC_H */

Generated by: LCOV version 1.14