LCOV - code coverage report
Current view: top level - mm - mprotect.c (source / functions) Hit Total Coverage
Test: landlock.info Lines: 212 302 70.2 %
Date: 2021-04-22 12:43:58 Functions: 10 14 71.4 %

          Line data    Source code
       1             : // SPDX-License-Identifier: GPL-2.0
       2             : /*
       3             :  *  mm/mprotect.c
       4             :  *
       5             :  *  (C) Copyright 1994 Linus Torvalds
       6             :  *  (C) Copyright 2002 Christoph Hellwig
       7             :  *
       8             :  *  Address space accounting code       <alan@lxorguk.ukuu.org.uk>
       9             :  *  (C) Copyright 2002 Red Hat Inc, All Rights Reserved
      10             :  */
      11             : 
      12             : #include <linux/pagewalk.h>
      13             : #include <linux/hugetlb.h>
      14             : #include <linux/shm.h>
      15             : #include <linux/mman.h>
      16             : #include <linux/fs.h>
      17             : #include <linux/highmem.h>
      18             : #include <linux/security.h>
      19             : #include <linux/mempolicy.h>
      20             : #include <linux/personality.h>
      21             : #include <linux/syscalls.h>
      22             : #include <linux/swap.h>
      23             : #include <linux/swapops.h>
      24             : #include <linux/mmu_notifier.h>
      25             : #include <linux/migrate.h>
      26             : #include <linux/perf_event.h>
      27             : #include <linux/pkeys.h>
      28             : #include <linux/ksm.h>
      29             : #include <linux/uaccess.h>
      30             : #include <linux/mm_inline.h>
      31             : #include <linux/pgtable.h>
      32             : #include <asm/cacheflush.h>
      33             : #include <asm/mmu_context.h>
      34             : #include <asm/tlbflush.h>
      35             : 
      36             : #include "internal.h"
      37             : 
      38        7963 : static unsigned long change_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
      39             :                 unsigned long addr, unsigned long end, pgprot_t newprot,
      40             :                 unsigned long cp_flags)
      41             : {
      42        7963 :         pte_t *pte, oldpte;
      43        7963 :         spinlock_t *ptl;
      44        7963 :         unsigned long pages = 0;
      45        7963 :         int target_node = NUMA_NO_NODE;
      46        7963 :         bool dirty_accountable = cp_flags & MM_CP_DIRTY_ACCT;
      47        7963 :         bool prot_numa = cp_flags & MM_CP_PROT_NUMA;
      48        7963 :         bool uffd_wp = cp_flags & MM_CP_UFFD_WP;
      49        7963 :         bool uffd_wp_resolve = cp_flags & MM_CP_UFFD_WP_RESOLVE;
      50             : 
      51             :         /*
      52             :          * Can be called with only the mmap_lock for reading by
      53             :          * prot_numa so we must check the pmd isn't constantly
      54             :          * changing from under us from pmd_none to pmd_trans_huge
      55             :          * and/or the other way around.
      56             :          */
      57        7963 :         if (pmd_trans_unstable(pmd))
      58             :                 return 0;
      59             : 
      60             :         /*
      61             :          * The pmd points to a regular pte so the pmd can't change
      62             :          * from under us even if the mmap_lock is only hold for
      63             :          * reading.
      64             :          */
      65       15926 :         pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
      66             : 
      67             :         /* Get target node for single threaded private VMAs */
      68        7963 :         if (prot_numa && !(vma->vm_flags & VM_SHARED) &&
      69           0 :             atomic_read(&vma->vm_mm->mm_users) == 1)
      70           0 :                 target_node = numa_node_id();
      71             : 
      72        7963 :         flush_tlb_batched_pending(vma->vm_mm);
      73      384085 :         arch_enter_lazy_mmu_mode();
      74      384085 :         do {
      75      384085 :                 oldpte = *pte;
      76      384085 :                 if (pte_present(oldpte)) {
      77       14563 :                         pte_t ptent;
      78       14563 :                         bool preserve_write = prot_numa && pte_write(oldpte);
      79             : 
      80             :                         /*
      81             :                          * Avoid trapping faults against the zero or KSM
      82             :                          * pages. See similar comment in change_huge_pmd.
      83             :                          */
      84       14563 :                         if (prot_numa) {
      85           0 :                                 struct page *page;
      86             : 
      87             :                                 /* Avoid TLB flush if possible */
      88           0 :                                 if (pte_protnone(oldpte))
      89      384085 :                                         continue;
      90             : 
      91           0 :                                 page = vm_normal_page(vma, addr, oldpte);
      92           0 :                                 if (!page || PageKsm(page))
      93           0 :                                         continue;
      94             : 
      95             :                                 /* Also skip shared copy-on-write pages */
      96           0 :                                 if (is_cow_mapping(vma->vm_flags) &&
      97           0 :                                     page_mapcount(page) != 1)
      98           0 :                                         continue;
      99             : 
     100             :                                 /*
     101             :                                  * While migration can move some dirty pages,
     102             :                                  * it cannot move them all from MIGRATE_ASYNC
     103             :                                  * context.
     104             :                                  */
     105           0 :                                 if (page_is_file_lru(page) && PageDirty(page))
     106           0 :                                         continue;
     107             : 
     108             :                                 /*
     109             :                                  * Don't mess with PTEs if page is already on the node
     110             :                                  * a single-threaded process is running on.
     111             :                                  */
     112           0 :                                 if (target_node == page_to_nid(page))
     113           0 :                                         continue;
     114             :                         }
     115             : 
     116       14563 :                         oldpte = ptep_modify_prot_start(vma, addr, pte);
     117       14563 :                         ptent = pte_modify(oldpte, newprot);
     118       14563 :                         if (preserve_write)
     119           0 :                                 ptent = pte_mk_savedwrite(ptent);
     120             : 
     121       14563 :                         if (uffd_wp) {
     122           0 :                                 ptent = pte_wrprotect(ptent);
     123           0 :                                 ptent = pte_mkuffd_wp(ptent);
     124             :                         } else if (uffd_wp_resolve) {
     125             :                                 /*
     126             :                                  * Leave the write bit to be handled
     127             :                                  * by PF interrupt handler, then
     128             :                                  * things like COW could be properly
     129             :                                  * handled.
     130             :                                  */
     131       14563 :                                 ptent = pte_clear_uffd_wp(ptent);
     132             :                         }
     133             : 
     134             :                         /* Avoid taking write faults for known dirty pages */
     135       14563 :                         if (dirty_accountable && pte_dirty(ptent) &&
     136             :                                         (pte_soft_dirty(ptent) ||
     137             :                                          !(vma->vm_flags & VM_SOFTDIRTY))) {
     138           0 :                                 ptent = pte_mkwrite(ptent);
     139             :                         }
     140       14563 :                         ptep_modify_prot_commit(vma, addr, pte, oldpte, ptent);
     141       14563 :                         pages++;
     142      369522 :                 } else if (is_swap_pte(oldpte)) {
     143           0 :                         swp_entry_t entry = pte_to_swp_entry(oldpte);
     144           0 :                         pte_t newpte;
     145             : 
     146           0 :                         if (is_write_migration_entry(entry)) {
     147             :                                 /*
     148             :                                  * A protection check is difficult so
     149             :                                  * just be safe and disable write
     150             :                                  */
     151           0 :                                 make_migration_entry_read(&entry);
     152           0 :                                 newpte = swp_entry_to_pte(entry);
     153           0 :                                 if (pte_swp_soft_dirty(oldpte))
     154             :                                         newpte = pte_swp_mksoft_dirty(newpte);
     155           0 :                                 if (pte_swp_uffd_wp(oldpte))
     156             :                                         newpte = pte_swp_mkuffd_wp(newpte);
     157             :                         } else if (is_write_device_private_entry(entry)) {
     158             :                                 /*
     159             :                                  * We do not preserve soft-dirtiness. See
     160             :                                  * copy_one_pte() for explanation.
     161             :                                  */
     162             :                                 make_device_private_entry_read(&entry);
     163             :                                 newpte = swp_entry_to_pte(entry);
     164             :                                 if (pte_swp_uffd_wp(oldpte))
     165             :                                         newpte = pte_swp_mkuffd_wp(newpte);
     166             :                         } else {
     167           0 :                                 newpte = oldpte;
     168             :                         }
     169             : 
     170           0 :                         if (uffd_wp)
     171           0 :                                 newpte = pte_swp_mkuffd_wp(newpte);
     172             :                         else if (uffd_wp_resolve)
     173           0 :                                 newpte = pte_swp_clear_uffd_wp(newpte);
     174             : 
     175           0 :                         if (!pte_same(oldpte, newpte)) {
     176           0 :                                 set_pte_at(vma->vm_mm, addr, pte, newpte);
     177           0 :                                 pages++;
     178             :                         }
     179             :                 }
     180      384085 :         } while (pte++, addr += PAGE_SIZE, addr != end);
     181        7963 :         arch_leave_lazy_mmu_mode();
     182        7963 :         pte_unmap_unlock(pte - 1, ptl);
     183             : 
     184        7963 :         return pages;
     185             : }
     186             : 
     187             : /*
     188             :  * Used when setting automatic NUMA hinting protection where it is
     189             :  * critical that a numa hinting PMD is not confused with a bad PMD.
     190             :  */
     191        9293 : static inline int pmd_none_or_clear_bad_unless_trans_huge(pmd_t *pmd)
     192             : {
     193        9293 :         pmd_t pmdval = pmd_read_atomic(pmd);
     194             : 
     195             :         /* See pmd_none_or_trans_huge_or_clear_bad for info on barrier */
     196             : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
     197        9293 :         barrier();
     198             : #endif
     199             : 
     200        9293 :         if (pmd_none(pmdval))
     201             :                 return 1;
     202        7963 :         if (pmd_trans_huge(pmdval))
     203             :                 return 0;
     204       15926 :         if (unlikely(pmd_bad(pmdval))) {
     205           0 :                 pmd_clear_bad(pmd);
     206           0 :                 return 1;
     207             :         }
     208             : 
     209             :         return 0;
     210             : }
     211             : 
     212        7977 : static inline unsigned long change_pmd_range(struct vm_area_struct *vma,
     213             :                 pud_t *pud, unsigned long addr, unsigned long end,
     214             :                 pgprot_t newprot, unsigned long cp_flags)
     215             : {
     216        7977 :         pmd_t *pmd;
     217        7977 :         unsigned long next;
     218        7977 :         unsigned long pages = 0;
     219        7977 :         unsigned long nr_huge_updates = 0;
     220        7977 :         struct mmu_notifier_range range;
     221             : 
     222        7977 :         range.start = 0;
     223             : 
     224       15954 :         pmd = pmd_offset(pud, addr);
     225        9293 :         do {
     226        9293 :                 unsigned long this_pages;
     227             : 
     228        9293 :                 next = pmd_addr_end(addr, end);
     229             : 
     230             :                 /*
     231             :                  * Automatic NUMA balancing walks the tables with mmap_lock
     232             :                  * held for read. It's possible a parallel update to occur
     233             :                  * between pmd_trans_huge() and a pmd_none_or_clear_bad()
     234             :                  * check leading to a false positive and clearing.
     235             :                  * Hence, it's necessary to atomically read the PMD value
     236             :                  * for all the checks.
     237             :                  */
     238       18586 :                 if (!is_swap_pmd(*pmd) && !pmd_devmap(*pmd) &&
     239        9293 :                      pmd_none_or_clear_bad_unless_trans_huge(pmd))
     240        1330 :                         goto next;
     241             : 
     242             :                 /* invoke the mmu notifier if the pmd is populated */
     243        7963 :                 if (!range.start) {
     244        7963 :                         mmu_notifier_range_init(&range,
     245             :                                 MMU_NOTIFY_PROTECTION_VMA, 0,
     246             :                                 vma, vma->vm_mm, addr, end);
     247        7963 :                         mmu_notifier_invalidate_range_start(&range);
     248             :                 }
     249             : 
     250        7963 :                 if (is_swap_pmd(*pmd) || pmd_trans_huge(*pmd) || pmd_devmap(*pmd)) {
     251           0 :                         if (next - addr != HPAGE_PMD_SIZE) {
     252           0 :                                 __split_huge_pmd(vma, pmd, addr, false, NULL);
     253             :                         } else {
     254           0 :                                 int nr_ptes = change_huge_pmd(vma, pmd, addr,
     255             :                                                               newprot, cp_flags);
     256             : 
     257           0 :                                 if (nr_ptes) {
     258           0 :                                         if (nr_ptes == HPAGE_PMD_NR) {
     259           0 :                                                 pages += HPAGE_PMD_NR;
     260           0 :                                                 nr_huge_updates++;
     261             :                                         }
     262             : 
     263             :                                         /* huge pmd was handled */
     264           0 :                                         goto next;
     265             :                                 }
     266             :                         }
     267             :                         /* fall through, the trans huge pmd just split */
     268             :                 }
     269        7963 :                 this_pages = change_pte_range(vma, pmd, addr, next, newprot,
     270             :                                               cp_flags);
     271        7963 :                 pages += this_pages;
     272        9293 : next:
     273        9293 :                 cond_resched();
     274        9293 :         } while (pmd++, addr = next, addr != end);
     275             : 
     276        7977 :         if (range.start)
     277        7977 :                 mmu_notifier_invalidate_range_end(&range);
     278             : 
     279        7977 :         if (nr_huge_updates)
     280        7977 :                 count_vm_numa_events(NUMA_HUGE_PTE_UPDATES, nr_huge_updates);
     281        7977 :         return pages;
     282             : }
     283             : 
     284        7977 : static inline unsigned long change_pud_range(struct vm_area_struct *vma,
     285             :                 p4d_t *p4d, unsigned long addr, unsigned long end,
     286             :                 pgprot_t newprot, unsigned long cp_flags)
     287             : {
     288        7977 :         pud_t *pud;
     289        7977 :         unsigned long next;
     290        7977 :         unsigned long pages = 0;
     291             : 
     292        7977 :         pud = pud_offset(p4d, addr);
     293        7980 :         do {
     294        7980 :                 next = pud_addr_end(addr, end);
     295        7980 :                 if (pud_none_or_clear_bad(pud))
     296           3 :                         continue;
     297        7977 :                 pages += change_pmd_range(vma, pud, addr, next, newprot,
     298             :                                           cp_flags);
     299        7980 :         } while (pud++, addr = next, addr != end);
     300             : 
     301        7977 :         return pages;
     302             : }
     303             : 
     304        7977 : static inline unsigned long change_p4d_range(struct vm_area_struct *vma,
     305             :                 pgd_t *pgd, unsigned long addr, unsigned long end,
     306             :                 pgprot_t newprot, unsigned long cp_flags)
     307             : {
     308        7977 :         p4d_t *p4d;
     309        7977 :         unsigned long next;
     310        7977 :         unsigned long pages = 0;
     311             : 
     312        7977 :         p4d = p4d_offset(pgd, addr);
     313        7977 :         do {
     314        7977 :                 next = p4d_addr_end(addr, end);
     315        7977 :                 if (p4d_none_or_clear_bad(p4d))
     316           0 :                         continue;
     317        7977 :                 pages += change_pud_range(vma, p4d, addr, next, newprot,
     318             :                                           cp_flags);
     319        7977 :         } while (p4d++, addr = next, addr != end);
     320             : 
     321        7977 :         return pages;
     322             : }
     323             : 
     324        7977 : static unsigned long change_protection_range(struct vm_area_struct *vma,
     325             :                 unsigned long addr, unsigned long end, pgprot_t newprot,
     326             :                 unsigned long cp_flags)
     327             : {
     328        7977 :         struct mm_struct *mm = vma->vm_mm;
     329        7977 :         pgd_t *pgd;
     330        7977 :         unsigned long next;
     331        7977 :         unsigned long start = addr;
     332        7977 :         unsigned long pages = 0;
     333             : 
     334        7977 :         BUG_ON(addr >= end);
     335        7977 :         pgd = pgd_offset(mm, addr);
     336        7977 :         flush_cache_range(vma, addr, end);
     337        7977 :         inc_tlb_flush_pending(mm);
     338        7977 :         do {
     339        7977 :                 next = pgd_addr_end(addr, end);
     340        7977 :                 if (pgd_none_or_clear_bad(pgd))
     341             :                         continue;
     342        7977 :                 pages += change_p4d_range(vma, pgd, addr, next, newprot,
     343             :                                           cp_flags);
     344        7977 :         } while (pgd++, addr = next, addr != end);
     345             : 
     346             :         /* Only flush the TLB if we actually modified any entries: */
     347        7977 :         if (pages)
     348        5970 :                 flush_tlb_range(vma, start, end);
     349        7977 :         dec_tlb_flush_pending(mm);
     350             : 
     351        7977 :         return pages;
     352             : }
     353             : 
     354        7977 : unsigned long change_protection(struct vm_area_struct *vma, unsigned long start,
     355             :                        unsigned long end, pgprot_t newprot,
     356             :                        unsigned long cp_flags)
     357             : {
     358        7977 :         unsigned long pages;
     359             : 
     360        7977 :         BUG_ON((cp_flags & MM_CP_UFFD_WP_ALL) == MM_CP_UFFD_WP_ALL);
     361             : 
     362        7977 :         if (is_vm_hugetlb_page(vma))
     363             :                 pages = hugetlb_change_protection(vma, start, end, newprot);
     364             :         else
     365        7977 :                 pages = change_protection_range(vma, start, end, newprot,
     366             :                                                 cp_flags);
     367             : 
     368        7977 :         return pages;
     369             : }
     370             : 
     371           0 : static int prot_none_pte_entry(pte_t *pte, unsigned long addr,
     372             :                                unsigned long next, struct mm_walk *walk)
     373             : {
     374           0 :         return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ?
     375           0 :                 0 : -EACCES;
     376             : }
     377             : 
     378           0 : static int prot_none_hugetlb_entry(pte_t *pte, unsigned long hmask,
     379             :                                    unsigned long addr, unsigned long next,
     380             :                                    struct mm_walk *walk)
     381             : {
     382           0 :         return pfn_modify_allowed(pte_pfn(*pte), *(pgprot_t *)(walk->private)) ?
     383           0 :                 0 : -EACCES;
     384             : }
     385             : 
     386           0 : static int prot_none_test(unsigned long addr, unsigned long next,
     387             :                           struct mm_walk *walk)
     388             : {
     389           0 :         return 0;
     390             : }
     391             : 
     392             : static const struct mm_walk_ops prot_none_walk_ops = {
     393             :         .pte_entry              = prot_none_pte_entry,
     394             :         .hugetlb_entry          = prot_none_hugetlb_entry,
     395             :         .test_walk              = prot_none_test,
     396             : };
     397             : 
     398             : int
     399        8919 : mprotect_fixup(struct vm_area_struct *vma, struct vm_area_struct **pprev,
     400             :         unsigned long start, unsigned long end, unsigned long newflags)
     401             : {
     402        8919 :         struct mm_struct *mm = vma->vm_mm;
     403        8919 :         unsigned long oldflags = vma->vm_flags;
     404        8919 :         long nrpages = (end - start) >> PAGE_SHIFT;
     405        8919 :         unsigned long charged = 0;
     406        8919 :         pgoff_t pgoff;
     407        8919 :         int error;
     408        8919 :         int dirty_accountable = 0;
     409             : 
     410        8919 :         if (newflags == oldflags) {
     411         942 :                 *pprev = vma;
     412         942 :                 return 0;
     413             :         }
     414             : 
     415             :         /*
     416             :          * Do PROT_NONE PFN permission checks here when we can still
     417             :          * bail out without undoing a lot of state. This is a rather
     418             :          * uncommon case, so doesn't need to be very optimized.
     419             :          */
     420        7977 :         if (arch_has_pfn_modify_check() &&
     421        7977 :             (vma->vm_flags & (VM_PFNMAP|VM_MIXEDMAP)) &&
     422           0 :             (newflags & VM_ACCESS_FLAGS) == 0) {
     423           0 :                 pgprot_t new_pgprot = vm_get_page_prot(newflags);
     424             : 
     425           0 :                 error = walk_page_range(current->mm, start, end,
     426             :                                 &prot_none_walk_ops, &new_pgprot);
     427           0 :                 if (error)
     428           0 :                         return error;
     429             :         }
     430             : 
     431             :         /*
     432             :          * If we make a private mapping writable we increase our commit;
     433             :          * but (without finer accounting) cannot reduce our commit if we
     434             :          * make it unwritable again. hugetlb mapping were accounted for
     435             :          * even if read-only so there is no need to account for them here
     436             :          */
     437        7977 :         if (newflags & VM_WRITE) {
     438             :                 /* Check space limits when area turns into data. */
     439          33 :                 if (!may_expand_vm(mm, newflags, nrpages) &&
     440           0 :                                 may_expand_vm(mm, oldflags, nrpages))
     441             :                         return -ENOMEM;
     442          33 :                 if (!(oldflags & (VM_ACCOUNT|VM_WRITE|VM_HUGETLB|
     443             :                                                 VM_SHARED|VM_NORESERVE))) {
     444           6 :                         charged = nrpages;
     445           6 :                         if (security_vm_enough_memory_mm(mm, charged))
     446             :                                 return -ENOMEM;
     447           6 :                         newflags |= VM_ACCOUNT;
     448             :                 }
     449             :         }
     450             : 
     451             :         /*
     452             :          * First try to merge with previous and/or next vma.
     453             :          */
     454        7977 :         pgoff = vma->vm_pgoff + ((start - vma->vm_start) >> PAGE_SHIFT);
     455        7977 :         *pprev = vma_merge(mm, *pprev, start, end, newflags,
     456             :                            vma->anon_vma, vma->vm_file, pgoff, vma_policy(vma),
     457             :                            vma->vm_userfaultfd_ctx);
     458        7977 :         if (*pprev) {
     459          24 :                 vma = *pprev;
     460          24 :                 VM_WARN_ON((vma->vm_flags ^ newflags) & ~VM_SOFTDIRTY);
     461          24 :                 goto success;
     462             :         }
     463             : 
     464        7953 :         *pprev = vma;
     465             : 
     466        7953 :         if (start != vma->vm_start) {
     467        1978 :                 error = split_vma(mm, vma, start, 1);
     468        1978 :                 if (error)
     469           0 :                         goto fail;
     470             :         }
     471             : 
     472        7953 :         if (end != vma->vm_end) {
     473        7953 :                 error = split_vma(mm, vma, end, 0);
     474        7953 :                 if (error)
     475           0 :                         goto fail;
     476             :         }
     477             : 
     478        7953 : success:
     479             :         /*
     480             :          * vm_flags and vm_page_prot are protected by the mmap_lock
     481             :          * held in write mode.
     482             :          */
     483        7977 :         vma->vm_flags = newflags;
     484        7977 :         dirty_accountable = vma_wants_writenotify(vma, vma->vm_page_prot);
     485        7977 :         vma_set_page_prot(vma);
     486             : 
     487       15954 :         change_protection(vma, start, end, vma->vm_page_prot,
     488             :                           dirty_accountable ? MM_CP_DIRTY_ACCT : 0);
     489             : 
     490             :         /*
     491             :          * Private VM_LOCKED VMA becoming writable: trigger COW to avoid major
     492             :          * fault on access.
     493             :          */
     494        7977 :         if ((oldflags & (VM_WRITE | VM_SHARED | VM_LOCKED)) == VM_LOCKED &&
     495           0 :                         (newflags & VM_WRITE)) {
     496           0 :                 populate_vma_page_range(vma, start, end, NULL);
     497             :         }
     498             : 
     499        7977 :         vm_stat_account(mm, oldflags, -nrpages);
     500        7977 :         vm_stat_account(mm, newflags, nrpages);
     501        7977 :         perf_event_mmap(vma);
     502        7977 :         return 0;
     503             : 
     504           0 : fail:
     505           0 :         vm_unacct_memory(charged);
     506           0 :         return error;
     507             : }
     508             : 
     509             : /*
     510             :  * pkey==-1 when doing a legacy mprotect()
     511             :  */
     512        7977 : static int do_mprotect_pkey(unsigned long start, size_t len,
     513             :                 unsigned long prot, int pkey)
     514             : {
     515        7977 :         unsigned long nstart, end, tmp, reqprot;
     516        7977 :         struct vm_area_struct *vma, *prev;
     517        7977 :         int error = -EINVAL;
     518        7977 :         const int grows = prot & (PROT_GROWSDOWN|PROT_GROWSUP);
     519        7977 :         const bool rier = (current->personality & READ_IMPLIES_EXEC) &&
     520           0 :                                 (prot & PROT_READ);
     521             : 
     522        7977 :         start = untagged_addr(start);
     523             : 
     524        7977 :         prot &= ~(PROT_GROWSDOWN|PROT_GROWSUP);
     525        7977 :         if (grows == (PROT_GROWSDOWN|PROT_GROWSUP)) /* can't be both */
     526             :                 return -EINVAL;
     527             : 
     528        7977 :         if (start & ~PAGE_MASK)
     529             :                 return -EINVAL;
     530        7977 :         if (!len)
     531             :                 return 0;
     532        7977 :         len = PAGE_ALIGN(len);
     533        7977 :         end = start + len;
     534        7977 :         if (end <= start)
     535             :                 return -ENOMEM;
     536        7977 :         if (!arch_validate_prot(prot, start))
     537             :                 return -EINVAL;
     538             : 
     539        7977 :         reqprot = prot;
     540             : 
     541        7977 :         if (mmap_write_lock_killable(current->mm))
     542             :                 return -EINTR;
     543             : 
     544             :         /*
     545             :          * If userspace did not allocate the pkey, do not let
     546             :          * them use it here.
     547             :          */
     548        7977 :         error = -EINVAL;
     549        7977 :         if ((pkey != -1) && !mm_pkey_is_allocated(current->mm, pkey))
     550           0 :                 goto out;
     551             : 
     552        7977 :         vma = find_vma(current->mm, start);
     553        7977 :         error = -ENOMEM;
     554        7977 :         if (!vma)
     555           0 :                 goto out;
     556        7977 :         prev = vma->vm_prev;
     557        7977 :         if (unlikely(grows & PROT_GROWSDOWN)) {
     558           0 :                 if (vma->vm_start >= end)
     559           0 :                         goto out;
     560           0 :                 start = vma->vm_start;
     561           0 :                 error = -EINVAL;
     562           0 :                 if (!(vma->vm_flags & VM_GROWSDOWN))
     563           0 :                         goto out;
     564             :         } else {
     565        7977 :                 if (vma->vm_start > start)
     566           0 :                         goto out;
     567        7977 :                 if (unlikely(grows & PROT_GROWSUP)) {
     568           0 :                         end = vma->vm_end;
     569           0 :                         error = -EINVAL;
     570           0 :                         if (!(vma->vm_flags & VM_GROWSUP))
     571           0 :                                 goto out;
     572             :                 }
     573             :         }
     574        7977 :         if (start > vma->vm_start)
     575        1981 :                 prev = vma;
     576             : 
     577             :         for (nstart = start ; ; ) {
     578        7977 :                 unsigned long mask_off_old_flags;
     579        7977 :                 unsigned long newflags;
     580        7977 :                 int new_vma_pkey;
     581             : 
     582             :                 /* Here we know that vma->vm_start <= nstart < vma->vm_end. */
     583             : 
     584             :                 /* Does the application expect PROT_READ to imply PROT_EXEC */
     585        7977 :                 if (rier && (vma->vm_flags & VM_MAYEXEC))
     586           0 :                         prot |= PROT_EXEC;
     587             : 
     588             :                 /*
     589             :                  * Each mprotect() call explicitly passes r/w/x permissions.
     590             :                  * If a permission is not passed to mprotect(), it must be
     591             :                  * cleared from the VMA.
     592             :                  */
     593        7977 :                 mask_off_old_flags = VM_READ | VM_WRITE | VM_EXEC |
     594             :                                         VM_FLAGS_CLEAR;
     595             : 
     596        7977 :                 new_vma_pkey = arch_override_mprotect_pkey(vma, prot, pkey);
     597        7977 :                 newflags = calc_vm_prot_bits(prot, new_vma_pkey);
     598        7977 :                 newflags |= (vma->vm_flags & ~mask_off_old_flags);
     599             : 
     600             :                 /* newflags >> 4 shift VM_MAY% in place of VM_% */
     601        7977 :                 if ((newflags & ~(newflags >> 4)) & VM_ACCESS_FLAGS) {
     602           0 :                         error = -EACCES;
     603           0 :                         goto out;
     604             :                 }
     605             : 
     606             :                 /* Allow architectures to sanity-check the new flags */
     607        7977 :                 if (!arch_validate_flags(newflags)) {
     608             :                         error = -EINVAL;
     609             :                         goto out;
     610             :                 }
     611             : 
     612        7977 :                 error = security_file_mprotect(vma, reqprot, prot);
     613        7977 :                 if (error)
     614           0 :                         goto out;
     615             : 
     616        7977 :                 tmp = vma->vm_end;
     617        7977 :                 if (tmp > end)
     618             :                         tmp = end;
     619             : 
     620        7977 :                 if (vma->vm_ops && vma->vm_ops->mprotect) {
     621           0 :                         error = vma->vm_ops->mprotect(vma, nstart, tmp, newflags);
     622           0 :                         if (error)
     623           0 :                                 goto out;
     624             :                 }
     625             : 
     626        7977 :                 error = mprotect_fixup(vma, &prev, nstart, tmp, newflags);
     627        7977 :                 if (error)
     628           0 :                         goto out;
     629             : 
     630        7977 :                 nstart = tmp;
     631             : 
     632        7977 :                 if (nstart < prev->vm_end)
     633             :                         nstart = prev->vm_end;
     634        7977 :                 if (nstart >= end)
     635        7977 :                         goto out;
     636             : 
     637           0 :                 vma = prev->vm_next;
     638           0 :                 if (!vma || vma->vm_start != nstart) {
     639           0 :                         error = -ENOMEM;
     640           0 :                         goto out;
     641             :                 }
     642             :                 prot = reqprot;
     643             :         }
     644        7977 : out:
     645        7977 :         mmap_write_unlock(current->mm);
     646        7977 :         return error;
     647             : }
     648             : 
     649       15954 : SYSCALL_DEFINE3(mprotect, unsigned long, start, size_t, len,
     650             :                 unsigned long, prot)
     651             : {
     652        7977 :         return do_mprotect_pkey(start, len, prot, -1);
     653             : }
     654             : 
     655             : #ifdef CONFIG_ARCH_HAS_PKEYS
     656             : 
     657             : SYSCALL_DEFINE4(pkey_mprotect, unsigned long, start, size_t, len,
     658             :                 unsigned long, prot, int, pkey)
     659             : {
     660             :         return do_mprotect_pkey(start, len, prot, pkey);
     661             : }
     662             : 
     663             : SYSCALL_DEFINE2(pkey_alloc, unsigned long, flags, unsigned long, init_val)
     664             : {
     665             :         int pkey;
     666             :         int ret;
     667             : 
     668             :         /* No flags supported yet. */
     669             :         if (flags)
     670             :                 return -EINVAL;
     671             :         /* check for unsupported init values */
     672             :         if (init_val & ~PKEY_ACCESS_MASK)
     673             :                 return -EINVAL;
     674             : 
     675             :         mmap_write_lock(current->mm);
     676             :         pkey = mm_pkey_alloc(current->mm);
     677             : 
     678             :         ret = -ENOSPC;
     679             :         if (pkey == -1)
     680             :                 goto out;
     681             : 
     682             :         ret = arch_set_user_pkey_access(current, pkey, init_val);
     683             :         if (ret) {
     684             :                 mm_pkey_free(current->mm, pkey);
     685             :                 goto out;
     686             :         }
     687             :         ret = pkey;
     688             : out:
     689             :         mmap_write_unlock(current->mm);
     690             :         return ret;
     691             : }
     692             : 
     693             : SYSCALL_DEFINE1(pkey_free, int, pkey)
     694             : {
     695             :         int ret;
     696             : 
     697             :         mmap_write_lock(current->mm);
     698             :         ret = mm_pkey_free(current->mm, pkey);
     699             :         mmap_write_unlock(current->mm);
     700             : 
     701             :         /*
     702             :          * We could provie warnings or errors if any VMA still
     703             :          * has the pkey set here.
     704             :          */
     705             :         return ret;
     706             : }
     707             : 
     708             : #endif /* CONFIG_ARCH_HAS_PKEYS */

Generated by: LCOV version 1.14