Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0
2 : #include <linux/pagewalk.h>
3 : #include <linux/vmacache.h>
4 : #include <linux/hugetlb.h>
5 : #include <linux/huge_mm.h>
6 : #include <linux/mount.h>
7 : #include <linux/seq_file.h>
8 : #include <linux/highmem.h>
9 : #include <linux/ptrace.h>
10 : #include <linux/slab.h>
11 : #include <linux/pagemap.h>
12 : #include <linux/mempolicy.h>
13 : #include <linux/rmap.h>
14 : #include <linux/swap.h>
15 : #include <linux/sched/mm.h>
16 : #include <linux/swapops.h>
17 : #include <linux/mmu_notifier.h>
18 : #include <linux/page_idle.h>
19 : #include <linux/shmem_fs.h>
20 : #include <linux/uaccess.h>
21 : #include <linux/pkeys.h>
22 :
23 : #include <asm/elf.h>
24 : #include <asm/tlb.h>
25 : #include <asm/tlbflush.h>
26 : #include "internal.h"
27 :
28 : #define SEQ_PUT_DEC(str, val) \
29 : seq_put_decimal_ull_width(m, str, (val) << (PAGE_SHIFT-10), 8)
30 55 : void task_mem(struct seq_file *m, struct mm_struct *mm)
31 : {
32 55 : unsigned long text, lib, swap, anon, file, shmem;
33 55 : unsigned long hiwater_vm, total_vm, hiwater_rss, total_rss;
34 :
35 55 : anon = get_mm_counter(mm, MM_ANONPAGES);
36 55 : file = get_mm_counter(mm, MM_FILEPAGES);
37 55 : shmem = get_mm_counter(mm, MM_SHMEMPAGES);
38 :
39 : /*
40 : * Note: to minimize their overhead, mm maintains hiwater_vm and
41 : * hiwater_rss only when about to *lower* total_vm or rss. Any
42 : * collector of these hiwater stats must therefore get total_vm
43 : * and rss too, which will usually be the higher. Barriers? not
44 : * worth the effort, such snapshots can always be inconsistent.
45 : */
46 55 : hiwater_vm = total_vm = mm->total_vm;
47 55 : if (hiwater_vm < mm->hiwater_vm)
48 : hiwater_vm = mm->hiwater_vm;
49 55 : hiwater_rss = total_rss = anon + file + shmem;
50 55 : if (hiwater_rss < mm->hiwater_rss)
51 : hiwater_rss = mm->hiwater_rss;
52 :
53 : /* split executable areas between text and lib */
54 55 : text = PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK);
55 55 : text = min(text, mm->exec_vm << PAGE_SHIFT);
56 55 : lib = (mm->exec_vm << PAGE_SHIFT) - text;
57 :
58 55 : swap = get_mm_counter(mm, MM_SWAPENTS);
59 55 : SEQ_PUT_DEC("VmPeak:\t", hiwater_vm);
60 55 : SEQ_PUT_DEC(" kB\nVmSize:\t", total_vm);
61 55 : SEQ_PUT_DEC(" kB\nVmLck:\t", mm->locked_vm);
62 55 : SEQ_PUT_DEC(" kB\nVmPin:\t", atomic64_read(&mm->pinned_vm));
63 55 : SEQ_PUT_DEC(" kB\nVmHWM:\t", hiwater_rss);
64 55 : SEQ_PUT_DEC(" kB\nVmRSS:\t", total_rss);
65 55 : SEQ_PUT_DEC(" kB\nRssAnon:\t", anon);
66 55 : SEQ_PUT_DEC(" kB\nRssFile:\t", file);
67 55 : SEQ_PUT_DEC(" kB\nRssShmem:\t", shmem);
68 55 : SEQ_PUT_DEC(" kB\nVmData:\t", mm->data_vm);
69 55 : SEQ_PUT_DEC(" kB\nVmStk:\t", mm->stack_vm);
70 55 : seq_put_decimal_ull_width(m,
71 55 : " kB\nVmExe:\t", text >> 10, 8);
72 55 : seq_put_decimal_ull_width(m,
73 55 : " kB\nVmLib:\t", lib >> 10, 8);
74 55 : seq_put_decimal_ull_width(m,
75 55 : " kB\nVmPTE:\t", mm_pgtables_bytes(mm) >> 10, 8);
76 55 : SEQ_PUT_DEC(" kB\nVmSwap:\t", swap);
77 55 : seq_puts(m, " kB\n");
78 55 : hugetlb_report_usage(m, mm);
79 55 : }
80 : #undef SEQ_PUT_DEC
81 :
82 96 : unsigned long task_vsize(struct mm_struct *mm)
83 : {
84 96 : return PAGE_SIZE * mm->total_vm;
85 : }
86 :
87 0 : unsigned long task_statm(struct mm_struct *mm,
88 : unsigned long *shared, unsigned long *text,
89 : unsigned long *data, unsigned long *resident)
90 : {
91 0 : *shared = get_mm_counter(mm, MM_FILEPAGES) +
92 0 : get_mm_counter(mm, MM_SHMEMPAGES);
93 0 : *text = (PAGE_ALIGN(mm->end_code) - (mm->start_code & PAGE_MASK))
94 0 : >> PAGE_SHIFT;
95 0 : *data = mm->data_vm + mm->stack_vm;
96 0 : *resident = *shared + get_mm_counter(mm, MM_ANONPAGES);
97 0 : return mm->total_vm;
98 : }
99 :
100 : #ifdef CONFIG_NUMA
101 : /*
102 : * Save get_task_policy() for show_numa_map().
103 : */
104 0 : static void hold_task_mempolicy(struct proc_maps_private *priv)
105 : {
106 0 : struct task_struct *task = priv->task;
107 :
108 0 : task_lock(task);
109 0 : priv->task_mempolicy = get_task_policy(task);
110 0 : mpol_get(priv->task_mempolicy);
111 0 : task_unlock(task);
112 0 : }
113 0 : static void release_task_mempolicy(struct proc_maps_private *priv)
114 : {
115 0 : mpol_put(priv->task_mempolicy);
116 : }
117 : #else
118 : static void hold_task_mempolicy(struct proc_maps_private *priv)
119 : {
120 : }
121 : static void release_task_mempolicy(struct proc_maps_private *priv)
122 : {
123 : }
124 : #endif
125 :
126 0 : static void *m_start(struct seq_file *m, loff_t *ppos)
127 : {
128 0 : struct proc_maps_private *priv = m->private;
129 0 : unsigned long last_addr = *ppos;
130 0 : struct mm_struct *mm;
131 0 : struct vm_area_struct *vma;
132 :
133 : /* See m_next(). Zero at the start or after lseek. */
134 0 : if (last_addr == -1UL)
135 : return NULL;
136 :
137 0 : priv->task = get_proc_task(priv->inode);
138 0 : if (!priv->task)
139 0 : return ERR_PTR(-ESRCH);
140 :
141 0 : mm = priv->mm;
142 0 : if (!mm || !mmget_not_zero(mm)) {
143 0 : put_task_struct(priv->task);
144 0 : priv->task = NULL;
145 0 : return NULL;
146 : }
147 :
148 0 : if (mmap_read_lock_killable(mm)) {
149 0 : mmput(mm);
150 0 : put_task_struct(priv->task);
151 0 : priv->task = NULL;
152 0 : return ERR_PTR(-EINTR);
153 : }
154 :
155 0 : hold_task_mempolicy(priv);
156 0 : priv->tail_vma = get_gate_vma(mm);
157 :
158 0 : vma = find_vma(mm, last_addr);
159 0 : if (vma)
160 : return vma;
161 :
162 0 : return priv->tail_vma;
163 : }
164 :
165 0 : static void *m_next(struct seq_file *m, void *v, loff_t *ppos)
166 : {
167 0 : struct proc_maps_private *priv = m->private;
168 0 : struct vm_area_struct *next, *vma = v;
169 :
170 0 : if (vma == priv->tail_vma)
171 : next = NULL;
172 0 : else if (vma->vm_next)
173 : next = vma->vm_next;
174 : else
175 0 : next = priv->tail_vma;
176 :
177 0 : *ppos = next ? next->vm_start : -1UL;
178 :
179 0 : return next;
180 : }
181 :
182 0 : static void m_stop(struct seq_file *m, void *v)
183 : {
184 0 : struct proc_maps_private *priv = m->private;
185 0 : struct mm_struct *mm = priv->mm;
186 :
187 0 : if (!priv->task)
188 : return;
189 :
190 0 : release_task_mempolicy(priv);
191 0 : mmap_read_unlock(mm);
192 0 : mmput(mm);
193 0 : put_task_struct(priv->task);
194 0 : priv->task = NULL;
195 : }
196 :
197 0 : static int proc_maps_open(struct inode *inode, struct file *file,
198 : const struct seq_operations *ops, int psize)
199 : {
200 0 : struct proc_maps_private *priv = __seq_open_private(file, ops, psize);
201 :
202 0 : if (!priv)
203 : return -ENOMEM;
204 :
205 0 : priv->inode = inode;
206 0 : priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
207 0 : if (IS_ERR(priv->mm)) {
208 0 : int err = PTR_ERR(priv->mm);
209 :
210 0 : seq_release_private(inode, file);
211 0 : return err;
212 : }
213 :
214 : return 0;
215 : }
216 :
217 0 : static int proc_map_release(struct inode *inode, struct file *file)
218 : {
219 0 : struct seq_file *seq = file->private_data;
220 0 : struct proc_maps_private *priv = seq->private;
221 :
222 0 : if (priv->mm)
223 0 : mmdrop(priv->mm);
224 :
225 0 : return seq_release_private(inode, file);
226 : }
227 :
228 0 : static int do_maps_open(struct inode *inode, struct file *file,
229 : const struct seq_operations *ops)
230 : {
231 0 : return proc_maps_open(inode, file, ops,
232 : sizeof(struct proc_maps_private));
233 : }
234 :
235 : /*
236 : * Indicate if the VMA is a stack for the given task; for
237 : * /proc/PID/maps that is the stack of the main task.
238 : */
239 0 : static int is_stack(struct vm_area_struct *vma)
240 : {
241 : /*
242 : * We make no effort to guess what a given thread considers to be
243 : * its "stack". It's not even well-defined for programs written
244 : * languages like Go.
245 : */
246 0 : return vma->vm_start <= vma->vm_mm->start_stack &&
247 0 : vma->vm_end >= vma->vm_mm->start_stack;
248 : }
249 :
250 0 : static void show_vma_header_prefix(struct seq_file *m,
251 : unsigned long start, unsigned long end,
252 : vm_flags_t flags, unsigned long long pgoff,
253 : dev_t dev, unsigned long ino)
254 : {
255 0 : seq_setwidth(m, 25 + sizeof(void *) * 6 - 1);
256 0 : seq_put_hex_ll(m, NULL, start, 8);
257 0 : seq_put_hex_ll(m, "-", end, 8);
258 0 : seq_putc(m, ' ');
259 0 : seq_putc(m, flags & VM_READ ? 'r' : '-');
260 0 : seq_putc(m, flags & VM_WRITE ? 'w' : '-');
261 0 : seq_putc(m, flags & VM_EXEC ? 'x' : '-');
262 0 : seq_putc(m, flags & VM_MAYSHARE ? 's' : 'p');
263 0 : seq_put_hex_ll(m, " ", pgoff, 8);
264 0 : seq_put_hex_ll(m, " ", MAJOR(dev), 2);
265 0 : seq_put_hex_ll(m, ":", MINOR(dev), 2);
266 0 : seq_put_decimal_ull(m, " ", ino);
267 0 : seq_putc(m, ' ');
268 0 : }
269 :
270 : static void
271 0 : show_map_vma(struct seq_file *m, struct vm_area_struct *vma)
272 : {
273 0 : struct mm_struct *mm = vma->vm_mm;
274 0 : struct file *file = vma->vm_file;
275 0 : vm_flags_t flags = vma->vm_flags;
276 0 : unsigned long ino = 0;
277 0 : unsigned long long pgoff = 0;
278 0 : unsigned long start, end;
279 0 : dev_t dev = 0;
280 0 : const char *name = NULL;
281 :
282 0 : if (file) {
283 0 : struct inode *inode = file_inode(vma->vm_file);
284 0 : dev = inode->i_sb->s_dev;
285 0 : ino = inode->i_ino;
286 0 : pgoff = ((loff_t)vma->vm_pgoff) << PAGE_SHIFT;
287 : }
288 :
289 0 : start = vma->vm_start;
290 0 : end = vma->vm_end;
291 0 : show_vma_header_prefix(m, start, end, flags, pgoff, dev, ino);
292 :
293 : /*
294 : * Print the dentry name for named mappings, and a
295 : * special [heap] marker for the heap:
296 : */
297 0 : if (file) {
298 0 : seq_pad(m, ' ');
299 0 : seq_file_path(m, file, "\n");
300 0 : goto done;
301 : }
302 :
303 0 : if (vma->vm_ops && vma->vm_ops->name) {
304 0 : name = vma->vm_ops->name(vma);
305 0 : if (name)
306 0 : goto done;
307 : }
308 :
309 0 : name = arch_vma_name(vma);
310 0 : if (!name) {
311 0 : if (!mm) {
312 0 : name = "[vdso]";
313 0 : goto done;
314 : }
315 :
316 0 : if (vma->vm_start <= mm->brk &&
317 0 : vma->vm_end >= mm->start_brk) {
318 0 : name = "[heap]";
319 0 : goto done;
320 : }
321 :
322 0 : if (is_stack(vma))
323 : name = "[stack]";
324 : }
325 :
326 0 : done:
327 0 : if (name) {
328 0 : seq_pad(m, ' ');
329 0 : seq_puts(m, name);
330 : }
331 0 : seq_putc(m, '\n');
332 0 : }
333 :
334 0 : static int show_map(struct seq_file *m, void *v)
335 : {
336 0 : show_map_vma(m, v);
337 0 : return 0;
338 : }
339 :
340 : static const struct seq_operations proc_pid_maps_op = {
341 : .start = m_start,
342 : .next = m_next,
343 : .stop = m_stop,
344 : .show = show_map
345 : };
346 :
347 0 : static int pid_maps_open(struct inode *inode, struct file *file)
348 : {
349 0 : return do_maps_open(inode, file, &proc_pid_maps_op);
350 : }
351 :
352 : const struct file_operations proc_pid_maps_operations = {
353 : .open = pid_maps_open,
354 : .read = seq_read,
355 : .llseek = seq_lseek,
356 : .release = proc_map_release,
357 : };
358 :
359 : /*
360 : * Proportional Set Size(PSS): my share of RSS.
361 : *
362 : * PSS of a process is the count of pages it has in memory, where each
363 : * page is divided by the number of processes sharing it. So if a
364 : * process has 1000 pages all to itself, and 1000 shared with one other
365 : * process, its PSS will be 1500.
366 : *
367 : * To keep (accumulated) division errors low, we adopt a 64bit
368 : * fixed-point pss counter to minimize division errors. So (pss >>
369 : * PSS_SHIFT) would be the real byte count.
370 : *
371 : * A shift of 12 before division means (assuming 4K page size):
372 : * - 1M 3-user-pages add up to 8KB errors;
373 : * - supports mapcount up to 2^24, or 16M;
374 : * - supports PSS up to 2^52 bytes, or 4PB.
375 : */
376 : #define PSS_SHIFT 12
377 :
378 : #ifdef CONFIG_PROC_PAGE_MONITOR
379 : struct mem_size_stats {
380 : unsigned long resident;
381 : unsigned long shared_clean;
382 : unsigned long shared_dirty;
383 : unsigned long private_clean;
384 : unsigned long private_dirty;
385 : unsigned long referenced;
386 : unsigned long anonymous;
387 : unsigned long lazyfree;
388 : unsigned long anonymous_thp;
389 : unsigned long shmem_thp;
390 : unsigned long file_thp;
391 : unsigned long swap;
392 : unsigned long shared_hugetlb;
393 : unsigned long private_hugetlb;
394 : u64 pss;
395 : u64 pss_anon;
396 : u64 pss_file;
397 : u64 pss_shmem;
398 : u64 pss_locked;
399 : u64 swap_pss;
400 : bool check_shmem_swap;
401 : };
402 :
403 : static void smaps_page_accumulate(struct mem_size_stats *mss,
404 : struct page *page, unsigned long size, unsigned long pss,
405 : bool dirty, bool locked, bool private)
406 : {
407 : mss->pss += pss;
408 :
409 : if (PageAnon(page))
410 : mss->pss_anon += pss;
411 : else if (PageSwapBacked(page))
412 : mss->pss_shmem += pss;
413 : else
414 : mss->pss_file += pss;
415 :
416 : if (locked)
417 : mss->pss_locked += pss;
418 :
419 : if (dirty || PageDirty(page)) {
420 : if (private)
421 : mss->private_dirty += size;
422 : else
423 : mss->shared_dirty += size;
424 : } else {
425 : if (private)
426 : mss->private_clean += size;
427 : else
428 : mss->shared_clean += size;
429 : }
430 : }
431 :
432 : static void smaps_account(struct mem_size_stats *mss, struct page *page,
433 : bool compound, bool young, bool dirty, bool locked)
434 : {
435 : int i, nr = compound ? compound_nr(page) : 1;
436 : unsigned long size = nr * PAGE_SIZE;
437 :
438 : /*
439 : * First accumulate quantities that depend only on |size| and the type
440 : * of the compound page.
441 : */
442 : if (PageAnon(page)) {
443 : mss->anonymous += size;
444 : if (!PageSwapBacked(page) && !dirty && !PageDirty(page))
445 : mss->lazyfree += size;
446 : }
447 :
448 : mss->resident += size;
449 : /* Accumulate the size in pages that have been accessed. */
450 : if (young || page_is_young(page) || PageReferenced(page))
451 : mss->referenced += size;
452 :
453 : /*
454 : * Then accumulate quantities that may depend on sharing, or that may
455 : * differ page-by-page.
456 : *
457 : * page_count(page) == 1 guarantees the page is mapped exactly once.
458 : * If any subpage of the compound page mapped with PTE it would elevate
459 : * page_count().
460 : */
461 : if (page_count(page) == 1) {
462 : smaps_page_accumulate(mss, page, size, size << PSS_SHIFT, dirty,
463 : locked, true);
464 : return;
465 : }
466 : for (i = 0; i < nr; i++, page++) {
467 : int mapcount = page_mapcount(page);
468 : unsigned long pss = PAGE_SIZE << PSS_SHIFT;
469 : if (mapcount >= 2)
470 : pss /= mapcount;
471 : smaps_page_accumulate(mss, page, PAGE_SIZE, pss, dirty, locked,
472 : mapcount < 2);
473 : }
474 : }
475 :
476 : #ifdef CONFIG_SHMEM
477 : static int smaps_pte_hole(unsigned long addr, unsigned long end,
478 : __always_unused int depth, struct mm_walk *walk)
479 : {
480 : struct mem_size_stats *mss = walk->private;
481 :
482 : mss->swap += shmem_partial_swap_usage(
483 : walk->vma->vm_file->f_mapping, addr, end);
484 :
485 : return 0;
486 : }
487 : #else
488 : #define smaps_pte_hole NULL
489 : #endif /* CONFIG_SHMEM */
490 :
491 : static void smaps_pte_entry(pte_t *pte, unsigned long addr,
492 : struct mm_walk *walk)
493 : {
494 : struct mem_size_stats *mss = walk->private;
495 : struct vm_area_struct *vma = walk->vma;
496 : bool locked = !!(vma->vm_flags & VM_LOCKED);
497 : struct page *page = NULL;
498 :
499 : if (pte_present(*pte)) {
500 : page = vm_normal_page(vma, addr, *pte);
501 : } else if (is_swap_pte(*pte)) {
502 : swp_entry_t swpent = pte_to_swp_entry(*pte);
503 :
504 : if (!non_swap_entry(swpent)) {
505 : int mapcount;
506 :
507 : mss->swap += PAGE_SIZE;
508 : mapcount = swp_swapcount(swpent);
509 : if (mapcount >= 2) {
510 : u64 pss_delta = (u64)PAGE_SIZE << PSS_SHIFT;
511 :
512 : do_div(pss_delta, mapcount);
513 : mss->swap_pss += pss_delta;
514 : } else {
515 : mss->swap_pss += (u64)PAGE_SIZE << PSS_SHIFT;
516 : }
517 : } else if (is_migration_entry(swpent))
518 : page = migration_entry_to_page(swpent);
519 : else if (is_device_private_entry(swpent))
520 : page = device_private_entry_to_page(swpent);
521 : } else if (unlikely(IS_ENABLED(CONFIG_SHMEM) && mss->check_shmem_swap
522 : && pte_none(*pte))) {
523 : page = xa_load(&vma->vm_file->f_mapping->i_pages,
524 : linear_page_index(vma, addr));
525 : if (xa_is_value(page))
526 : mss->swap += PAGE_SIZE;
527 : return;
528 : }
529 :
530 : if (!page)
531 : return;
532 :
533 : smaps_account(mss, page, false, pte_young(*pte), pte_dirty(*pte), locked);
534 : }
535 :
536 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
537 : static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
538 : struct mm_walk *walk)
539 : {
540 : struct mem_size_stats *mss = walk->private;
541 : struct vm_area_struct *vma = walk->vma;
542 : bool locked = !!(vma->vm_flags & VM_LOCKED);
543 : struct page *page = NULL;
544 :
545 : if (pmd_present(*pmd)) {
546 : /* FOLL_DUMP will return -EFAULT on huge zero page */
547 : page = follow_trans_huge_pmd(vma, addr, pmd, FOLL_DUMP);
548 : } else if (unlikely(thp_migration_supported() && is_swap_pmd(*pmd))) {
549 : swp_entry_t entry = pmd_to_swp_entry(*pmd);
550 :
551 : if (is_migration_entry(entry))
552 : page = migration_entry_to_page(entry);
553 : }
554 : if (IS_ERR_OR_NULL(page))
555 : return;
556 : if (PageAnon(page))
557 : mss->anonymous_thp += HPAGE_PMD_SIZE;
558 : else if (PageSwapBacked(page))
559 : mss->shmem_thp += HPAGE_PMD_SIZE;
560 : else if (is_zone_device_page(page))
561 : /* pass */;
562 : else
563 : mss->file_thp += HPAGE_PMD_SIZE;
564 : smaps_account(mss, page, true, pmd_young(*pmd), pmd_dirty(*pmd), locked);
565 : }
566 : #else
567 : static void smaps_pmd_entry(pmd_t *pmd, unsigned long addr,
568 : struct mm_walk *walk)
569 : {
570 : }
571 : #endif
572 :
573 : static int smaps_pte_range(pmd_t *pmd, unsigned long addr, unsigned long end,
574 : struct mm_walk *walk)
575 : {
576 : struct vm_area_struct *vma = walk->vma;
577 : pte_t *pte;
578 : spinlock_t *ptl;
579 :
580 : ptl = pmd_trans_huge_lock(pmd, vma);
581 : if (ptl) {
582 : smaps_pmd_entry(pmd, addr, walk);
583 : spin_unlock(ptl);
584 : goto out;
585 : }
586 :
587 : if (pmd_trans_unstable(pmd))
588 : goto out;
589 : /*
590 : * The mmap_lock held all the way back in m_start() is what
591 : * keeps khugepaged out of here and from collapsing things
592 : * in here.
593 : */
594 : pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
595 : for (; addr != end; pte++, addr += PAGE_SIZE)
596 : smaps_pte_entry(pte, addr, walk);
597 : pte_unmap_unlock(pte - 1, ptl);
598 : out:
599 : cond_resched();
600 : return 0;
601 : }
602 :
603 : static void show_smap_vma_flags(struct seq_file *m, struct vm_area_struct *vma)
604 : {
605 : /*
606 : * Don't forget to update Documentation/ on changes.
607 : */
608 : static const char mnemonics[BITS_PER_LONG][2] = {
609 : /*
610 : * In case if we meet a flag we don't know about.
611 : */
612 : [0 ... (BITS_PER_LONG-1)] = "??",
613 :
614 : [ilog2(VM_READ)] = "rd",
615 : [ilog2(VM_WRITE)] = "wr",
616 : [ilog2(VM_EXEC)] = "ex",
617 : [ilog2(VM_SHARED)] = "sh",
618 : [ilog2(VM_MAYREAD)] = "mr",
619 : [ilog2(VM_MAYWRITE)] = "mw",
620 : [ilog2(VM_MAYEXEC)] = "me",
621 : [ilog2(VM_MAYSHARE)] = "ms",
622 : [ilog2(VM_GROWSDOWN)] = "gd",
623 : [ilog2(VM_PFNMAP)] = "pf",
624 : [ilog2(VM_DENYWRITE)] = "dw",
625 : [ilog2(VM_LOCKED)] = "lo",
626 : [ilog2(VM_IO)] = "io",
627 : [ilog2(VM_SEQ_READ)] = "sr",
628 : [ilog2(VM_RAND_READ)] = "rr",
629 : [ilog2(VM_DONTCOPY)] = "dc",
630 : [ilog2(VM_DONTEXPAND)] = "de",
631 : [ilog2(VM_ACCOUNT)] = "ac",
632 : [ilog2(VM_NORESERVE)] = "nr",
633 : [ilog2(VM_HUGETLB)] = "ht",
634 : [ilog2(VM_SYNC)] = "sf",
635 : [ilog2(VM_ARCH_1)] = "ar",
636 : [ilog2(VM_WIPEONFORK)] = "wf",
637 : [ilog2(VM_DONTDUMP)] = "dd",
638 : #ifdef CONFIG_ARM64_BTI
639 : [ilog2(VM_ARM64_BTI)] = "bt",
640 : #endif
641 : #ifdef CONFIG_MEM_SOFT_DIRTY
642 : [ilog2(VM_SOFTDIRTY)] = "sd",
643 : #endif
644 : [ilog2(VM_MIXEDMAP)] = "mm",
645 : [ilog2(VM_HUGEPAGE)] = "hg",
646 : [ilog2(VM_NOHUGEPAGE)] = "nh",
647 : [ilog2(VM_MERGEABLE)] = "mg",
648 : [ilog2(VM_UFFD_MISSING)]= "um",
649 : [ilog2(VM_UFFD_WP)] = "uw",
650 : #ifdef CONFIG_ARM64_MTE
651 : [ilog2(VM_MTE)] = "mt",
652 : [ilog2(VM_MTE_ALLOWED)] = "",
653 : #endif
654 : #ifdef CONFIG_ARCH_HAS_PKEYS
655 : /* These come out via ProtectionKey: */
656 : [ilog2(VM_PKEY_BIT0)] = "",
657 : [ilog2(VM_PKEY_BIT1)] = "",
658 : [ilog2(VM_PKEY_BIT2)] = "",
659 : [ilog2(VM_PKEY_BIT3)] = "",
660 : #if VM_PKEY_BIT4
661 : [ilog2(VM_PKEY_BIT4)] = "",
662 : #endif
663 : #endif /* CONFIG_ARCH_HAS_PKEYS */
664 : };
665 : size_t i;
666 :
667 : seq_puts(m, "VmFlags: ");
668 : for (i = 0; i < BITS_PER_LONG; i++) {
669 : if (!mnemonics[i][0])
670 : continue;
671 : if (vma->vm_flags & (1UL << i)) {
672 : seq_putc(m, mnemonics[i][0]);
673 : seq_putc(m, mnemonics[i][1]);
674 : seq_putc(m, ' ');
675 : }
676 : }
677 : seq_putc(m, '\n');
678 : }
679 :
680 : #ifdef CONFIG_HUGETLB_PAGE
681 : static int smaps_hugetlb_range(pte_t *pte, unsigned long hmask,
682 : unsigned long addr, unsigned long end,
683 : struct mm_walk *walk)
684 : {
685 : struct mem_size_stats *mss = walk->private;
686 : struct vm_area_struct *vma = walk->vma;
687 : struct page *page = NULL;
688 :
689 : if (pte_present(*pte)) {
690 : page = vm_normal_page(vma, addr, *pte);
691 : } else if (is_swap_pte(*pte)) {
692 : swp_entry_t swpent = pte_to_swp_entry(*pte);
693 :
694 : if (is_migration_entry(swpent))
695 : page = migration_entry_to_page(swpent);
696 : else if (is_device_private_entry(swpent))
697 : page = device_private_entry_to_page(swpent);
698 : }
699 : if (page) {
700 : int mapcount = page_mapcount(page);
701 :
702 : if (mapcount >= 2)
703 : mss->shared_hugetlb += huge_page_size(hstate_vma(vma));
704 : else
705 : mss->private_hugetlb += huge_page_size(hstate_vma(vma));
706 : }
707 : return 0;
708 : }
709 : #else
710 : #define smaps_hugetlb_range NULL
711 : #endif /* HUGETLB_PAGE */
712 :
713 : static const struct mm_walk_ops smaps_walk_ops = {
714 : .pmd_entry = smaps_pte_range,
715 : .hugetlb_entry = smaps_hugetlb_range,
716 : };
717 :
718 : static const struct mm_walk_ops smaps_shmem_walk_ops = {
719 : .pmd_entry = smaps_pte_range,
720 : .hugetlb_entry = smaps_hugetlb_range,
721 : .pte_hole = smaps_pte_hole,
722 : };
723 :
724 : /*
725 : * Gather mem stats from @vma with the indicated beginning
726 : * address @start, and keep them in @mss.
727 : *
728 : * Use vm_start of @vma as the beginning address if @start is 0.
729 : */
730 : static void smap_gather_stats(struct vm_area_struct *vma,
731 : struct mem_size_stats *mss, unsigned long start)
732 : {
733 : const struct mm_walk_ops *ops = &smaps_walk_ops;
734 :
735 : /* Invalid start */
736 : if (start >= vma->vm_end)
737 : return;
738 :
739 : #ifdef CONFIG_SHMEM
740 : /* In case of smaps_rollup, reset the value from previous vma */
741 : mss->check_shmem_swap = false;
742 : if (vma->vm_file && shmem_mapping(vma->vm_file->f_mapping)) {
743 : /*
744 : * For shared or readonly shmem mappings we know that all
745 : * swapped out pages belong to the shmem object, and we can
746 : * obtain the swap value much more efficiently. For private
747 : * writable mappings, we might have COW pages that are
748 : * not affected by the parent swapped out pages of the shmem
749 : * object, so we have to distinguish them during the page walk.
750 : * Unless we know that the shmem object (or the part mapped by
751 : * our VMA) has no swapped out pages at all.
752 : */
753 : unsigned long shmem_swapped = shmem_swap_usage(vma);
754 :
755 : if (!start && (!shmem_swapped || (vma->vm_flags & VM_SHARED) ||
756 : !(vma->vm_flags & VM_WRITE))) {
757 : mss->swap += shmem_swapped;
758 : } else {
759 : mss->check_shmem_swap = true;
760 : ops = &smaps_shmem_walk_ops;
761 : }
762 : }
763 : #endif
764 : /* mmap_lock is held in m_start */
765 : if (!start)
766 : walk_page_vma(vma, ops, mss);
767 : else
768 : walk_page_range(vma->vm_mm, start, vma->vm_end, ops, mss);
769 : }
770 :
771 : #define SEQ_PUT_DEC(str, val) \
772 : seq_put_decimal_ull_width(m, str, (val) >> 10, 8)
773 :
774 : /* Show the contents common for smaps and smaps_rollup */
775 : static void __show_smap(struct seq_file *m, const struct mem_size_stats *mss,
776 : bool rollup_mode)
777 : {
778 : SEQ_PUT_DEC("Rss: ", mss->resident);
779 : SEQ_PUT_DEC(" kB\nPss: ", mss->pss >> PSS_SHIFT);
780 : if (rollup_mode) {
781 : /*
782 : * These are meaningful only for smaps_rollup, otherwise two of
783 : * them are zero, and the other one is the same as Pss.
784 : */
785 : SEQ_PUT_DEC(" kB\nPss_Anon: ",
786 : mss->pss_anon >> PSS_SHIFT);
787 : SEQ_PUT_DEC(" kB\nPss_File: ",
788 : mss->pss_file >> PSS_SHIFT);
789 : SEQ_PUT_DEC(" kB\nPss_Shmem: ",
790 : mss->pss_shmem >> PSS_SHIFT);
791 : }
792 : SEQ_PUT_DEC(" kB\nShared_Clean: ", mss->shared_clean);
793 : SEQ_PUT_DEC(" kB\nShared_Dirty: ", mss->shared_dirty);
794 : SEQ_PUT_DEC(" kB\nPrivate_Clean: ", mss->private_clean);
795 : SEQ_PUT_DEC(" kB\nPrivate_Dirty: ", mss->private_dirty);
796 : SEQ_PUT_DEC(" kB\nReferenced: ", mss->referenced);
797 : SEQ_PUT_DEC(" kB\nAnonymous: ", mss->anonymous);
798 : SEQ_PUT_DEC(" kB\nLazyFree: ", mss->lazyfree);
799 : SEQ_PUT_DEC(" kB\nAnonHugePages: ", mss->anonymous_thp);
800 : SEQ_PUT_DEC(" kB\nShmemPmdMapped: ", mss->shmem_thp);
801 : SEQ_PUT_DEC(" kB\nFilePmdMapped: ", mss->file_thp);
802 : SEQ_PUT_DEC(" kB\nShared_Hugetlb: ", mss->shared_hugetlb);
803 : seq_put_decimal_ull_width(m, " kB\nPrivate_Hugetlb: ",
804 : mss->private_hugetlb >> 10, 7);
805 : SEQ_PUT_DEC(" kB\nSwap: ", mss->swap);
806 : SEQ_PUT_DEC(" kB\nSwapPss: ",
807 : mss->swap_pss >> PSS_SHIFT);
808 : SEQ_PUT_DEC(" kB\nLocked: ",
809 : mss->pss_locked >> PSS_SHIFT);
810 : seq_puts(m, " kB\n");
811 : }
812 :
813 : static int show_smap(struct seq_file *m, void *v)
814 : {
815 : struct vm_area_struct *vma = v;
816 : struct mem_size_stats mss;
817 :
818 : memset(&mss, 0, sizeof(mss));
819 :
820 : smap_gather_stats(vma, &mss, 0);
821 :
822 : show_map_vma(m, vma);
823 :
824 : SEQ_PUT_DEC("Size: ", vma->vm_end - vma->vm_start);
825 : SEQ_PUT_DEC(" kB\nKernelPageSize: ", vma_kernel_pagesize(vma));
826 : SEQ_PUT_DEC(" kB\nMMUPageSize: ", vma_mmu_pagesize(vma));
827 : seq_puts(m, " kB\n");
828 :
829 : __show_smap(m, &mss, false);
830 :
831 : seq_printf(m, "THPeligible: %d\n",
832 : transparent_hugepage_enabled(vma));
833 :
834 : if (arch_pkeys_enabled())
835 : seq_printf(m, "ProtectionKey: %8u\n", vma_pkey(vma));
836 : show_smap_vma_flags(m, vma);
837 :
838 : return 0;
839 : }
840 :
841 : static int show_smaps_rollup(struct seq_file *m, void *v)
842 : {
843 : struct proc_maps_private *priv = m->private;
844 : struct mem_size_stats mss;
845 : struct mm_struct *mm;
846 : struct vm_area_struct *vma;
847 : unsigned long last_vma_end = 0;
848 : int ret = 0;
849 :
850 : priv->task = get_proc_task(priv->inode);
851 : if (!priv->task)
852 : return -ESRCH;
853 :
854 : mm = priv->mm;
855 : if (!mm || !mmget_not_zero(mm)) {
856 : ret = -ESRCH;
857 : goto out_put_task;
858 : }
859 :
860 : memset(&mss, 0, sizeof(mss));
861 :
862 : ret = mmap_read_lock_killable(mm);
863 : if (ret)
864 : goto out_put_mm;
865 :
866 : hold_task_mempolicy(priv);
867 :
868 : for (vma = priv->mm->mmap; vma;) {
869 : smap_gather_stats(vma, &mss, 0);
870 : last_vma_end = vma->vm_end;
871 :
872 : /*
873 : * Release mmap_lock temporarily if someone wants to
874 : * access it for write request.
875 : */
876 : if (mmap_lock_is_contended(mm)) {
877 : mmap_read_unlock(mm);
878 : ret = mmap_read_lock_killable(mm);
879 : if (ret) {
880 : release_task_mempolicy(priv);
881 : goto out_put_mm;
882 : }
883 :
884 : /*
885 : * After dropping the lock, there are four cases to
886 : * consider. See the following example for explanation.
887 : *
888 : * +------+------+-----------+
889 : * | VMA1 | VMA2 | VMA3 |
890 : * +------+------+-----------+
891 : * | | | |
892 : * 4k 8k 16k 400k
893 : *
894 : * Suppose we drop the lock after reading VMA2 due to
895 : * contention, then we get:
896 : *
897 : * last_vma_end = 16k
898 : *
899 : * 1) VMA2 is freed, but VMA3 exists:
900 : *
901 : * find_vma(mm, 16k - 1) will return VMA3.
902 : * In this case, just continue from VMA3.
903 : *
904 : * 2) VMA2 still exists:
905 : *
906 : * find_vma(mm, 16k - 1) will return VMA2.
907 : * Iterate the loop like the original one.
908 : *
909 : * 3) No more VMAs can be found:
910 : *
911 : * find_vma(mm, 16k - 1) will return NULL.
912 : * No more things to do, just break.
913 : *
914 : * 4) (last_vma_end - 1) is the middle of a vma (VMA'):
915 : *
916 : * find_vma(mm, 16k - 1) will return VMA' whose range
917 : * contains last_vma_end.
918 : * Iterate VMA' from last_vma_end.
919 : */
920 : vma = find_vma(mm, last_vma_end - 1);
921 : /* Case 3 above */
922 : if (!vma)
923 : break;
924 :
925 : /* Case 1 above */
926 : if (vma->vm_start >= last_vma_end)
927 : continue;
928 :
929 : /* Case 4 above */
930 : if (vma->vm_end > last_vma_end)
931 : smap_gather_stats(vma, &mss, last_vma_end);
932 : }
933 : /* Case 2 above */
934 : vma = vma->vm_next;
935 : }
936 :
937 : show_vma_header_prefix(m, priv->mm->mmap->vm_start,
938 : last_vma_end, 0, 0, 0, 0);
939 : seq_pad(m, ' ');
940 : seq_puts(m, "[rollup]\n");
941 :
942 : __show_smap(m, &mss, true);
943 :
944 : release_task_mempolicy(priv);
945 : mmap_read_unlock(mm);
946 :
947 : out_put_mm:
948 : mmput(mm);
949 : out_put_task:
950 : put_task_struct(priv->task);
951 : priv->task = NULL;
952 :
953 : return ret;
954 : }
955 : #undef SEQ_PUT_DEC
956 :
957 : static const struct seq_operations proc_pid_smaps_op = {
958 : .start = m_start,
959 : .next = m_next,
960 : .stop = m_stop,
961 : .show = show_smap
962 : };
963 :
964 : static int pid_smaps_open(struct inode *inode, struct file *file)
965 : {
966 : return do_maps_open(inode, file, &proc_pid_smaps_op);
967 : }
968 :
969 : static int smaps_rollup_open(struct inode *inode, struct file *file)
970 : {
971 : int ret;
972 : struct proc_maps_private *priv;
973 :
974 : priv = kzalloc(sizeof(*priv), GFP_KERNEL_ACCOUNT);
975 : if (!priv)
976 : return -ENOMEM;
977 :
978 : ret = single_open(file, show_smaps_rollup, priv);
979 : if (ret)
980 : goto out_free;
981 :
982 : priv->inode = inode;
983 : priv->mm = proc_mem_open(inode, PTRACE_MODE_READ);
984 : if (IS_ERR(priv->mm)) {
985 : ret = PTR_ERR(priv->mm);
986 :
987 : single_release(inode, file);
988 : goto out_free;
989 : }
990 :
991 : return 0;
992 :
993 : out_free:
994 : kfree(priv);
995 : return ret;
996 : }
997 :
998 : static int smaps_rollup_release(struct inode *inode, struct file *file)
999 : {
1000 : struct seq_file *seq = file->private_data;
1001 : struct proc_maps_private *priv = seq->private;
1002 :
1003 : if (priv->mm)
1004 : mmdrop(priv->mm);
1005 :
1006 : kfree(priv);
1007 : return single_release(inode, file);
1008 : }
1009 :
1010 : const struct file_operations proc_pid_smaps_operations = {
1011 : .open = pid_smaps_open,
1012 : .read = seq_read,
1013 : .llseek = seq_lseek,
1014 : .release = proc_map_release,
1015 : };
1016 :
1017 : const struct file_operations proc_pid_smaps_rollup_operations = {
1018 : .open = smaps_rollup_open,
1019 : .read = seq_read,
1020 : .llseek = seq_lseek,
1021 : .release = smaps_rollup_release,
1022 : };
1023 :
1024 : enum clear_refs_types {
1025 : CLEAR_REFS_ALL = 1,
1026 : CLEAR_REFS_ANON,
1027 : CLEAR_REFS_MAPPED,
1028 : CLEAR_REFS_SOFT_DIRTY,
1029 : CLEAR_REFS_MM_HIWATER_RSS,
1030 : CLEAR_REFS_LAST,
1031 : };
1032 :
1033 : struct clear_refs_private {
1034 : enum clear_refs_types type;
1035 : };
1036 :
1037 : #ifdef CONFIG_MEM_SOFT_DIRTY
1038 :
1039 : static inline bool pte_is_pinned(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1040 : {
1041 : struct page *page;
1042 :
1043 : if (!pte_write(pte))
1044 : return false;
1045 : if (!is_cow_mapping(vma->vm_flags))
1046 : return false;
1047 : if (likely(!atomic_read(&vma->vm_mm->has_pinned)))
1048 : return false;
1049 : page = vm_normal_page(vma, addr, pte);
1050 : if (!page)
1051 : return false;
1052 : return page_maybe_dma_pinned(page);
1053 : }
1054 :
1055 : static inline void clear_soft_dirty(struct vm_area_struct *vma,
1056 : unsigned long addr, pte_t *pte)
1057 : {
1058 : /*
1059 : * The soft-dirty tracker uses #PF-s to catch writes
1060 : * to pages, so write-protect the pte as well. See the
1061 : * Documentation/admin-guide/mm/soft-dirty.rst for full description
1062 : * of how soft-dirty works.
1063 : */
1064 : pte_t ptent = *pte;
1065 :
1066 : if (pte_present(ptent)) {
1067 : pte_t old_pte;
1068 :
1069 : if (pte_is_pinned(vma, addr, ptent))
1070 : return;
1071 : old_pte = ptep_modify_prot_start(vma, addr, pte);
1072 : ptent = pte_wrprotect(old_pte);
1073 : ptent = pte_clear_soft_dirty(ptent);
1074 : ptep_modify_prot_commit(vma, addr, pte, old_pte, ptent);
1075 : } else if (is_swap_pte(ptent)) {
1076 : ptent = pte_swp_clear_soft_dirty(ptent);
1077 : set_pte_at(vma->vm_mm, addr, pte, ptent);
1078 : }
1079 : }
1080 : #else
1081 : static inline void clear_soft_dirty(struct vm_area_struct *vma,
1082 : unsigned long addr, pte_t *pte)
1083 : {
1084 : }
1085 : #endif
1086 :
1087 : #if defined(CONFIG_MEM_SOFT_DIRTY) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
1088 : static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1089 : unsigned long addr, pmd_t *pmdp)
1090 : {
1091 : pmd_t old, pmd = *pmdp;
1092 :
1093 : if (pmd_present(pmd)) {
1094 : /* See comment in change_huge_pmd() */
1095 : old = pmdp_invalidate(vma, addr, pmdp);
1096 : if (pmd_dirty(old))
1097 : pmd = pmd_mkdirty(pmd);
1098 : if (pmd_young(old))
1099 : pmd = pmd_mkyoung(pmd);
1100 :
1101 : pmd = pmd_wrprotect(pmd);
1102 : pmd = pmd_clear_soft_dirty(pmd);
1103 :
1104 : set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1105 : } else if (is_migration_entry(pmd_to_swp_entry(pmd))) {
1106 : pmd = pmd_swp_clear_soft_dirty(pmd);
1107 : set_pmd_at(vma->vm_mm, addr, pmdp, pmd);
1108 : }
1109 : }
1110 : #else
1111 : static inline void clear_soft_dirty_pmd(struct vm_area_struct *vma,
1112 : unsigned long addr, pmd_t *pmdp)
1113 : {
1114 : }
1115 : #endif
1116 :
1117 : static int clear_refs_pte_range(pmd_t *pmd, unsigned long addr,
1118 : unsigned long end, struct mm_walk *walk)
1119 : {
1120 : struct clear_refs_private *cp = walk->private;
1121 : struct vm_area_struct *vma = walk->vma;
1122 : pte_t *pte, ptent;
1123 : spinlock_t *ptl;
1124 : struct page *page;
1125 :
1126 : ptl = pmd_trans_huge_lock(pmd, vma);
1127 : if (ptl) {
1128 : if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1129 : clear_soft_dirty_pmd(vma, addr, pmd);
1130 : goto out;
1131 : }
1132 :
1133 : if (!pmd_present(*pmd))
1134 : goto out;
1135 :
1136 : page = pmd_page(*pmd);
1137 :
1138 : /* Clear accessed and referenced bits. */
1139 : pmdp_test_and_clear_young(vma, addr, pmd);
1140 : test_and_clear_page_young(page);
1141 : ClearPageReferenced(page);
1142 : out:
1143 : spin_unlock(ptl);
1144 : return 0;
1145 : }
1146 :
1147 : if (pmd_trans_unstable(pmd))
1148 : return 0;
1149 :
1150 : pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
1151 : for (; addr != end; pte++, addr += PAGE_SIZE) {
1152 : ptent = *pte;
1153 :
1154 : if (cp->type == CLEAR_REFS_SOFT_DIRTY) {
1155 : clear_soft_dirty(vma, addr, pte);
1156 : continue;
1157 : }
1158 :
1159 : if (!pte_present(ptent))
1160 : continue;
1161 :
1162 : page = vm_normal_page(vma, addr, ptent);
1163 : if (!page)
1164 : continue;
1165 :
1166 : /* Clear accessed and referenced bits. */
1167 : ptep_test_and_clear_young(vma, addr, pte);
1168 : test_and_clear_page_young(page);
1169 : ClearPageReferenced(page);
1170 : }
1171 : pte_unmap_unlock(pte - 1, ptl);
1172 : cond_resched();
1173 : return 0;
1174 : }
1175 :
1176 : static int clear_refs_test_walk(unsigned long start, unsigned long end,
1177 : struct mm_walk *walk)
1178 : {
1179 : struct clear_refs_private *cp = walk->private;
1180 : struct vm_area_struct *vma = walk->vma;
1181 :
1182 : if (vma->vm_flags & VM_PFNMAP)
1183 : return 1;
1184 :
1185 : /*
1186 : * Writing 1 to /proc/pid/clear_refs affects all pages.
1187 : * Writing 2 to /proc/pid/clear_refs only affects anonymous pages.
1188 : * Writing 3 to /proc/pid/clear_refs only affects file mapped pages.
1189 : * Writing 4 to /proc/pid/clear_refs affects all pages.
1190 : */
1191 : if (cp->type == CLEAR_REFS_ANON && vma->vm_file)
1192 : return 1;
1193 : if (cp->type == CLEAR_REFS_MAPPED && !vma->vm_file)
1194 : return 1;
1195 : return 0;
1196 : }
1197 :
1198 : static const struct mm_walk_ops clear_refs_walk_ops = {
1199 : .pmd_entry = clear_refs_pte_range,
1200 : .test_walk = clear_refs_test_walk,
1201 : };
1202 :
1203 : static ssize_t clear_refs_write(struct file *file, const char __user *buf,
1204 : size_t count, loff_t *ppos)
1205 : {
1206 : struct task_struct *task;
1207 : char buffer[PROC_NUMBUF];
1208 : struct mm_struct *mm;
1209 : struct vm_area_struct *vma;
1210 : enum clear_refs_types type;
1211 : int itype;
1212 : int rv;
1213 :
1214 : memset(buffer, 0, sizeof(buffer));
1215 : if (count > sizeof(buffer) - 1)
1216 : count = sizeof(buffer) - 1;
1217 : if (copy_from_user(buffer, buf, count))
1218 : return -EFAULT;
1219 : rv = kstrtoint(strstrip(buffer), 10, &itype);
1220 : if (rv < 0)
1221 : return rv;
1222 : type = (enum clear_refs_types)itype;
1223 : if (type < CLEAR_REFS_ALL || type >= CLEAR_REFS_LAST)
1224 : return -EINVAL;
1225 :
1226 : task = get_proc_task(file_inode(file));
1227 : if (!task)
1228 : return -ESRCH;
1229 : mm = get_task_mm(task);
1230 : if (mm) {
1231 : struct mmu_notifier_range range;
1232 : struct clear_refs_private cp = {
1233 : .type = type,
1234 : };
1235 :
1236 : if (mmap_write_lock_killable(mm)) {
1237 : count = -EINTR;
1238 : goto out_mm;
1239 : }
1240 : if (type == CLEAR_REFS_MM_HIWATER_RSS) {
1241 : /*
1242 : * Writing 5 to /proc/pid/clear_refs resets the peak
1243 : * resident set size to this mm's current rss value.
1244 : */
1245 : reset_mm_hiwater_rss(mm);
1246 : goto out_unlock;
1247 : }
1248 :
1249 : if (type == CLEAR_REFS_SOFT_DIRTY) {
1250 : for (vma = mm->mmap; vma; vma = vma->vm_next) {
1251 : if (!(vma->vm_flags & VM_SOFTDIRTY))
1252 : continue;
1253 : vma->vm_flags &= ~VM_SOFTDIRTY;
1254 : vma_set_page_prot(vma);
1255 : }
1256 :
1257 : inc_tlb_flush_pending(mm);
1258 : mmu_notifier_range_init(&range, MMU_NOTIFY_SOFT_DIRTY,
1259 : 0, NULL, mm, 0, -1UL);
1260 : mmu_notifier_invalidate_range_start(&range);
1261 : }
1262 : walk_page_range(mm, 0, mm->highest_vm_end, &clear_refs_walk_ops,
1263 : &cp);
1264 : if (type == CLEAR_REFS_SOFT_DIRTY) {
1265 : mmu_notifier_invalidate_range_end(&range);
1266 : flush_tlb_mm(mm);
1267 : dec_tlb_flush_pending(mm);
1268 : }
1269 : out_unlock:
1270 : mmap_write_unlock(mm);
1271 : out_mm:
1272 : mmput(mm);
1273 : }
1274 : put_task_struct(task);
1275 :
1276 : return count;
1277 : }
1278 :
1279 : const struct file_operations proc_clear_refs_operations = {
1280 : .write = clear_refs_write,
1281 : .llseek = noop_llseek,
1282 : };
1283 :
1284 : typedef struct {
1285 : u64 pme;
1286 : } pagemap_entry_t;
1287 :
1288 : struct pagemapread {
1289 : int pos, len; /* units: PM_ENTRY_BYTES, not bytes */
1290 : pagemap_entry_t *buffer;
1291 : bool show_pfn;
1292 : };
1293 :
1294 : #define PAGEMAP_WALK_SIZE (PMD_SIZE)
1295 : #define PAGEMAP_WALK_MASK (PMD_MASK)
1296 :
1297 : #define PM_ENTRY_BYTES sizeof(pagemap_entry_t)
1298 : #define PM_PFRAME_BITS 55
1299 : #define PM_PFRAME_MASK GENMASK_ULL(PM_PFRAME_BITS - 1, 0)
1300 : #define PM_SOFT_DIRTY BIT_ULL(55)
1301 : #define PM_MMAP_EXCLUSIVE BIT_ULL(56)
1302 : #define PM_FILE BIT_ULL(61)
1303 : #define PM_SWAP BIT_ULL(62)
1304 : #define PM_PRESENT BIT_ULL(63)
1305 :
1306 : #define PM_END_OF_BUFFER 1
1307 :
1308 : static inline pagemap_entry_t make_pme(u64 frame, u64 flags)
1309 : {
1310 : return (pagemap_entry_t) { .pme = (frame & PM_PFRAME_MASK) | flags };
1311 : }
1312 :
1313 : static int add_to_pagemap(unsigned long addr, pagemap_entry_t *pme,
1314 : struct pagemapread *pm)
1315 : {
1316 : pm->buffer[pm->pos++] = *pme;
1317 : if (pm->pos >= pm->len)
1318 : return PM_END_OF_BUFFER;
1319 : return 0;
1320 : }
1321 :
1322 : static int pagemap_pte_hole(unsigned long start, unsigned long end,
1323 : __always_unused int depth, struct mm_walk *walk)
1324 : {
1325 : struct pagemapread *pm = walk->private;
1326 : unsigned long addr = start;
1327 : int err = 0;
1328 :
1329 : while (addr < end) {
1330 : struct vm_area_struct *vma = find_vma(walk->mm, addr);
1331 : pagemap_entry_t pme = make_pme(0, 0);
1332 : /* End of address space hole, which we mark as non-present. */
1333 : unsigned long hole_end;
1334 :
1335 : if (vma)
1336 : hole_end = min(end, vma->vm_start);
1337 : else
1338 : hole_end = end;
1339 :
1340 : for (; addr < hole_end; addr += PAGE_SIZE) {
1341 : err = add_to_pagemap(addr, &pme, pm);
1342 : if (err)
1343 : goto out;
1344 : }
1345 :
1346 : if (!vma)
1347 : break;
1348 :
1349 : /* Addresses in the VMA. */
1350 : if (vma->vm_flags & VM_SOFTDIRTY)
1351 : pme = make_pme(0, PM_SOFT_DIRTY);
1352 : for (; addr < min(end, vma->vm_end); addr += PAGE_SIZE) {
1353 : err = add_to_pagemap(addr, &pme, pm);
1354 : if (err)
1355 : goto out;
1356 : }
1357 : }
1358 : out:
1359 : return err;
1360 : }
1361 :
1362 : static pagemap_entry_t pte_to_pagemap_entry(struct pagemapread *pm,
1363 : struct vm_area_struct *vma, unsigned long addr, pte_t pte)
1364 : {
1365 : u64 frame = 0, flags = 0;
1366 : struct page *page = NULL;
1367 :
1368 : if (pte_present(pte)) {
1369 : if (pm->show_pfn)
1370 : frame = pte_pfn(pte);
1371 : flags |= PM_PRESENT;
1372 : page = vm_normal_page(vma, addr, pte);
1373 : if (pte_soft_dirty(pte))
1374 : flags |= PM_SOFT_DIRTY;
1375 : } else if (is_swap_pte(pte)) {
1376 : swp_entry_t entry;
1377 : if (pte_swp_soft_dirty(pte))
1378 : flags |= PM_SOFT_DIRTY;
1379 : entry = pte_to_swp_entry(pte);
1380 : if (pm->show_pfn)
1381 : frame = swp_type(entry) |
1382 : (swp_offset(entry) << MAX_SWAPFILES_SHIFT);
1383 : flags |= PM_SWAP;
1384 : if (is_migration_entry(entry))
1385 : page = migration_entry_to_page(entry);
1386 :
1387 : if (is_device_private_entry(entry))
1388 : page = device_private_entry_to_page(entry);
1389 : }
1390 :
1391 : if (page && !PageAnon(page))
1392 : flags |= PM_FILE;
1393 : if (page && page_mapcount(page) == 1)
1394 : flags |= PM_MMAP_EXCLUSIVE;
1395 : if (vma->vm_flags & VM_SOFTDIRTY)
1396 : flags |= PM_SOFT_DIRTY;
1397 :
1398 : return make_pme(frame, flags);
1399 : }
1400 :
1401 : static int pagemap_pmd_range(pmd_t *pmdp, unsigned long addr, unsigned long end,
1402 : struct mm_walk *walk)
1403 : {
1404 : struct vm_area_struct *vma = walk->vma;
1405 : struct pagemapread *pm = walk->private;
1406 : spinlock_t *ptl;
1407 : pte_t *pte, *orig_pte;
1408 : int err = 0;
1409 :
1410 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1411 : ptl = pmd_trans_huge_lock(pmdp, vma);
1412 : if (ptl) {
1413 : u64 flags = 0, frame = 0;
1414 : pmd_t pmd = *pmdp;
1415 : struct page *page = NULL;
1416 :
1417 : if (vma->vm_flags & VM_SOFTDIRTY)
1418 : flags |= PM_SOFT_DIRTY;
1419 :
1420 : if (pmd_present(pmd)) {
1421 : page = pmd_page(pmd);
1422 :
1423 : flags |= PM_PRESENT;
1424 : if (pmd_soft_dirty(pmd))
1425 : flags |= PM_SOFT_DIRTY;
1426 : if (pm->show_pfn)
1427 : frame = pmd_pfn(pmd) +
1428 : ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1429 : }
1430 : #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
1431 : else if (is_swap_pmd(pmd)) {
1432 : swp_entry_t entry = pmd_to_swp_entry(pmd);
1433 : unsigned long offset;
1434 :
1435 : if (pm->show_pfn) {
1436 : offset = swp_offset(entry) +
1437 : ((addr & ~PMD_MASK) >> PAGE_SHIFT);
1438 : frame = swp_type(entry) |
1439 : (offset << MAX_SWAPFILES_SHIFT);
1440 : }
1441 : flags |= PM_SWAP;
1442 : if (pmd_swp_soft_dirty(pmd))
1443 : flags |= PM_SOFT_DIRTY;
1444 : VM_BUG_ON(!is_pmd_migration_entry(pmd));
1445 : page = migration_entry_to_page(entry);
1446 : }
1447 : #endif
1448 :
1449 : if (page && page_mapcount(page) == 1)
1450 : flags |= PM_MMAP_EXCLUSIVE;
1451 :
1452 : for (; addr != end; addr += PAGE_SIZE) {
1453 : pagemap_entry_t pme = make_pme(frame, flags);
1454 :
1455 : err = add_to_pagemap(addr, &pme, pm);
1456 : if (err)
1457 : break;
1458 : if (pm->show_pfn) {
1459 : if (flags & PM_PRESENT)
1460 : frame++;
1461 : else if (flags & PM_SWAP)
1462 : frame += (1 << MAX_SWAPFILES_SHIFT);
1463 : }
1464 : }
1465 : spin_unlock(ptl);
1466 : return err;
1467 : }
1468 :
1469 : if (pmd_trans_unstable(pmdp))
1470 : return 0;
1471 : #endif /* CONFIG_TRANSPARENT_HUGEPAGE */
1472 :
1473 : /*
1474 : * We can assume that @vma always points to a valid one and @end never
1475 : * goes beyond vma->vm_end.
1476 : */
1477 : orig_pte = pte = pte_offset_map_lock(walk->mm, pmdp, addr, &ptl);
1478 : for (; addr < end; pte++, addr += PAGE_SIZE) {
1479 : pagemap_entry_t pme;
1480 :
1481 : pme = pte_to_pagemap_entry(pm, vma, addr, *pte);
1482 : err = add_to_pagemap(addr, &pme, pm);
1483 : if (err)
1484 : break;
1485 : }
1486 : pte_unmap_unlock(orig_pte, ptl);
1487 :
1488 : cond_resched();
1489 :
1490 : return err;
1491 : }
1492 :
1493 : #ifdef CONFIG_HUGETLB_PAGE
1494 : /* This function walks within one hugetlb entry in the single call */
1495 : static int pagemap_hugetlb_range(pte_t *ptep, unsigned long hmask,
1496 : unsigned long addr, unsigned long end,
1497 : struct mm_walk *walk)
1498 : {
1499 : struct pagemapread *pm = walk->private;
1500 : struct vm_area_struct *vma = walk->vma;
1501 : u64 flags = 0, frame = 0;
1502 : int err = 0;
1503 : pte_t pte;
1504 :
1505 : if (vma->vm_flags & VM_SOFTDIRTY)
1506 : flags |= PM_SOFT_DIRTY;
1507 :
1508 : pte = huge_ptep_get(ptep);
1509 : if (pte_present(pte)) {
1510 : struct page *page = pte_page(pte);
1511 :
1512 : if (!PageAnon(page))
1513 : flags |= PM_FILE;
1514 :
1515 : if (page_mapcount(page) == 1)
1516 : flags |= PM_MMAP_EXCLUSIVE;
1517 :
1518 : flags |= PM_PRESENT;
1519 : if (pm->show_pfn)
1520 : frame = pte_pfn(pte) +
1521 : ((addr & ~hmask) >> PAGE_SHIFT);
1522 : }
1523 :
1524 : for (; addr != end; addr += PAGE_SIZE) {
1525 : pagemap_entry_t pme = make_pme(frame, flags);
1526 :
1527 : err = add_to_pagemap(addr, &pme, pm);
1528 : if (err)
1529 : return err;
1530 : if (pm->show_pfn && (flags & PM_PRESENT))
1531 : frame++;
1532 : }
1533 :
1534 : cond_resched();
1535 :
1536 : return err;
1537 : }
1538 : #else
1539 : #define pagemap_hugetlb_range NULL
1540 : #endif /* HUGETLB_PAGE */
1541 :
1542 : static const struct mm_walk_ops pagemap_ops = {
1543 : .pmd_entry = pagemap_pmd_range,
1544 : .pte_hole = pagemap_pte_hole,
1545 : .hugetlb_entry = pagemap_hugetlb_range,
1546 : };
1547 :
1548 : /*
1549 : * /proc/pid/pagemap - an array mapping virtual pages to pfns
1550 : *
1551 : * For each page in the address space, this file contains one 64-bit entry
1552 : * consisting of the following:
1553 : *
1554 : * Bits 0-54 page frame number (PFN) if present
1555 : * Bits 0-4 swap type if swapped
1556 : * Bits 5-54 swap offset if swapped
1557 : * Bit 55 pte is soft-dirty (see Documentation/admin-guide/mm/soft-dirty.rst)
1558 : * Bit 56 page exclusively mapped
1559 : * Bits 57-60 zero
1560 : * Bit 61 page is file-page or shared-anon
1561 : * Bit 62 page swapped
1562 : * Bit 63 page present
1563 : *
1564 : * If the page is not present but in swap, then the PFN contains an
1565 : * encoding of the swap file number and the page's offset into the
1566 : * swap. Unmapped pages return a null PFN. This allows determining
1567 : * precisely which pages are mapped (or in swap) and comparing mapped
1568 : * pages between processes.
1569 : *
1570 : * Efficient users of this interface will use /proc/pid/maps to
1571 : * determine which areas of memory are actually mapped and llseek to
1572 : * skip over unmapped regions.
1573 : */
1574 : static ssize_t pagemap_read(struct file *file, char __user *buf,
1575 : size_t count, loff_t *ppos)
1576 : {
1577 : struct mm_struct *mm = file->private_data;
1578 : struct pagemapread pm;
1579 : unsigned long src;
1580 : unsigned long svpfn;
1581 : unsigned long start_vaddr;
1582 : unsigned long end_vaddr;
1583 : int ret = 0, copied = 0;
1584 :
1585 : if (!mm || !mmget_not_zero(mm))
1586 : goto out;
1587 :
1588 : ret = -EINVAL;
1589 : /* file position must be aligned */
1590 : if ((*ppos % PM_ENTRY_BYTES) || (count % PM_ENTRY_BYTES))
1591 : goto out_mm;
1592 :
1593 : ret = 0;
1594 : if (!count)
1595 : goto out_mm;
1596 :
1597 : /* do not disclose physical addresses: attack vector */
1598 : pm.show_pfn = file_ns_capable(file, &init_user_ns, CAP_SYS_ADMIN);
1599 :
1600 : pm.len = (PAGEMAP_WALK_SIZE >> PAGE_SHIFT);
1601 : pm.buffer = kmalloc_array(pm.len, PM_ENTRY_BYTES, GFP_KERNEL);
1602 : ret = -ENOMEM;
1603 : if (!pm.buffer)
1604 : goto out_mm;
1605 :
1606 : src = *ppos;
1607 : svpfn = src / PM_ENTRY_BYTES;
1608 : end_vaddr = mm->task_size;
1609 :
1610 : /* watch out for wraparound */
1611 : start_vaddr = end_vaddr;
1612 : if (svpfn <= (ULONG_MAX >> PAGE_SHIFT))
1613 : start_vaddr = untagged_addr(svpfn << PAGE_SHIFT);
1614 :
1615 : /* Ensure the address is inside the task */
1616 : if (start_vaddr > mm->task_size)
1617 : start_vaddr = end_vaddr;
1618 :
1619 : /*
1620 : * The odds are that this will stop walking way
1621 : * before end_vaddr, because the length of the
1622 : * user buffer is tracked in "pm", and the walk
1623 : * will stop when we hit the end of the buffer.
1624 : */
1625 : ret = 0;
1626 : while (count && (start_vaddr < end_vaddr)) {
1627 : int len;
1628 : unsigned long end;
1629 :
1630 : pm.pos = 0;
1631 : end = (start_vaddr + PAGEMAP_WALK_SIZE) & PAGEMAP_WALK_MASK;
1632 : /* overflow ? */
1633 : if (end < start_vaddr || end > end_vaddr)
1634 : end = end_vaddr;
1635 : ret = mmap_read_lock_killable(mm);
1636 : if (ret)
1637 : goto out_free;
1638 : ret = walk_page_range(mm, start_vaddr, end, &pagemap_ops, &pm);
1639 : mmap_read_unlock(mm);
1640 : start_vaddr = end;
1641 :
1642 : len = min(count, PM_ENTRY_BYTES * pm.pos);
1643 : if (copy_to_user(buf, pm.buffer, len)) {
1644 : ret = -EFAULT;
1645 : goto out_free;
1646 : }
1647 : copied += len;
1648 : buf += len;
1649 : count -= len;
1650 : }
1651 : *ppos += copied;
1652 : if (!ret || ret == PM_END_OF_BUFFER)
1653 : ret = copied;
1654 :
1655 : out_free:
1656 : kfree(pm.buffer);
1657 : out_mm:
1658 : mmput(mm);
1659 : out:
1660 : return ret;
1661 : }
1662 :
1663 : static int pagemap_open(struct inode *inode, struct file *file)
1664 : {
1665 : struct mm_struct *mm;
1666 :
1667 : mm = proc_mem_open(inode, PTRACE_MODE_READ);
1668 : if (IS_ERR(mm))
1669 : return PTR_ERR(mm);
1670 : file->private_data = mm;
1671 : return 0;
1672 : }
1673 :
1674 : static int pagemap_release(struct inode *inode, struct file *file)
1675 : {
1676 : struct mm_struct *mm = file->private_data;
1677 :
1678 : if (mm)
1679 : mmdrop(mm);
1680 : return 0;
1681 : }
1682 :
1683 : const struct file_operations proc_pagemap_operations = {
1684 : .llseek = mem_lseek, /* borrow this */
1685 : .read = pagemap_read,
1686 : .open = pagemap_open,
1687 : .release = pagemap_release,
1688 : };
1689 : #endif /* CONFIG_PROC_PAGE_MONITOR */
1690 :
1691 : #ifdef CONFIG_NUMA
1692 :
1693 : struct numa_maps {
1694 : unsigned long pages;
1695 : unsigned long anon;
1696 : unsigned long active;
1697 : unsigned long writeback;
1698 : unsigned long mapcount_max;
1699 : unsigned long dirty;
1700 : unsigned long swapcache;
1701 : unsigned long node[MAX_NUMNODES];
1702 : };
1703 :
1704 : struct numa_maps_private {
1705 : struct proc_maps_private proc_maps;
1706 : struct numa_maps md;
1707 : };
1708 :
1709 0 : static void gather_stats(struct page *page, struct numa_maps *md, int pte_dirty,
1710 : unsigned long nr_pages)
1711 : {
1712 0 : int count = page_mapcount(page);
1713 :
1714 0 : md->pages += nr_pages;
1715 0 : if (pte_dirty || PageDirty(page))
1716 0 : md->dirty += nr_pages;
1717 :
1718 0 : if (PageSwapCache(page))
1719 : md->swapcache += nr_pages;
1720 :
1721 0 : if (PageActive(page) || PageUnevictable(page))
1722 0 : md->active += nr_pages;
1723 :
1724 0 : if (PageWriteback(page))
1725 0 : md->writeback += nr_pages;
1726 :
1727 0 : if (PageAnon(page))
1728 0 : md->anon += nr_pages;
1729 :
1730 0 : if (count > md->mapcount_max)
1731 0 : md->mapcount_max = count;
1732 :
1733 0 : md->node[page_to_nid(page)] += nr_pages;
1734 0 : }
1735 :
1736 0 : static struct page *can_gather_numa_stats(pte_t pte, struct vm_area_struct *vma,
1737 : unsigned long addr)
1738 : {
1739 0 : struct page *page;
1740 0 : int nid;
1741 :
1742 0 : if (!pte_present(pte))
1743 : return NULL;
1744 :
1745 0 : page = vm_normal_page(vma, addr, pte);
1746 0 : if (!page)
1747 : return NULL;
1748 :
1749 0 : if (PageReserved(page))
1750 : return NULL;
1751 :
1752 0 : nid = page_to_nid(page);
1753 0 : if (!node_isset(nid, node_states[N_MEMORY]))
1754 0 : return NULL;
1755 :
1756 : return page;
1757 : }
1758 :
1759 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1760 0 : static struct page *can_gather_numa_stats_pmd(pmd_t pmd,
1761 : struct vm_area_struct *vma,
1762 : unsigned long addr)
1763 : {
1764 0 : struct page *page;
1765 0 : int nid;
1766 :
1767 0 : if (!pmd_present(pmd))
1768 : return NULL;
1769 :
1770 0 : page = vm_normal_page_pmd(vma, addr, pmd);
1771 0 : if (!page)
1772 : return NULL;
1773 :
1774 0 : if (PageReserved(page))
1775 : return NULL;
1776 :
1777 0 : nid = page_to_nid(page);
1778 0 : if (!node_isset(nid, node_states[N_MEMORY]))
1779 0 : return NULL;
1780 :
1781 : return page;
1782 : }
1783 : #endif
1784 :
1785 0 : static int gather_pte_stats(pmd_t *pmd, unsigned long addr,
1786 : unsigned long end, struct mm_walk *walk)
1787 : {
1788 0 : struct numa_maps *md = walk->private;
1789 0 : struct vm_area_struct *vma = walk->vma;
1790 0 : spinlock_t *ptl;
1791 0 : pte_t *orig_pte;
1792 0 : pte_t *pte;
1793 :
1794 : #ifdef CONFIG_TRANSPARENT_HUGEPAGE
1795 0 : ptl = pmd_trans_huge_lock(pmd, vma);
1796 0 : if (ptl) {
1797 0 : struct page *page;
1798 :
1799 0 : page = can_gather_numa_stats_pmd(*pmd, vma, addr);
1800 0 : if (page)
1801 0 : gather_stats(page, md, pmd_dirty(*pmd),
1802 : HPAGE_PMD_SIZE/PAGE_SIZE);
1803 0 : spin_unlock(ptl);
1804 0 : return 0;
1805 : }
1806 :
1807 0 : if (pmd_trans_unstable(pmd))
1808 : return 0;
1809 : #endif
1810 0 : orig_pte = pte = pte_offset_map_lock(walk->mm, pmd, addr, &ptl);
1811 0 : do {
1812 0 : struct page *page = can_gather_numa_stats(*pte, vma, addr);
1813 0 : if (!page)
1814 0 : continue;
1815 0 : gather_stats(page, md, pte_dirty(*pte), 1);
1816 :
1817 0 : } while (pte++, addr += PAGE_SIZE, addr != end);
1818 0 : pte_unmap_unlock(orig_pte, ptl);
1819 0 : cond_resched();
1820 0 : return 0;
1821 : }
1822 : #ifdef CONFIG_HUGETLB_PAGE
1823 : static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1824 : unsigned long addr, unsigned long end, struct mm_walk *walk)
1825 : {
1826 : pte_t huge_pte = huge_ptep_get(pte);
1827 : struct numa_maps *md;
1828 : struct page *page;
1829 :
1830 : if (!pte_present(huge_pte))
1831 : return 0;
1832 :
1833 : page = pte_page(huge_pte);
1834 : if (!page)
1835 : return 0;
1836 :
1837 : md = walk->private;
1838 : gather_stats(page, md, pte_dirty(huge_pte), 1);
1839 : return 0;
1840 : }
1841 :
1842 : #else
1843 0 : static int gather_hugetlb_stats(pte_t *pte, unsigned long hmask,
1844 : unsigned long addr, unsigned long end, struct mm_walk *walk)
1845 : {
1846 0 : return 0;
1847 : }
1848 : #endif
1849 :
1850 : static const struct mm_walk_ops show_numa_ops = {
1851 : .hugetlb_entry = gather_hugetlb_stats,
1852 : .pmd_entry = gather_pte_stats,
1853 : };
1854 :
1855 : /*
1856 : * Display pages allocated per node and memory policy via /proc.
1857 : */
1858 0 : static int show_numa_map(struct seq_file *m, void *v)
1859 : {
1860 0 : struct numa_maps_private *numa_priv = m->private;
1861 0 : struct proc_maps_private *proc_priv = &numa_priv->proc_maps;
1862 0 : struct vm_area_struct *vma = v;
1863 0 : struct numa_maps *md = &numa_priv->md;
1864 0 : struct file *file = vma->vm_file;
1865 0 : struct mm_struct *mm = vma->vm_mm;
1866 0 : struct mempolicy *pol;
1867 0 : char buffer[64];
1868 0 : int nid;
1869 :
1870 0 : if (!mm)
1871 : return 0;
1872 :
1873 : /* Ensure we start with an empty set of numa_maps statistics. */
1874 0 : memset(md, 0, sizeof(*md));
1875 :
1876 0 : pol = __get_vma_policy(vma, vma->vm_start);
1877 0 : if (pol) {
1878 0 : mpol_to_str(buffer, sizeof(buffer), pol);
1879 0 : mpol_cond_put(pol);
1880 : } else {
1881 0 : mpol_to_str(buffer, sizeof(buffer), proc_priv->task_mempolicy);
1882 : }
1883 :
1884 0 : seq_printf(m, "%08lx %s", vma->vm_start, buffer);
1885 :
1886 0 : if (file) {
1887 0 : seq_puts(m, " file=");
1888 0 : seq_file_path(m, file, "\n\t= ");
1889 0 : } else if (vma->vm_start <= mm->brk && vma->vm_end >= mm->start_brk) {
1890 0 : seq_puts(m, " heap");
1891 0 : } else if (is_stack(vma)) {
1892 0 : seq_puts(m, " stack");
1893 : }
1894 :
1895 0 : if (is_vm_hugetlb_page(vma))
1896 : seq_puts(m, " huge");
1897 :
1898 : /* mmap_lock is held by m_start */
1899 0 : walk_page_vma(vma, &show_numa_ops, md);
1900 :
1901 0 : if (!md->pages)
1902 0 : goto out;
1903 :
1904 0 : if (md->anon)
1905 0 : seq_printf(m, " anon=%lu", md->anon);
1906 :
1907 0 : if (md->dirty)
1908 0 : seq_printf(m, " dirty=%lu", md->dirty);
1909 :
1910 0 : if (md->pages != md->anon && md->pages != md->dirty)
1911 0 : seq_printf(m, " mapped=%lu", md->pages);
1912 :
1913 0 : if (md->mapcount_max > 1)
1914 0 : seq_printf(m, " mapmax=%lu", md->mapcount_max);
1915 :
1916 0 : if (md->swapcache)
1917 0 : seq_printf(m, " swapcache=%lu", md->swapcache);
1918 :
1919 0 : if (md->active < md->pages && !is_vm_hugetlb_page(vma))
1920 0 : seq_printf(m, " active=%lu", md->active);
1921 :
1922 0 : if (md->writeback)
1923 0 : seq_printf(m, " writeback=%lu", md->writeback);
1924 :
1925 0 : for_each_node_state(nid, N_MEMORY)
1926 0 : if (md->node[nid])
1927 0 : seq_printf(m, " N%d=%lu", nid, md->node[nid]);
1928 :
1929 0 : seq_printf(m, " kernelpagesize_kB=%lu", vma_kernel_pagesize(vma) >> 10);
1930 0 : out:
1931 0 : seq_putc(m, '\n');
1932 0 : return 0;
1933 : }
1934 :
1935 : static const struct seq_operations proc_pid_numa_maps_op = {
1936 : .start = m_start,
1937 : .next = m_next,
1938 : .stop = m_stop,
1939 : .show = show_numa_map,
1940 : };
1941 :
1942 0 : static int pid_numa_maps_open(struct inode *inode, struct file *file)
1943 : {
1944 0 : return proc_maps_open(inode, file, &proc_pid_numa_maps_op,
1945 : sizeof(struct numa_maps_private));
1946 : }
1947 :
1948 : const struct file_operations proc_pid_numa_maps_operations = {
1949 : .open = pid_numa_maps_open,
1950 : .read = seq_read,
1951 : .llseek = seq_lseek,
1952 : .release = proc_map_release,
1953 : };
1954 :
1955 : #endif /* CONFIG_NUMA */
|
