Line data Source code
1 : // SPDX-License-Identifier: GPL-2.0-only
2 : /*
3 : * mm/balloon_compaction.c
4 : *
5 : * Common interface for making balloon pages movable by compaction.
6 : *
7 : * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <aquini@redhat.com>
8 : */
9 : #include <linux/mm.h>
10 : #include <linux/slab.h>
11 : #include <linux/export.h>
12 : #include <linux/balloon_compaction.h>
13 :
14 0 : static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,
15 : struct page *page)
16 : {
17 : /*
18 : * Block others from accessing the 'page' when we get around to
19 : * establishing additional references. We should be the only one
20 : * holding a reference to the 'page' at this point. If we are not, then
21 : * memory corruption is possible and we should stop execution.
22 : */
23 0 : BUG_ON(!trylock_page(page));
24 0 : balloon_page_insert(b_dev_info, page);
25 0 : unlock_page(page);
26 0 : __count_vm_event(BALLOON_INFLATE);
27 0 : }
28 :
29 : /**
30 : * balloon_page_list_enqueue() - inserts a list of pages into the balloon page
31 : * list.
32 : * @b_dev_info: balloon device descriptor where we will insert a new page to
33 : * @pages: pages to enqueue - allocated using balloon_page_alloc.
34 : *
35 : * Driver must call this function to properly enqueue balloon pages before
36 : * definitively removing them from the guest system.
37 : *
38 : * Return: number of pages that were enqueued.
39 : */
40 0 : size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
41 : struct list_head *pages)
42 : {
43 0 : struct page *page, *tmp;
44 0 : unsigned long flags;
45 0 : size_t n_pages = 0;
46 :
47 0 : spin_lock_irqsave(&b_dev_info->pages_lock, flags);
48 0 : list_for_each_entry_safe(page, tmp, pages, lru) {
49 0 : list_del(&page->lru);
50 0 : balloon_page_enqueue_one(b_dev_info, page);
51 0 : n_pages++;
52 : }
53 0 : spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
54 0 : return n_pages;
55 : }
56 : EXPORT_SYMBOL_GPL(balloon_page_list_enqueue);
57 :
58 : /**
59 : * balloon_page_list_dequeue() - removes pages from balloon's page list and
60 : * returns a list of the pages.
61 : * @b_dev_info: balloon device decriptor where we will grab a page from.
62 : * @pages: pointer to the list of pages that would be returned to the caller.
63 : * @n_req_pages: number of requested pages.
64 : *
65 : * Driver must call this function to properly de-allocate a previous enlisted
66 : * balloon pages before definitively releasing it back to the guest system.
67 : * This function tries to remove @n_req_pages from the ballooned pages and
68 : * return them to the caller in the @pages list.
69 : *
70 : * Note that this function may fail to dequeue some pages even if the balloon
71 : * isn't empty - since the page list can be temporarily empty due to compaction
72 : * of isolated pages.
73 : *
74 : * Return: number of pages that were added to the @pages list.
75 : */
76 0 : size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
77 : struct list_head *pages, size_t n_req_pages)
78 : {
79 0 : struct page *page, *tmp;
80 0 : unsigned long flags;
81 0 : size_t n_pages = 0;
82 :
83 0 : spin_lock_irqsave(&b_dev_info->pages_lock, flags);
84 0 : list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
85 0 : if (n_pages == n_req_pages)
86 : break;
87 :
88 : /*
89 : * Block others from accessing the 'page' while we get around to
90 : * establishing additional references and preparing the 'page'
91 : * to be released by the balloon driver.
92 : */
93 0 : if (!trylock_page(page))
94 0 : continue;
95 :
96 0 : if (IS_ENABLED(CONFIG_BALLOON_COMPACTION) &&
97 : PageIsolated(page)) {
98 : /* raced with isolation */
99 : unlock_page(page);
100 : continue;
101 : }
102 0 : balloon_page_delete(page);
103 0 : __count_vm_event(BALLOON_DEFLATE);
104 0 : list_add(&page->lru, pages);
105 0 : unlock_page(page);
106 0 : n_pages++;
107 : }
108 0 : spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
109 :
110 0 : return n_pages;
111 : }
112 : EXPORT_SYMBOL_GPL(balloon_page_list_dequeue);
113 :
114 : /*
115 : * balloon_page_alloc - allocates a new page for insertion into the balloon
116 : * page list.
117 : *
118 : * Driver must call this function to properly allocate a new balloon page.
119 : * Driver must call balloon_page_enqueue before definitively removing the page
120 : * from the guest system.
121 : *
122 : * Return: struct page for the allocated page or NULL on allocation failure.
123 : */
124 0 : struct page *balloon_page_alloc(void)
125 : {
126 0 : struct page *page = alloc_page(balloon_mapping_gfp_mask() |
127 : __GFP_NOMEMALLOC | __GFP_NORETRY |
128 : __GFP_NOWARN);
129 0 : return page;
130 : }
131 : EXPORT_SYMBOL_GPL(balloon_page_alloc);
132 :
133 : /*
134 : * balloon_page_enqueue - inserts a new page into the balloon page list.
135 : *
136 : * @b_dev_info: balloon device descriptor where we will insert a new page
137 : * @page: new page to enqueue - allocated using balloon_page_alloc.
138 : *
139 : * Drivers must call this function to properly enqueue a new allocated balloon
140 : * page before definitively removing the page from the guest system.
141 : *
142 : * Drivers must not call balloon_page_enqueue on pages that have been pushed to
143 : * a list with balloon_page_push before removing them with balloon_page_pop. To
144 : * enqueue a list of pages, use balloon_page_list_enqueue instead.
145 : */
146 0 : void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
147 : struct page *page)
148 : {
149 0 : unsigned long flags;
150 :
151 0 : spin_lock_irqsave(&b_dev_info->pages_lock, flags);
152 0 : balloon_page_enqueue_one(b_dev_info, page);
153 0 : spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
154 0 : }
155 : EXPORT_SYMBOL_GPL(balloon_page_enqueue);
156 :
157 : /*
158 : * balloon_page_dequeue - removes a page from balloon's page list and returns
159 : * its address to allow the driver to release the page.
160 : * @b_dev_info: balloon device decriptor where we will grab a page from.
161 : *
162 : * Driver must call this function to properly dequeue a previously enqueued page
163 : * before definitively releasing it back to the guest system.
164 : *
165 : * Caller must perform its own accounting to ensure that this
166 : * function is called only if some pages are actually enqueued.
167 : *
168 : * Note that this function may fail to dequeue some pages even if there are
169 : * some enqueued pages - since the page list can be temporarily empty due to
170 : * the compaction of isolated pages.
171 : *
172 : * TODO: remove the caller accounting requirements, and allow caller to wait
173 : * until all pages can be dequeued.
174 : *
175 : * Return: struct page for the dequeued page, or NULL if no page was dequeued.
176 : */
177 0 : struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
178 : {
179 0 : unsigned long flags;
180 0 : LIST_HEAD(pages);
181 0 : int n_pages;
182 :
183 0 : n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1);
184 :
185 0 : if (n_pages != 1) {
186 : /*
187 : * If we are unable to dequeue a balloon page because the page
188 : * list is empty and there are no isolated pages, then something
189 : * went out of track and some balloon pages are lost.
190 : * BUG() here, otherwise the balloon driver may get stuck in
191 : * an infinite loop while attempting to release all its pages.
192 : */
193 0 : spin_lock_irqsave(&b_dev_info->pages_lock, flags);
194 0 : if (unlikely(list_empty(&b_dev_info->pages) &&
195 : !b_dev_info->isolated_pages))
196 0 : BUG();
197 0 : spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
198 0 : return NULL;
199 : }
200 0 : return list_first_entry(&pages, struct page, lru);
201 : }
202 : EXPORT_SYMBOL_GPL(balloon_page_dequeue);
203 :
204 : #ifdef CONFIG_BALLOON_COMPACTION
205 :
206 : bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
207 :
208 : {
209 : struct balloon_dev_info *b_dev_info = balloon_page_device(page);
210 : unsigned long flags;
211 :
212 : spin_lock_irqsave(&b_dev_info->pages_lock, flags);
213 : list_del(&page->lru);
214 : b_dev_info->isolated_pages++;
215 : spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
216 :
217 : return true;
218 : }
219 :
220 : void balloon_page_putback(struct page *page)
221 : {
222 : struct balloon_dev_info *b_dev_info = balloon_page_device(page);
223 : unsigned long flags;
224 :
225 : spin_lock_irqsave(&b_dev_info->pages_lock, flags);
226 : list_add(&page->lru, &b_dev_info->pages);
227 : b_dev_info->isolated_pages--;
228 : spin_unlock_irqrestore(&b_dev_info->pages_lock, flags);
229 : }
230 :
231 :
232 : /* move_to_new_page() counterpart for a ballooned page */
233 : int balloon_page_migrate(struct address_space *mapping,
234 : struct page *newpage, struct page *page,
235 : enum migrate_mode mode)
236 : {
237 : struct balloon_dev_info *balloon = balloon_page_device(page);
238 :
239 : /*
240 : * We can not easily support the no copy case here so ignore it as it
241 : * is unlikely to be used with balloon pages. See include/linux/hmm.h
242 : * for a user of the MIGRATE_SYNC_NO_COPY mode.
243 : */
244 : if (mode == MIGRATE_SYNC_NO_COPY)
245 : return -EINVAL;
246 :
247 : VM_BUG_ON_PAGE(!PageLocked(page), page);
248 : VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
249 :
250 : return balloon->migratepage(balloon, newpage, page, mode);
251 : }
252 :
253 : const struct address_space_operations balloon_aops = {
254 : .migratepage = balloon_page_migrate,
255 : .isolate_page = balloon_page_isolate,
256 : .putback_page = balloon_page_putback,
257 : };
258 : EXPORT_SYMBOL_GPL(balloon_aops);
259 :
260 : #endif /* CONFIG_BALLOON_COMPACTION */
|
