projects/linux·blob master · refs

344 lines · 10.4 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Common interface for implementing a memory balloon, including support
4 * for migration of pages inflated in a memory balloon.
5 *
6 * Copyright (C) 2012, Red Hat, Inc. Rafael Aquini <[email protected]>
7 */
8#include <linux/mm.h>
9#include <linux/slab.h>
10#include <linux/export.h>
11#include <linux/balloon.h>
12
13/*
14 * Lock protecting the balloon_dev_info of all devices. We don't really
15 * expect more than one device.
16 */
17static DEFINE_SPINLOCK(balloon_pages_lock);
18
19/**
20 * balloon_page_insert - insert a page into the balloon's page list and make
21 * the page->private assignment accordingly.
22 * @balloon : pointer to balloon device
23 * @page : page to be assigned as a 'balloon page'
24 *
25 * Caller must ensure the balloon_pages_lock is held.
26 */
27static void balloon_page_insert(struct balloon_dev_info *balloon,
28 struct page *page)
29{
30 lockdep_assert_held(&balloon_pages_lock);
31 __SetPageOffline(page);
32 if (IS_ENABLED(CONFIG_BALLOON_MIGRATION)) {
33 SetPageMovableOps(page);
34 set_page_private(page, (unsigned long)balloon);
35 }
36 list_add(&page->lru, &balloon->pages);
37}
38
39/**
40 * balloon_page_finalize - prepare a balloon page that was removed from the
41 * balloon list for release to the page allocator
42 * @page: page to be released to the page allocator
43 *
44 * Caller must ensure the balloon_pages_lock is held.
45 */
46static void balloon_page_finalize(struct page *page)
47{
48 lockdep_assert_held(&balloon_pages_lock);
49 if (IS_ENABLED(CONFIG_BALLOON_MIGRATION))
50 set_page_private(page, 0);
51 /* PageOffline is sticky until the page is freed to the buddy. */
52}
53
54static void balloon_page_enqueue_one(struct balloon_dev_info *b_dev_info,
55 struct page *page)
56{
57 balloon_page_insert(b_dev_info, page);
58 if (b_dev_info->adjust_managed_page_count)
59 adjust_managed_page_count(page, -1);
60 __count_vm_event(BALLOON_INFLATE);
61 inc_node_page_state(page, NR_BALLOON_PAGES);
62}
63
64/**
65 * balloon_page_list_enqueue() - inserts a list of pages into the balloon page
66 * list.
67 * @b_dev_info: balloon device descriptor where we will insert a new page to
68 * @pages: pages to enqueue - allocated using balloon_page_alloc.
69 *
70 * Driver must call this function to properly enqueue balloon pages before
71 * definitively removing them from the guest system.
72 *
73 * Return: number of pages that were enqueued.
74 */
75size_t balloon_page_list_enqueue(struct balloon_dev_info *b_dev_info,
76 struct list_head *pages)
77{
78 struct page *page, *tmp;
79 unsigned long flags;
80 size_t n_pages = 0;
81
82 spin_lock_irqsave(&balloon_pages_lock, flags);
83 list_for_each_entry_safe(page, tmp, pages, lru) {
84 list_del(&page->lru);
85 balloon_page_enqueue_one(b_dev_info, page);
86 n_pages++;
87 }
88 spin_unlock_irqrestore(&balloon_pages_lock, flags);
89 return n_pages;
90}
91EXPORT_SYMBOL_GPL(balloon_page_list_enqueue);
92
93/**
94 * balloon_page_list_dequeue() - removes pages from balloon's page list and
95 * returns a list of the pages.
96 * @b_dev_info: balloon device descriptor where we will grab a page from.
97 * @pages: pointer to the list of pages that would be returned to the caller.
98 * @n_req_pages: number of requested pages.
99 *
100 * Driver must call this function to properly de-allocate a previous enlisted
101 * balloon pages before definitively releasing it back to the guest system.
102 * This function tries to remove @n_req_pages from the ballooned pages and
103 * return them to the caller in the @pages list.
104 *
105 * Note that this function may fail to dequeue some pages even if the balloon
106 * isn't empty - since the page list can be temporarily empty due to compaction
107 * of isolated pages.
108 *
109 * Return: number of pages that were added to the @pages list.
110 */
111size_t balloon_page_list_dequeue(struct balloon_dev_info *b_dev_info,
112 struct list_head *pages, size_t n_req_pages)
113{
114 struct page *page, *tmp;
115 unsigned long flags;
116 size_t n_pages = 0;
117
118 spin_lock_irqsave(&balloon_pages_lock, flags);
119 list_for_each_entry_safe(page, tmp, &b_dev_info->pages, lru) {
120 if (n_pages == n_req_pages)
121 break;
122 list_del(&page->lru);
123 if (b_dev_info->adjust_managed_page_count)
124 adjust_managed_page_count(page, 1);
125 balloon_page_finalize(page);
126 __count_vm_event(BALLOON_DEFLATE);
127 list_add(&page->lru, pages);
128 dec_node_page_state(page, NR_BALLOON_PAGES);
129 n_pages++;
130 }
131 spin_unlock_irqrestore(&balloon_pages_lock, flags);
132
133 return n_pages;
134}
135EXPORT_SYMBOL_GPL(balloon_page_list_dequeue);
136
137/**
138 * balloon_page_alloc - allocates a new page for insertion into the balloon
139 * page list.
140 *
141 * Driver must call this function to properly allocate a new balloon page.
142 * Driver must call balloon_page_enqueue before definitively removing the page
143 * from the guest system.
144 *
145 * Return: struct page for the allocated page or NULL on allocation failure.
146 */
147struct page *balloon_page_alloc(void)
148{
149 gfp_t gfp_flags = __GFP_NOMEMALLOC | __GFP_NORETRY | __GFP_NOWARN;
150
151 if (IS_ENABLED(CONFIG_BALLOON_MIGRATION))
152 gfp_flags |= GFP_HIGHUSER_MOVABLE;
153 else
154 gfp_flags |= GFP_HIGHUSER;
155
156 return alloc_page(gfp_flags);
157}
158EXPORT_SYMBOL_GPL(balloon_page_alloc);
159
160/**
161 * balloon_page_enqueue - inserts a new page into the balloon page list.
162 *
163 * @b_dev_info: balloon device descriptor where we will insert a new page
164 * @page: new page to enqueue - allocated using balloon_page_alloc.
165 *
166 * Drivers must call this function to properly enqueue a new allocated balloon
167 * page before definitively removing the page from the guest system.
168 *
169 * Drivers must not enqueue pages while page->lru is still in
170 * use, and must not use page->lru until a page was unqueued again.
171 */
172void balloon_page_enqueue(struct balloon_dev_info *b_dev_info,
173 struct page *page)
174{
175 unsigned long flags;
176
177 spin_lock_irqsave(&balloon_pages_lock, flags);
178 balloon_page_enqueue_one(b_dev_info, page);
179 spin_unlock_irqrestore(&balloon_pages_lock, flags);
180}
181EXPORT_SYMBOL_GPL(balloon_page_enqueue);
182
183/**
184 * balloon_page_dequeue - removes a page from balloon's page list and returns
185 * its address to allow the driver to release the page.
186 * @b_dev_info: balloon device descriptor where we will grab a page from.
187 *
188 * Driver must call this function to properly dequeue a previously enqueued page
189 * before definitively releasing it back to the guest system.
190 *
191 * Caller must perform its own accounting to ensure that this
192 * function is called only if some pages are actually enqueued.
193 *
194 * Note that this function may fail to dequeue some pages even if there are
195 * some enqueued pages - since the page list can be temporarily empty due to
196 * the compaction of isolated pages.
197 *
198 * TODO: remove the caller accounting requirements, and allow caller to wait
199 * until all pages can be dequeued.
200 *
201 * Return: struct page for the dequeued page, or NULL if no page was dequeued.
202 */
203struct page *balloon_page_dequeue(struct balloon_dev_info *b_dev_info)
204{
205 unsigned long flags;
206 LIST_HEAD(pages);
207 int n_pages;
208
209 n_pages = balloon_page_list_dequeue(b_dev_info, &pages, 1);
210
211 if (n_pages != 1) {
212 /*
213 * If we are unable to dequeue a balloon page because the page
214 * list is empty and there are no isolated pages, then something
215 * went out of track and some balloon pages are lost.
216 * BUG() here, otherwise the balloon driver may get stuck in
217 * an infinite loop while attempting to release all its pages.
218 */
219 spin_lock_irqsave(&balloon_pages_lock, flags);
220 if (unlikely(list_empty(&b_dev_info->pages) &&
221 !b_dev_info->isolated_pages))
222 BUG();
223 spin_unlock_irqrestore(&balloon_pages_lock, flags);
224 return NULL;
225 }
226 return list_first_entry(&pages, struct page, lru);
227}
228EXPORT_SYMBOL_GPL(balloon_page_dequeue);
229
230#ifdef CONFIG_BALLOON_MIGRATION
231static struct balloon_dev_info *balloon_page_device(struct page *page)
232{
233 return (struct balloon_dev_info *)page_private(page);
234}
235
236static bool balloon_page_isolate(struct page *page, isolate_mode_t mode)
237
238{
239 struct balloon_dev_info *b_dev_info;
240 unsigned long flags;
241
242 spin_lock_irqsave(&balloon_pages_lock, flags);
243 b_dev_info = balloon_page_device(page);
244 if (!b_dev_info) {
245 /*
246 * The page already got deflated and removed from the
247 * balloon list.
248 */
249 spin_unlock_irqrestore(&balloon_pages_lock, flags);
250 return false;
251 }
252 list_del(&page->lru);
253 b_dev_info->isolated_pages++;
254 spin_unlock_irqrestore(&balloon_pages_lock, flags);
255
256 return true;
257}
258
259static void balloon_page_putback(struct page *page)
260{
261 struct balloon_dev_info *b_dev_info = balloon_page_device(page);
262 unsigned long flags;
263
264 /*
265 * When we isolated the page, the page was still inflated in a balloon
266 * device. As isolated balloon pages cannot get deflated, we still have
267 * a balloon device here.
268 */
269 if (WARN_ON_ONCE(!b_dev_info))
270 return;
271
272 spin_lock_irqsave(&balloon_pages_lock, flags);
273 list_add(&page->lru, &b_dev_info->pages);
274 b_dev_info->isolated_pages--;
275 spin_unlock_irqrestore(&balloon_pages_lock, flags);
276}
277
278static int balloon_page_migrate(struct page *newpage, struct page *page,
279 enum migrate_mode mode)
280{
281 struct balloon_dev_info *b_dev_info = balloon_page_device(page);
282 unsigned long flags;
283 int rc;
284
285 /*
286 * When we isolated the page, the page was still inflated in a balloon
287 * device. As isolated balloon pages cannot get deflated, we still have
288 * a balloon device here.
289 */
290 if (WARN_ON_ONCE(!b_dev_info))
291 return -EAGAIN;
292
293 rc = b_dev_info->migratepage(b_dev_info, newpage, page, mode);
294 if (rc < 0 && rc != -ENOENT)
295 return rc;
296
297 spin_lock_irqsave(&balloon_pages_lock, flags);
298 if (!rc) {
299 /* Insert the new page into the balloon list. */
300 get_page(newpage);
301 balloon_page_insert(b_dev_info, newpage);
302 __count_vm_event(BALLOON_MIGRATE);
303
304 if (b_dev_info->adjust_managed_page_count &&
305 page_zone(page) != page_zone(newpage)) {
306 /*
307 * When we migrate a page to a different zone we
308 * have to fixup the count of both involved zones.
309 */
310 adjust_managed_page_count(page, 1);
311 adjust_managed_page_count(newpage, -1);
312 }
313 } else {
314 /* Old page was deflated but new page not inflated. */
315 __count_vm_event(BALLOON_DEFLATE);
316
317 if (b_dev_info->adjust_managed_page_count)
318 adjust_managed_page_count(page, 1);
319 }
320
321 b_dev_info->isolated_pages--;
322
323 /* Free the now-deflated page we isolated in balloon_page_isolate(). */
324 balloon_page_finalize(page);
325 spin_unlock_irqrestore(&balloon_pages_lock, flags);
326
327 put_page(page);
328
329 return 0;
330}
331
332static const struct movable_operations balloon_mops = {
333 .migrate_page = balloon_page_migrate,
334 .isolate_page = balloon_page_isolate,
335 .putback_page = balloon_page_putback,
336};
337
338static int __init balloon_init(void)
339{
340 return set_movable_ops(&balloon_mops, PGTY_offline);
341}
342core_initcall(balloon_init);
343
344#endif /* CONFIG_BALLOON_MIGRATION */