projects/linux·blob master · refs

429 lines · 12.0 KiB

1// SPDX-License-Identifier: GPL-2.0
2#include <linux/mm.h>
3#include <linux/mmzone.h>
4#include <linux/page_reporting.h>
5#include <linux/gfp.h>
6#include <linux/export.h>
7#include <linux/module.h>
8#include <linux/delay.h>
9#include <linux/scatterlist.h>
10
11#include "page_alloc.h"
12#include "page_reporting.h"
13#include "internal.h"
14
15/* Initialize to an unsupported value */
16unsigned int page_reporting_order = PAGE_REPORTING_ORDER_UNSPECIFIED;
17
18static int page_order_update_notify(const char *val, const struct kernel_param *kp)
19{
20 /*
21 * If param is set beyond this limit, order is set to default
22 * pageblock_order value
23 */
24 return param_set_uint_minmax(val, kp, 0, MAX_PAGE_ORDER);
25}
26
27static const struct kernel_param_ops page_reporting_param_ops = {
28 .set = &page_order_update_notify,
29 /*
30 * For the get op, use param_get_int instead of param_get_uint.
31 * This is to make sure that when unset the initialized value of
32 * -1 is shown correctly
33 */
34 .get = &param_get_int,
35};
36
37module_param_cb(page_reporting_order, &page_reporting_param_ops,
38 &page_reporting_order, 0644);
39MODULE_PARM_DESC(page_reporting_order, "Set page reporting order");
40
41/*
42 * This symbol is also a kernel parameter. Export the page_reporting_order
43 * symbol so that other drivers can access it to control order values without
44 * having to introduce another configurable parameter. Only one driver can
45 * register with the page_reporting driver for the service, so we have just
46 * one control parameter for the use case(which can be accessed in both
47 * drivers)
48 */
49EXPORT_SYMBOL_GPL(page_reporting_order);
50
51static unsigned int page_reporting_delay_ms = 2 * MSEC_PER_SEC;
52module_param(page_reporting_delay_ms, uint, 0644);
53MODULE_PARM_DESC(page_reporting_delay_ms,
54 "Set page reporting delay in milliseconds");
55
56static struct page_reporting_dev_info __rcu *pr_dev_info __read_mostly;
57
58enum {
59 PAGE_REPORTING_IDLE = 0,
60 PAGE_REPORTING_REQUESTED,
61 PAGE_REPORTING_ACTIVE
62};
63
64/* schedule work for page reporting */
65static void page_reporting_schedule_work(struct page_reporting_dev_info *prdev)
66{
67 queue_delayed_work(system_freezable_wq, &prdev->work,
68 msecs_to_jiffies(page_reporting_delay_ms));
69}
70
71/* request page reporting */
72static void
73__page_reporting_request(struct page_reporting_dev_info *prdev)
74{
75 unsigned int state;
76
77 /* Check to see if we are in desired state */
78 state = atomic_read(&prdev->state);
79 if (state == PAGE_REPORTING_REQUESTED)
80 return;
81
82 /*
83 * If reporting is already active there is nothing we need to do.
84 * Test against 0 as that represents PAGE_REPORTING_IDLE.
85 */
86 state = atomic_xchg(&prdev->state, PAGE_REPORTING_REQUESTED);
87 if (state != PAGE_REPORTING_IDLE)
88 return;
89
90 /*
91 * Delay the start of work to allow a sizable queue to build.
92 * We limit this based on page_reporting_delay_ms.
93 */
94 page_reporting_schedule_work(prdev);
95}
96
97/* notify prdev of free page reporting request */
98void __page_reporting_notify(void)
99{
100 struct page_reporting_dev_info *prdev;
101
102 /*
103 * We use RCU to protect the pr_dev_info pointer. In almost all
104 * cases this should be present, however in the unlikely case of
105 * a shutdown this will be NULL and we should exit.
106 */
107 rcu_read_lock();
108 prdev = rcu_dereference(pr_dev_info);
109 if (likely(prdev))
110 __page_reporting_request(prdev);
111
112 rcu_read_unlock();
113}
114
115static void
116page_reporting_drain(struct page_reporting_dev_info *prdev,
117 struct scatterlist *sgl, unsigned int nents, bool reported)
118{
119 struct scatterlist *sg = sgl;
120
121 /*
122 * Drain the now reported pages back into their respective
123 * free lists/areas. We assume at least one page is populated.
124 */
125 do {
126 struct page *page = sg_page(sg);
127 int mt = get_pageblock_migratetype(page);
128 unsigned int order = get_order(sg->length);
129
130 __putback_isolated_page(page, order, mt);
131
132 /* If the pages were not reported due to error skip flagging */
133 if (!reported)
134 continue;
135
136 /*
137 * If page was not commingled with another page we can
138 * consider the result to be "reported" since the page
139 * hasn't been modified, otherwise we will need to
140 * report on the new larger page when we make our way
141 * up to that higher order.
142 */
143 if (PageBuddy(page) && buddy_order(page) == order)
144 __SetPageReported(page);
145 } while ((sg = sg_next(sg)));
146
147 /* reinitialize scatterlist now that it is empty */
148 sg_init_table(sgl, nents);
149}
150
151/*
152 * The page reporting cycle consists of 4 stages, fill, report, drain, and
153 * idle. We will cycle through the first 3 stages until we cannot obtain a
154 * full scatterlist of pages, in that case we will switch to idle.
155 */
156static int
157page_reporting_cycle(struct page_reporting_dev_info *prdev, struct zone *zone,
158 unsigned int order, unsigned int mt,
159 struct scatterlist *sgl, unsigned int *offset)
160{
161 struct free_area *area = &zone->free_area[order];
162 struct list_head *list = &area->free_list[mt];
163 unsigned int page_len = PAGE_SIZE << order;
164 struct page *page, *next;
165 long budget;
166 int err = 0;
167
168 /*
169 * Perform early check, if free area is empty there is
170 * nothing to process so we can skip this free_list.
171 */
172 if (list_empty(list))
173 return err;
174
175 spin_lock_irq(&zone->lock);
176
177 /*
178 * Limit how many calls we will be making to the page reporting
179 * device for this list. By doing this we avoid processing any
180 * given list for too long.
181 *
182 * The current value used allows us enough calls to process over a
183 * sixteenth of the current list plus one additional call to handle
184 * any pages that may have already been present from the previous
185 * list processed. This should result in us reporting all pages on
186 * an idle system in about 30 seconds.
187 */
188 budget = DIV_ROUND_UP(area->nr_free, prdev->capacity * 16);
189
190 /* loop through free list adding unreported pages to sg list */
191 list_for_each_entry_safe(page, next, list, lru) {
192 /* We are going to skip over the reported pages. */
193 if (PageReported(page))
194 continue;
195
196 /*
197 * If we fully consumed our budget then update our
198 * state to indicate that we are requesting additional
199 * processing and exit this list.
200 */
201 if (budget < 0) {
202 atomic_set(&prdev->state, PAGE_REPORTING_REQUESTED);
203 next = page;
204 break;
205 }
206
207 /* Attempt to pull page from list and place in scatterlist */
208 if (*offset) {
209 if (!__isolate_free_page(page, order)) {
210 next = page;
211 break;
212 }
213
214 /* Add page to scatter list */
215 --(*offset);
216 sg_set_page(&sgl[*offset], page, page_len, 0);
217
218 continue;
219 }
220
221 /*
222 * Make the first non-reported page in the free list
223 * the new head of the free list before we release the
224 * zone lock.
225 */
226 if (!list_is_first(&page->lru, list))
227 list_rotate_to_front(&page->lru, list);
228
229 /* release lock before waiting on report processing */
230 spin_unlock_irq(&zone->lock);
231
232 /* begin processing pages in local list */
233 err = prdev->report(prdev, sgl, prdev->capacity);
234
235 /* reset offset since the full list was reported */
236 *offset = prdev->capacity;
237
238 /* update budget to reflect call to report function */
239 budget--;
240
241 /* reacquire zone lock and resume processing */
242 spin_lock_irq(&zone->lock);
243
244 /* flush reported pages from the sg list */
245 page_reporting_drain(prdev, sgl, prdev->capacity, !err);
246
247 /*
248 * Reset next to first entry, the old next isn't valid
249 * since we dropped the lock to report the pages
250 */
251 next = list_first_entry(list, struct page, lru);
252
253 /* exit on error */
254 if (err)
255 break;
256 }
257
258 /* Rotate any leftover pages to the head of the freelist */
259 if (!list_entry_is_head(next, list, lru) && !list_is_first(&next->lru, list))
260 list_rotate_to_front(&next->lru, list);
261
262 spin_unlock_irq(&zone->lock);
263
264 return err;
265}
266
267static int
268page_reporting_process_zone(struct page_reporting_dev_info *prdev,
269 struct scatterlist *sgl, struct zone *zone)
270{
271 unsigned int order, mt, leftover, offset = prdev->capacity;
272 unsigned long watermark;
273 int err = 0;
274
275 /* Generate minimum watermark to be able to guarantee progress */
276 watermark = low_wmark_pages(zone) +
277 (prdev->capacity << page_reporting_order);
278
279 /*
280 * Cancel request if insufficient free memory or if we failed
281 * to allocate page reporting statistics for the zone.
282 */
283 if (!zone_watermark_ok(zone, 0, watermark, 0, ALLOC_CMA))
284 return err;
285
286 /* Process each free list starting from lowest order/mt */
287 for (order = page_reporting_order; order < NR_PAGE_ORDERS; order++) {
288 for (mt = 0; mt < MIGRATE_TYPES; mt++) {
289 /* We do not pull pages from the isolate free list */
290 if (is_migrate_isolate(mt))
291 continue;
292
293 err = page_reporting_cycle(prdev, zone, order, mt,
294 sgl, &offset);
295 if (err)
296 return err;
297 }
298 }
299
300 /* report the leftover pages before going idle */
301 leftover = prdev->capacity - offset;
302 if (leftover) {
303 sgl = &sgl[offset];
304 err = prdev->report(prdev, sgl, leftover);
305
306 /* flush any remaining pages out from the last report */
307 spin_lock_irq(&zone->lock);
308 page_reporting_drain(prdev, sgl, leftover, !err);
309 spin_unlock_irq(&zone->lock);
310 }
311
312 return err;
313}
314
315static void page_reporting_process(struct work_struct *work)
316{
317 struct delayed_work *d_work = to_delayed_work(work);
318 struct page_reporting_dev_info *prdev =
319 container_of(d_work, struct page_reporting_dev_info, work);
320 int err = 0, state = PAGE_REPORTING_ACTIVE;
321 struct scatterlist *sgl;
322 struct zone *zone;
323
324 /*
325 * Change the state to "Active" so that we can track if there is
326 * anyone requests page reporting after we complete our pass. If
327 * the state is not altered by the end of the pass we will switch
328 * to idle and quit scheduling reporting runs.
329 */
330 atomic_set(&prdev->state, state);
331
332 /* allocate scatterlist to store pages being reported on */
333 sgl = kmalloc_objs(*sgl, prdev->capacity);
334 if (!sgl)
335 goto err_out;
336
337 sg_init_table(sgl, prdev->capacity);
338
339 for_each_zone(zone) {
340 err = page_reporting_process_zone(prdev, sgl, zone);
341 if (err)
342 break;
343 }
344
345 kfree(sgl);
346err_out:
347 /*
348 * If the state has reverted back to requested then there may be
349 * additional pages to be processed. We will defer by
350 * page_reporting_delay_ms to allow more pages to accumulate.
351 */
352 state = atomic_cmpxchg(&prdev->state, state, PAGE_REPORTING_IDLE);
353 if (state == PAGE_REPORTING_REQUESTED)
354 page_reporting_schedule_work(prdev);
355}
356
357static DEFINE_MUTEX(page_reporting_mutex);
358DEFINE_STATIC_KEY_FALSE(page_reporting_enabled);
359
360int page_reporting_register(struct page_reporting_dev_info *prdev)
361{
362 int err = 0;
363
364 mutex_lock(&page_reporting_mutex);
365
366 /* nothing to do if already in use */
367 if (rcu_dereference_protected(pr_dev_info,
368 lockdep_is_held(&page_reporting_mutex))) {
369 err = -EBUSY;
370 goto err_out;
371 }
372
373 /*
374 * If the page_reporting_order value is not set, we check if
375 * an order is provided from the driver that is performing the
376 * registration. If that is not provided either, we default to
377 * pageblock_order.
378 */
379
380 if (page_reporting_order == PAGE_REPORTING_ORDER_UNSPECIFIED) {
381 if (prdev->order != PAGE_REPORTING_ORDER_UNSPECIFIED &&
382 prdev->order <= MAX_PAGE_ORDER)
383 page_reporting_order = prdev->order;
384 else
385 page_reporting_order = pageblock_order;
386 }
387
388 if (!prdev->capacity || prdev->capacity > PAGE_REPORTING_CAPACITY)
389 prdev->capacity = PAGE_REPORTING_CAPACITY;
390
391 /* initialize state and work structures */
392 atomic_set(&prdev->state, PAGE_REPORTING_IDLE);
393 INIT_DELAYED_WORK(&prdev->work, &page_reporting_process);
394
395 /* Begin initial flush of zones */
396 __page_reporting_request(prdev);
397
398 /* Assign device to allow notifications */
399 rcu_assign_pointer(pr_dev_info, prdev);
400
401 /* enable page reporting notification */
402 if (!static_key_enabled(&page_reporting_enabled)) {
403 static_branch_enable(&page_reporting_enabled);
404 pr_info("Free page reporting enabled\n");
405 }
406err_out:
407 mutex_unlock(&page_reporting_mutex);
408
409 return err;
410}
411EXPORT_SYMBOL_GPL(page_reporting_register);
412
413void page_reporting_unregister(struct page_reporting_dev_info *prdev)
414{
415 mutex_lock(&page_reporting_mutex);
416
417 if (prdev == rcu_dereference_protected(pr_dev_info,
418 lockdep_is_held(&page_reporting_mutex))) {
419 /* Disable page reporting notification */
420 RCU_INIT_POINTER(pr_dev_info, NULL);
421 synchronize_rcu();
422
423 /* Flush any existing work, and lock it out */
424 cancel_delayed_work_sync(&prdev->work);
425 }
426
427 mutex_unlock(&page_reporting_mutex);
428}
429EXPORT_SYMBOL_GPL(page_reporting_unregister);