linux - kernel/power/snapshot.c

2931 lines · 77.9 KiB

1	// SPDX-License-Identifier: GPL-2.0-only
2	/*
3	* linux/kernel/power/snapshot.c
4	*
5	* This file provides system snapshot/restore functionality for swsusp.
6	*
7	* Copyright (C) 1998-2005 Pavel Machek <[email protected]>
8	* Copyright (C) 2006 Rafael J. Wysocki <[email protected]>
9	*/
10
11	#define pr_fmt(fmt) "PM: hibernation: " fmt
12
13	#include <linux/version.h>
14	#include <linux/module.h>
15	#include <linux/mm.h>
16	#include <linux/suspend.h>
17	#include <linux/delay.h>
18	#include <linux/bitops.h>
19	#include <linux/spinlock.h>
20	#include <linux/kernel.h>
21	#include <linux/pm.h>
22	#include <linux/device.h>
23	#include <linux/init.h>
24	#include <linux/memblock.h>
25	#include <linux/nmi.h>
26	#include <linux/syscalls.h>
27	#include <linux/console.h>
28	#include <linux/highmem.h>
29	#include <linux/list.h>
30	#include <linux/slab.h>
31	#include <linux/compiler.h>
32	#include <linux/ktime.h>
33	#include <linux/set_memory.h>
34
35	#include <linux/uaccess.h>
36	#include <asm/mmu_context.h>
37	#include <asm/tlbflush.h>
38	#include <asm/io.h>
39
40	#include "power.h"
41
42	#if defined(CONFIG_STRICT_KERNEL_RWX) && defined(CONFIG_ARCH_HAS_SET_MEMORY)
43	static bool hibernate_restore_protection;
44	static bool hibernate_restore_protection_active;
45
46	void enable_restore_image_protection(void)
47	{
48	hibernate_restore_protection = true;
49	}
50
51	static inline void hibernate_restore_protection_begin(void)
52	{
53	hibernate_restore_protection_active = hibernate_restore_protection;
54	}
55
56	static inline void hibernate_restore_protection_end(void)
57	{
58	hibernate_restore_protection_active = false;
59	}
60
61	static inline int __must_check hibernate_restore_protect_page(void *page_address)
62	{
63	if (hibernate_restore_protection_active)
64	return set_memory_ro((unsigned long)page_address, 1);
65	return 0;
66	}
67
68	static inline int hibernate_restore_unprotect_page(void *page_address)
69	{
70	if (hibernate_restore_protection_active)
71	return set_memory_rw((unsigned long)page_address, 1);
72	return 0;
73	}
74	#else
75	static inline void hibernate_restore_protection_begin(void) {}
76	static inline void hibernate_restore_protection_end(void) {}
77	static inline int __must_check hibernate_restore_protect_page(void *page_address) {return 0; }
78	static inline int hibernate_restore_unprotect_page(void *page_address) {return 0; }
79	#endif /* CONFIG_STRICT_KERNEL_RWX && CONFIG_ARCH_HAS_SET_MEMORY */
80
81
82	/*
83	* The calls to set_direct_map_*() should not fail because remapping a page
84	* here means that we only update protection bits in an existing PTE.
85	* It is still worth to have a warning here if something changes and this
86	* will no longer be the case.
87	*/
88	static inline void hibernate_map_page(struct page *page)
89	{
90	if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) {
91	int ret = set_direct_map_default_noflush(page);
92
93	if (ret)
94	pr_warn_once("Failed to remap page\n");
95	} else {
96	debug_pagealloc_map_pages(page, 1);
97	}
98	}
99
100	static inline void hibernate_unmap_page(struct page *page)
101	{
102	if (IS_ENABLED(CONFIG_ARCH_HAS_SET_DIRECT_MAP)) {
103	unsigned long addr = (unsigned long)page_address(page);
104	int ret = set_direct_map_invalid_noflush(page);
105
106	if (ret)
107	pr_warn_once("Failed to remap page\n");
108
109	flush_tlb_kernel_range(addr, addr + PAGE_SIZE);
110	} else {
111	debug_pagealloc_unmap_pages(page, 1);
112	}
113	}
114
115	static int swsusp_page_is_free(struct page *);
116	static void swsusp_set_page_forbidden(struct page *);
117	static void swsusp_unset_page_forbidden(struct page *);
118
119	/*
120	* Number of bytes to reserve for memory allocations made by device drivers
121	* from their ->freeze() and ->freeze_noirq() callbacks so that they don't
122	* cause image creation to fail (tunable via /sys/power/reserved_size).
123	*/
124	unsigned long reserved_size;
125
126	void __init hibernate_reserved_size_init(void)
127	{
128	reserved_size = SPARE_PAGES * PAGE_SIZE;
129	}
130
131	/*
132	* Preferred image size in bytes (tunable via /sys/power/image_size).
133	* When it is set to N, swsusp will do its best to ensure the image
134	* size will not exceed N bytes, but if that is impossible, it will
135	* try to create the smallest image possible.
136	*/
137	unsigned long image_size;
138
139	void __init hibernate_image_size_init(void)
140	{
141	image_size = ((totalram_pages() * 2) / 5) * PAGE_SIZE;
142	}
143
144	/*
145	* List of PBEs needed for restoring the pages that were allocated before
146	* the suspend and included in the suspend image, but have also been
147	* allocated by the "resume" kernel, so their contents cannot be written
148	* directly to their "original" page frames.
149	*/
150	struct pbe *restore_pblist;
151
152	/* struct linked_page is used to build chains of pages */
153
154	#define LINKED_PAGE_DATA_SIZE (PAGE_SIZE - sizeof(void *))
155
156	struct linked_page {
157	struct linked_page *next;
158	char data[LINKED_PAGE_DATA_SIZE];
159	} __packed;
160
161	/*
162	* List of "safe" pages (ie. pages that were not used by the image kernel
163	* before hibernation) that may be used as temporary storage for image kernel
164	* memory contents.
165	*/
166	static struct linked_page *safe_pages_list;
167
168	/* Pointer to an auxiliary buffer (1 page) */
169	static void *buffer;
170
171	#define PG_ANY 0
172	#define PG_SAFE 1
173	#define PG_UNSAFE_CLEAR 1
174	#define PG_UNSAFE_KEEP 0
175
176	static unsigned int allocated_unsafe_pages;
177
178	/**
179	* get_image_page - Allocate a page for a hibernation image.
180	* @gfp_mask: GFP mask for the allocation.
181	* @safe_needed: Get pages that were not used before hibernation (restore only)
182	*
183	* During image restoration, for storing the PBE list and the image data, we can
184	* only use memory pages that do not conflict with the pages used before
185	* hibernation. The "unsafe" pages have PageNosaveFree set and we count them
186	* using allocated_unsafe_pages.
187	*
188	* Each allocated image page is marked as PageNosave and PageNosaveFree so that
189	* swsusp_free() can release it.
190	*/
191	static void *get_image_page(gfp_t gfp_mask, int safe_needed)
192	{
193	void *res;
194
195	res = (void *)get_zeroed_page(gfp_mask);
196	if (safe_needed)
197	while (res && swsusp_page_is_free(virt_to_page(res))) {
198	/* The page is unsafe, mark it for swsusp_free() */
199	swsusp_set_page_forbidden(virt_to_page(res));
200	allocated_unsafe_pages++;
201	res = (void *)get_zeroed_page(gfp_mask);
202	}
203	if (res) {
204	swsusp_set_page_forbidden(virt_to_page(res));
205	swsusp_set_page_free(virt_to_page(res));
206	}
207	return res;
208	}
209
210	static void *__get_safe_page(gfp_t gfp_mask)
211	{
212	if (safe_pages_list) {
213	void *ret = safe_pages_list;
214
215	safe_pages_list = safe_pages_list->next;
216	memset(ret, 0, PAGE_SIZE);
217	return ret;
218	}
219	return get_image_page(gfp_mask, PG_SAFE);
220	}
221
222	unsigned long get_safe_page(gfp_t gfp_mask)
223	{
224	return (unsigned long)__get_safe_page(gfp_mask);
225	}
226
227	static struct page *alloc_image_page(gfp_t gfp_mask)
228	{
229	struct page *page;
230
231	page = alloc_page(gfp_mask);
232	if (page) {
233	swsusp_set_page_forbidden(page);
234	swsusp_set_page_free(page);
235	}
236	return page;
237	}
238
239	static void recycle_safe_page(void *page_address)
240	{
241	struct linked_page *lp = page_address;
242
243	lp->next = safe_pages_list;
244	safe_pages_list = lp;
245	}
246
247	/**
248	* free_image_page - Free a page allocated for hibernation image.
249	* @addr: Address of the page to free.
250	* @clear_nosave_free: If set, clear the PageNosaveFree bit for the page.
251	*
252	* The page to free should have been allocated by get_image_page() (page flags
253	* set by it are affected).
254	*/
255	static inline void free_image_page(void *addr, int clear_nosave_free)
256	{
257	struct page *page;
258
259	BUG_ON(!virt_addr_valid(addr));
260
261	page = virt_to_page(addr);
262
263	swsusp_unset_page_forbidden(page);
264	if (clear_nosave_free)
265	swsusp_unset_page_free(page);
266
267	__free_page(page);
268	}
269
270	static inline void free_list_of_pages(struct linked_page *list,
271	int clear_page_nosave)
272	{
273	while (list) {
274	struct linked_page *lp = list->next;
275
276	free_image_page(list, clear_page_nosave);
277	list = lp;
278	}
279	}
280
281	/*
282	* struct chain_allocator is used for allocating small objects out of
283	* a linked list of pages called 'the chain'.
284	*
285	* The chain grows each time when there is no room for a new object in
286	* the current page. The allocated objects cannot be freed individually.
287	* It is only possible to free them all at once, by freeing the entire
288	* chain.
289	*
290	* NOTE: The chain allocator may be inefficient if the allocated objects
291	* are not much smaller than PAGE_SIZE.
292	*/
293	struct chain_allocator {
294	struct linked_page chain; / the chain */
295	unsigned int used_space; /* total size of objects allocated out
296	of the current page */
297	gfp_t gfp_mask; /* mask for allocating pages */
298	int safe_needed; /* if set, only "safe" pages are allocated */
299	};
300
301	static void chain_init(struct chain_allocator *ca, gfp_t gfp_mask,
302	int safe_needed)
303	{
304	ca->chain = NULL;
305	ca->used_space = LINKED_PAGE_DATA_SIZE;
306	ca->gfp_mask = gfp_mask;
307	ca->safe_needed = safe_needed;
308	}
309
310	static void chain_alloc(struct chain_allocator ca, unsigned int size)
311	{
312	void *ret;
313
314	if (LINKED_PAGE_DATA_SIZE - ca->used_space < size) {
315	struct linked_page *lp;
316
317	lp = ca->safe_needed ? __get_safe_page(ca->gfp_mask) :
318	get_image_page(ca->gfp_mask, PG_ANY);
319	if (!lp)
320	return NULL;
321
322	lp->next = ca->chain;
323	ca->chain = lp;
324	ca->used_space = 0;
325	}
326	ret = ca->chain->data + ca->used_space;
327	ca->used_space += size;
328	return ret;
329	}
330
331	/*
332	* Data types related to memory bitmaps.
333	*
334	* Memory bitmap is a structure consisting of many linked lists of
335	* objects. The main list's elements are of type struct zone_bitmap
336	* and each of them corresponds to one zone. For each zone bitmap
337	* object there is a list of objects of type struct bm_block that
338	* represent each blocks of bitmap in which information is stored.
339	*
340	* struct memory_bitmap contains a pointer to the main list of zone
341	* bitmap objects, a struct bm_position used for browsing the bitmap,
342	* and a pointer to the list of pages used for allocating all of the
343	* zone bitmap objects and bitmap block objects.
344	*
345	* NOTE: It has to be possible to lay out the bitmap in memory
346	* using only allocations of order 0. Additionally, the bitmap is
347	* designed to work with arbitrary number of zones (this is over the
348	* top for now, but let's avoid making unnecessary assumptions ;-).
349	*
350	* struct zone_bitmap contains a pointer to a list of bitmap block
351	* objects and a pointer to the bitmap block object that has been
352	* most recently used for setting bits. Additionally, it contains the
353	* PFNs that correspond to the start and end of the represented zone.
354	*
355	* struct bm_block contains a pointer to the memory page in which
356	* information is stored (in the form of a block of bitmap)
357	* It also contains the pfns that correspond to the start and end of
358	* the represented memory area.
359	*
360	* The memory bitmap is organized as a radix tree to guarantee fast random
361	* access to the bits. There is one radix tree for each zone (as returned
362	* from create_mem_extents).
363	*
364	* One radix tree is represented by one struct mem_zone_bm_rtree. There are
365	* two linked lists for the nodes of the tree, one for the inner nodes and
366	* one for the leaf nodes. The linked leaf nodes are used for fast linear
367	* access of the memory bitmap.
368	*
369	* The struct rtree_node represents one node of the radix tree.
370	*/
371
372	#define BM_END_OF_MAP (~0UL)
373
374	#define BM_BITS_PER_BLOCK (PAGE_SIZE * BITS_PER_BYTE)
375	#define BM_BLOCK_SHIFT (PAGE_SHIFT + 3)
376	#define BM_BLOCK_MASK ((1UL << BM_BLOCK_SHIFT) - 1)
377
378	/*
379	* struct rtree_node is a wrapper struct to link the nodes
380	* of the rtree together for easy linear iteration over
381	* bits and easy freeing
382	*/
383	struct rtree_node {
384	struct list_head list;
385	unsigned long *data;
386	};
387
388	/*
389	* struct mem_zone_bm_rtree represents a bitmap used for one
390	* populated memory zone.
391	*/
392	struct mem_zone_bm_rtree {
393	struct list_head list; /* Link Zones together */
394	struct list_head nodes; /* Radix Tree inner nodes */
395	struct list_head leaves; /* Radix Tree leaves */
396	unsigned long start_pfn; /* Zone start page frame */
397	unsigned long end_pfn; /* Zone end page frame + 1 */
398	struct rtree_node rtree; / Radix Tree Root */
399	int levels; /* Number of Radix Tree Levels */
400	unsigned int blocks; /* Number of Bitmap Blocks */
401	};
402
403	/* struct bm_position is used for browsing memory bitmaps */
404
405	struct bm_position {
406	struct mem_zone_bm_rtree *zone;
407	struct rtree_node *node;
408	unsigned long node_pfn;
409	unsigned long cur_pfn;
410	int node_bit;
411	};
412
413	struct memory_bitmap {
414	struct list_head zones;
415	struct linked_page p_list; / list of pages used to store zone
416	bitmap objects and bitmap block
417	objects */
418	struct bm_position cur; /* most recently used bit position */
419	};
420
421	/* Functions that operate on memory bitmaps */
422
423	#define BM_ENTRIES_PER_LEVEL (PAGE_SIZE / sizeof(unsigned long))
424	#if BITS_PER_LONG == 32
425	#define BM_RTREE_LEVEL_SHIFT (PAGE_SHIFT - 2)
426	#else
427	#define BM_RTREE_LEVEL_SHIFT (PAGE_SHIFT - 3)
428	#endif
429	#define BM_RTREE_LEVEL_MASK ((1UL << BM_RTREE_LEVEL_SHIFT) - 1)
430
431	/**
432	* alloc_rtree_node - Allocate a new node and add it to the radix tree.
433	* @gfp_mask: GFP mask for the allocation.
434	* @safe_needed: Get pages not used before hibernation (restore only)
435	* @ca: Pointer to a linked list of pages ("a chain") to allocate from
436	* @list: Radix Tree node to add.
437	*
438	* This function is used to allocate inner nodes as well as the
439	* leave nodes of the radix tree. It also adds the node to the
440	* corresponding linked list passed in by the *list parameter.
441	*/
442	static struct rtree_node *alloc_rtree_node(gfp_t gfp_mask, int safe_needed,
443	struct chain_allocator *ca,
444	struct list_head *list)
445	{
446	struct rtree_node *node;
447
448	node = chain_alloc(ca, sizeof(struct rtree_node));
449	if (!node)
450	return NULL;
451
452	node->data = get_image_page(gfp_mask, safe_needed);
453	if (!node->data)
454	return NULL;
455
456	list_add_tail(&node->list, list);
457
458	return node;
459	}
460
461	/*
462	* add_rtree_block - Add a new leave node to the radix tree.
463	*
464	* The leave nodes need to be allocated in order to keep the leaves
465	* linked list in order. This is guaranteed by the zone->blocks
466	* counter.
467	*/
468	static int add_rtree_block(struct mem_zone_bm_rtree *zone, gfp_t gfp_mask,
469	int safe_needed, struct chain_allocator *ca)
470	{
471	struct rtree_node node, block, **dst;
472	unsigned int levels_needed, block_nr;
473	int i;
474
475	block_nr = zone->blocks;
476	levels_needed = 0;
477
478	/* How many levels do we need for this block nr? */
479	while (block_nr) {
480	levels_needed += 1;
481	block_nr >>= BM_RTREE_LEVEL_SHIFT;
482	}
483
484	/* Make sure the rtree has enough levels */
485	for (i = zone->levels; i < levels_needed; i++) {
486	node = alloc_rtree_node(gfp_mask, safe_needed, ca,
487	&zone->nodes);
488	if (!node)
489	return -ENOMEM;
490
491	node->data[0] = (unsigned long)zone->rtree;
492	zone->rtree = node;
493	zone->levels += 1;
494	}
495
496	/* Allocate new block */
497	block = alloc_rtree_node(gfp_mask, safe_needed, ca, &zone->leaves);
498	if (!block)
499	return -ENOMEM;
500
501	/* Now walk the rtree to insert the block */
502	node = zone->rtree;
503	dst = &zone->rtree;
504	block_nr = zone->blocks;
505	for (i = zone->levels; i > 0; i--) {
506	int index;
507
508	if (!node) {
509	node = alloc_rtree_node(gfp_mask, safe_needed, ca,
510	&zone->nodes);
511	if (!node)
512	return -ENOMEM;
513	*dst = node;
514	}
515
516	index = block_nr >> ((i - 1) * BM_RTREE_LEVEL_SHIFT);
517	index &= BM_RTREE_LEVEL_MASK;
518	dst = (struct rtree_node *)&((dst)->data[index]);
519	node = *dst;
520	}
521
522	zone->blocks += 1;
523	*dst = block;
524
525	return 0;
526	}
527
528	static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
529	int clear_nosave_free);
530
531	/*
532	* create_zone_bm_rtree - Create a radix tree for one zone.
533	*
534	* Allocated the mem_zone_bm_rtree structure and initializes it.
535	* This function also allocated and builds the radix tree for the
536	* zone.
537	*/
538	static struct mem_zone_bm_rtree *create_zone_bm_rtree(gfp_t gfp_mask,
539	int safe_needed,
540	struct chain_allocator *ca,
541	unsigned long start,
542	unsigned long end)
543	{
544	struct mem_zone_bm_rtree *zone;
545	unsigned int i, nr_blocks;
546	unsigned long pages;
547
548	pages = end - start;
549	zone = chain_alloc(ca, sizeof(struct mem_zone_bm_rtree));
550	if (!zone)
551	return NULL;
552
553	INIT_LIST_HEAD(&zone->nodes);
554	INIT_LIST_HEAD(&zone->leaves);
555	zone->start_pfn = start;
556	zone->end_pfn = end;
557	nr_blocks = DIV_ROUND_UP(pages, BM_BITS_PER_BLOCK);
558
559	for (i = 0; i < nr_blocks; i++) {
560	if (add_rtree_block(zone, gfp_mask, safe_needed, ca)) {
561	free_zone_bm_rtree(zone, PG_UNSAFE_CLEAR);
562	return NULL;
563	}
564	}
565
566	return zone;
567	}
568
569	/*
570	* free_zone_bm_rtree - Free the memory of the radix tree.
571	*
572	* Free all node pages of the radix tree. The mem_zone_bm_rtree
573	* structure itself is not freed here nor are the rtree_node
574	* structs.
575	*/
576	static void free_zone_bm_rtree(struct mem_zone_bm_rtree *zone,
577	int clear_nosave_free)
578	{
579	struct rtree_node *node;
580
581	list_for_each_entry(node, &zone->nodes, list)
582	free_image_page(node->data, clear_nosave_free);
583
584	list_for_each_entry(node, &zone->leaves, list)
585	free_image_page(node->data, clear_nosave_free);
586	}
587
588	static void memory_bm_position_reset(struct memory_bitmap *bm)
589	{
590	bm->cur.zone = list_entry(bm->zones.next, struct mem_zone_bm_rtree,
591	list);
592	bm->cur.node = list_entry(bm->cur.zone->leaves.next,
593	struct rtree_node, list);
594	bm->cur.node_pfn = 0;
595	bm->cur.cur_pfn = BM_END_OF_MAP;
596	bm->cur.node_bit = 0;
597	}
598
599	static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free);
600
601	struct mem_extent {
602	struct list_head hook;
603	unsigned long start;
604	unsigned long end;
605	};
606
607	/**
608	* free_mem_extents - Free a list of memory extents.
609	* @list: List of extents to free.
610	*/
611	static void free_mem_extents(struct list_head *list)
612	{
613	struct mem_extent ext, aux;
614
615	list_for_each_entry_safe(ext, aux, list, hook) {
616	list_del(&ext->hook);
617	kfree(ext);
618	}
619	}
620
621	/**
622	* create_mem_extents - Create a list of memory extents.
623	* @list: List to put the extents into.
624	* @gfp_mask: Mask to use for memory allocations.
625	*
626	* The extents represent contiguous ranges of PFNs.
627	*/
628	static int create_mem_extents(struct list_head *list, gfp_t gfp_mask)
629	{
630	struct zone *zone;
631
632	INIT_LIST_HEAD(list);
633
634	for_each_populated_zone(zone) {
635	unsigned long zone_start, zone_end;
636	struct mem_extent ext, cur, *aux;
637
638	zone_start = zone->zone_start_pfn;
639	zone_end = zone_end_pfn(zone);
640
641	list_for_each_entry(ext, list, hook)
642	if (zone_start <= ext->end)
643	break;
644
645	if (&ext->hook == list \|\| zone_end < ext->start) {
646	/* New extent is necessary */
647	struct mem_extent *new_ext;
648
649	new_ext = kzalloc_obj(struct mem_extent, gfp_mask);
650	if (!new_ext) {
651	free_mem_extents(list);
652	return -ENOMEM;
653	}
654	new_ext->start = zone_start;
655	new_ext->end = zone_end;
656	list_add_tail(&new_ext->hook, &ext->hook);
657	continue;
658	}
659
660	/* Merge this zone's range of PFNs with the existing one */
661	if (zone_start < ext->start)
662	ext->start = zone_start;
663	if (zone_end > ext->end)
664	ext->end = zone_end;
665
666	/* More merging may be possible */
667	cur = ext;
668	list_for_each_entry_safe_continue(cur, aux, list, hook) {
669	if (zone_end < cur->start)
670	break;
671	if (zone_end < cur->end)
672	ext->end = cur->end;
673	list_del(&cur->hook);
674	kfree(cur);
675	}
676	}
677
678	return 0;
679	}
680
681	/*
682	* memory_bm_create - Allocate memory for a memory bitmap.
683	*/
684	static int memory_bm_create(struct memory_bitmap *bm, gfp_t gfp_mask,
685	int safe_needed)
686	{
687	struct chain_allocator ca;
688	struct list_head mem_extents;
689	struct mem_extent *ext;
690	int error;
691
692	chain_init(&ca, gfp_mask, safe_needed);
693	INIT_LIST_HEAD(&bm->zones);
694
695	error = create_mem_extents(&mem_extents, gfp_mask);
696	if (error)
697	return error;
698
699	list_for_each_entry(ext, &mem_extents, hook) {
700	struct mem_zone_bm_rtree *zone;
701
702	zone = create_zone_bm_rtree(gfp_mask, safe_needed, &ca,
703	ext->start, ext->end);
704	if (!zone) {
705	error = -ENOMEM;
706	goto Error;
707	}
708	list_add_tail(&zone->list, &bm->zones);
709	}
710
711	bm->p_list = ca.chain;
712	memory_bm_position_reset(bm);
713	Exit:
714	free_mem_extents(&mem_extents);
715	return error;
716
717	Error:
718	bm->p_list = ca.chain;
719	memory_bm_free(bm, PG_UNSAFE_CLEAR);
720	goto Exit;
721	}
722
723	/*
724	* memory_bm_free - Free memory occupied by the memory bitmap.
725	* @bm: Memory bitmap.
726	*/
727	static void memory_bm_free(struct memory_bitmap *bm, int clear_nosave_free)
728	{
729	struct mem_zone_bm_rtree *zone;
730
731	list_for_each_entry(zone, &bm->zones, list)
732	free_zone_bm_rtree(zone, clear_nosave_free);
733
734	free_list_of_pages(bm->p_list, clear_nosave_free);
735
736	INIT_LIST_HEAD(&bm->zones);
737	}
738
739	/*
740	* memory_bm_find_bit - Find the bit for a given PFN in a memory bitmap.
741	*
742	* Find the bit in memory bitmap @bm that corresponds to the given PFN.
743	* The cur.zone, cur.block and cur.node_pfn members of @bm are updated.
744	*
745	* Walk the radix tree to find the page containing the bit that represents @pfn
746	* and return the position of the bit in @addr and @bit_nr.
747	*/
748	static int memory_bm_find_bit(struct memory_bitmap *bm, unsigned long pfn,
749	void *addr, unsigned int bit_nr)
750	{
751	struct mem_zone_bm_rtree curr, zone;
752	struct rtree_node *node;
753	int i, block_nr;
754
755	zone = bm->cur.zone;
756
757	if (pfn >= zone->start_pfn && pfn < zone->end_pfn)
758	goto zone_found;
759
760	zone = NULL;
761
762	/* Find the right zone */
763	list_for_each_entry(curr, &bm->zones, list) {
764	if (pfn >= curr->start_pfn && pfn < curr->end_pfn) {
765	zone = curr;
766	break;
767	}
768	}
769
770	if (!zone)
771	return -EFAULT;
772
773	zone_found:
774	/*
775	* We have found the zone. Now walk the radix tree to find the leaf node
776	* for our PFN.
777	*/
778
779	/*
780	* If the zone we wish to scan is the current zone and the
781	* pfn falls into the current node then we do not need to walk
782	* the tree.
783	*/
784	node = bm->cur.node;
785	if (zone == bm->cur.zone &&
786	((pfn - zone->start_pfn) & ~BM_BLOCK_MASK) == bm->cur.node_pfn)
787	goto node_found;
788
789	node = zone->rtree;
790	block_nr = (pfn - zone->start_pfn) >> BM_BLOCK_SHIFT;
791
792	for (i = zone->levels; i > 0; i--) {
793	int index;
794
795	index = block_nr >> ((i - 1) * BM_RTREE_LEVEL_SHIFT);
796	index &= BM_RTREE_LEVEL_MASK;
797	BUG_ON(node->data[index] == 0);
798	node = (struct rtree_node *)node->data[index];
799	}
800
801	node_found:
802	/* Update last position */
803	bm->cur.zone = zone;
804	bm->cur.node = node;
805	bm->cur.node_pfn = (pfn - zone->start_pfn) & ~BM_BLOCK_MASK;
806	bm->cur.cur_pfn = pfn;
807
808	/* Set return values */
809	*addr = node->data;
810	*bit_nr = (pfn - zone->start_pfn) & BM_BLOCK_MASK;
811
812	return 0;
813	}
814
815	static void memory_bm_set_bit(struct memory_bitmap *bm, unsigned long pfn)
816	{
817	void *addr;
818	unsigned int bit;
819	int error;
820
821	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
822	BUG_ON(error);
823	set_bit(bit, addr);
824	}
825
826	static int mem_bm_set_bit_check(struct memory_bitmap *bm, unsigned long pfn)
827	{
828	void *addr;
829	unsigned int bit;
830	int error;
831
832	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
833	if (!error)
834	set_bit(bit, addr);
835
836	return error;
837	}
838
839	static void memory_bm_clear_bit(struct memory_bitmap *bm, unsigned long pfn)
840	{
841	void *addr;
842	unsigned int bit;
843	int error;
844
845	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
846	BUG_ON(error);
847	clear_bit(bit, addr);
848	}
849
850	static void memory_bm_clear_current(struct memory_bitmap *bm)
851	{
852	int bit;
853
854	bit = max(bm->cur.node_bit - 1, 0);
855	clear_bit(bit, bm->cur.node->data);
856	}
857
858	static unsigned long memory_bm_get_current(struct memory_bitmap *bm)
859	{
860	return bm->cur.cur_pfn;
861	}
862
863	static int memory_bm_test_bit(struct memory_bitmap *bm, unsigned long pfn)
864	{
865	void *addr;
866	unsigned int bit;
867	int error;
868
869	error = memory_bm_find_bit(bm, pfn, &addr, &bit);
870	BUG_ON(error);
871	return test_bit(bit, addr);
872	}
873
874	static bool memory_bm_pfn_present(struct memory_bitmap *bm, unsigned long pfn)
875	{
876	void *addr;
877	unsigned int bit;
878
879	return !memory_bm_find_bit(bm, pfn, &addr, &bit);
880	}
881
882	/*
883	* rtree_next_node - Jump to the next leaf node.
884	*
885	* Set the position to the beginning of the next node in the
886	* memory bitmap. This is either the next node in the current
887	* zone's radix tree or the first node in the radix tree of the
888	* next zone.
889	*
890	* Return true if there is a next node, false otherwise.
891	*/
892	static bool rtree_next_node(struct memory_bitmap *bm)
893	{
894	if (!list_is_last(&bm->cur.node->list, &bm->cur.zone->leaves)) {
895	bm->cur.node = list_entry(bm->cur.node->list.next,
896	struct rtree_node, list);
897	bm->cur.node_pfn += BM_BITS_PER_BLOCK;
898	bm->cur.node_bit = 0;
899	touch_softlockup_watchdog();
900	return true;
901	}
902
903	/* No more nodes, goto next zone */
904	if (!list_is_last(&bm->cur.zone->list, &bm->zones)) {
905	bm->cur.zone = list_entry(bm->cur.zone->list.next,
906	struct mem_zone_bm_rtree, list);
907	bm->cur.node = list_entry(bm->cur.zone->leaves.next,
908	struct rtree_node, list);
909	bm->cur.node_pfn = 0;
910	bm->cur.node_bit = 0;
911	return true;
912	}
913
914	/* No more zones */
915	return false;
916	}
917
918	/**
919	* memory_bm_next_pfn - Find the next set bit in a memory bitmap.
920	* @bm: Memory bitmap.
921	*
922	* Starting from the last returned position this function searches for the next
923	* set bit in @bm and returns the PFN represented by it. If no more bits are
924	* set, BM_END_OF_MAP is returned.
925	*
926	* It is required to run memory_bm_position_reset() before the first call to
927	* this function for the given memory bitmap.
928	*/
929	static unsigned long memory_bm_next_pfn(struct memory_bitmap *bm)
930	{
931	unsigned long bits, pfn, pages;
932	int bit;
933
934	do {
935	pages = bm->cur.zone->end_pfn - bm->cur.zone->start_pfn;
936	bits = min(pages - bm->cur.node_pfn, BM_BITS_PER_BLOCK);
937	bit = find_next_bit(bm->cur.node->data, bits,
938	bm->cur.node_bit);
939	if (bit < bits) {
940	pfn = bm->cur.zone->start_pfn + bm->cur.node_pfn + bit;
941	bm->cur.node_bit = bit + 1;
942	bm->cur.cur_pfn = pfn;
943	return pfn;
944	}
945	} while (rtree_next_node(bm));
946
947	bm->cur.cur_pfn = BM_END_OF_MAP;
948	return BM_END_OF_MAP;
949	}
950
951	/*
952	* This structure represents a range of page frames the contents of which
953	* should not be saved during hibernation.
954	*/
955	struct nosave_region {
956	struct list_head list;
957	unsigned long start_pfn;
958	unsigned long end_pfn;
959	};
960
961	static LIST_HEAD(nosave_regions);
962
963	static void recycle_zone_bm_rtree(struct mem_zone_bm_rtree *zone)
964	{
965	struct rtree_node *node;
966
967	list_for_each_entry(node, &zone->nodes, list)
968	recycle_safe_page(node->data);
969
970	list_for_each_entry(node, &zone->leaves, list)
971	recycle_safe_page(node->data);
972	}
973
974	static void memory_bm_recycle(struct memory_bitmap *bm)
975	{
976	struct mem_zone_bm_rtree *zone;
977	struct linked_page *p_list;
978
979	list_for_each_entry(zone, &bm->zones, list)
980	recycle_zone_bm_rtree(zone);
981
982	p_list = bm->p_list;
983	while (p_list) {
984	struct linked_page *lp = p_list;
985
986	p_list = lp->next;
987	recycle_safe_page(lp);
988	}
989	}
990
991	/*
992	* register_nosave_region - Register a region of unsaveable memory.
993	*
994	* Register a range of page frames the contents of which should not be saved
995	* during hibernation (to be used in the early initialization code).
996	*/
997	void __init register_nosave_region(unsigned long start_pfn, unsigned long end_pfn)
998	{
999	struct nosave_region *region;
1000
1001	if (start_pfn >= end_pfn)
1002	return;
1003
1004	if (!list_empty(&nosave_regions)) {
1005	/* Try to extend the previous region (they should be sorted) */
1006	region = list_entry(nosave_regions.prev,
1007	struct nosave_region, list);
1008	if (region->end_pfn == start_pfn) {
1009	region->end_pfn = end_pfn;
1010	goto Report;
1011	}
1012	}
1013	/* This allocation cannot fail */
1014	region = memblock_alloc_or_panic(sizeof(struct nosave_region),
1015	SMP_CACHE_BYTES);
1016	region->start_pfn = start_pfn;
1017	region->end_pfn = end_pfn;
1018	list_add_tail(&region->list, &nosave_regions);
1019	Report:
1020	pr_info("Registered nosave memory: [mem %#010llx-%#010llx]\n",
1021	(unsigned long long) start_pfn << PAGE_SHIFT,
1022	((unsigned long long) end_pfn << PAGE_SHIFT) - 1);
1023	}
1024
1025	/*
1026	* Set bits in this map correspond to the page frames the contents of which
1027	* should not be saved during the suspend.
1028	*/
1029	static struct memory_bitmap *forbidden_pages_map;
1030
1031	/* Set bits in this map correspond to free page frames. */
1032	static struct memory_bitmap *free_pages_map;
1033
1034	/*
1035	* Each page frame allocated for creating the image is marked by setting the
1036	* corresponding bits in forbidden_pages_map and free_pages_map simultaneously
1037	*/
1038
1039	void swsusp_set_page_free(struct page *page)
1040	{
1041	if (free_pages_map)
1042	memory_bm_set_bit(free_pages_map, page_to_pfn(page));
1043	}
1044
1045	static int swsusp_page_is_free(struct page *page)
1046	{
1047	return free_pages_map ?
1048	memory_bm_test_bit(free_pages_map, page_to_pfn(page)) : 0;
1049	}
1050
1051	void swsusp_unset_page_free(struct page *page)
1052	{
1053	if (free_pages_map)
1054	memory_bm_clear_bit(free_pages_map, page_to_pfn(page));
1055	}
1056
1057	static void swsusp_set_page_forbidden(struct page *page)
1058	{
1059	if (forbidden_pages_map)
1060	memory_bm_set_bit(forbidden_pages_map, page_to_pfn(page));
1061	}
1062
1063	int swsusp_page_is_forbidden(struct page *page)
1064	{
1065	return forbidden_pages_map ?
1066	memory_bm_test_bit(forbidden_pages_map, page_to_pfn(page)) : 0;
1067	}
1068
1069	static void swsusp_unset_page_forbidden(struct page *page)
1070	{
1071	if (forbidden_pages_map)
1072	memory_bm_clear_bit(forbidden_pages_map, page_to_pfn(page));
1073	}
1074
1075	/**
1076	* mark_nosave_pages - Mark pages that should not be saved.
1077	* @bm: Memory bitmap.
1078	*
1079	* Set the bits in @bm that correspond to the page frames the contents of which
1080	* should not be saved.
1081	*/
1082	static void mark_nosave_pages(struct memory_bitmap *bm)
1083	{
1084	struct nosave_region *region;
1085
1086	if (list_empty(&nosave_regions))
1087	return;
1088
1089	list_for_each_entry(region, &nosave_regions, list) {
1090	unsigned long pfn;
1091
1092	pr_debug("Marking nosave pages: [mem %#010llx-%#010llx]\n",
1093	(unsigned long long) region->start_pfn << PAGE_SHIFT,
1094	((unsigned long long) region->end_pfn << PAGE_SHIFT)
1095	- 1);
1096
1097	for_each_valid_pfn(pfn, region->start_pfn, region->end_pfn) {
1098	/*
1099	* It is safe to ignore the result of
1100	* mem_bm_set_bit_check() here, since we won't
1101	* touch the PFNs for which the error is
1102	* returned anyway.
1103	*/
1104	mem_bm_set_bit_check(bm, pfn);
1105	}
1106	}
1107	}
1108
1109	/**
1110	* create_basic_memory_bitmaps - Create bitmaps to hold basic page information.
1111	*
1112	* Create bitmaps needed for marking page frames that should not be saved and
1113	* free page frames. The forbidden_pages_map and free_pages_map pointers are
1114	* only modified if everything goes well, because we don't want the bits to be
1115	* touched before both bitmaps are set up.
1116	*/
1117	int create_basic_memory_bitmaps(void)
1118	{
1119	struct memory_bitmap bm1, bm2;
1120	int error;
1121
1122	if (forbidden_pages_map && free_pages_map)
1123	return 0;
1124	else
1125	BUG_ON(forbidden_pages_map \|\| free_pages_map);
1126
1127	bm1 = kzalloc_obj(struct memory_bitmap);
1128	if (!bm1)
1129	return -ENOMEM;
1130
1131	error = memory_bm_create(bm1, GFP_KERNEL, PG_ANY);
1132	if (error)
1133	goto Free_first_object;
1134
1135	bm2 = kzalloc_obj(struct memory_bitmap);
1136	if (!bm2)
1137	goto Free_first_bitmap;
1138
1139	error = memory_bm_create(bm2, GFP_KERNEL, PG_ANY);
1140	if (error)
1141	goto Free_second_object;
1142
1143	forbidden_pages_map = bm1;
1144	free_pages_map = bm2;
1145	mark_nosave_pages(forbidden_pages_map);
1146
1147	pr_debug("Basic memory bitmaps created\n");
1148
1149	return 0;
1150
1151	Free_second_object:
1152	kfree(bm2);
1153	Free_first_bitmap:
1154	memory_bm_free(bm1, PG_UNSAFE_CLEAR);
1155	Free_first_object:
1156	kfree(bm1);
1157	return -ENOMEM;
1158	}
1159
1160	/**
1161	* free_basic_memory_bitmaps - Free memory bitmaps holding basic information.
1162	*
1163	* Free memory bitmaps allocated by create_basic_memory_bitmaps(). The
1164	* auxiliary pointers are necessary so that the bitmaps themselves are not
1165	* referred to while they are being freed.
1166	*/
1167	void free_basic_memory_bitmaps(void)
1168	{
1169	struct memory_bitmap bm1, bm2;
1170
1171	if (WARN_ON(!(forbidden_pages_map && free_pages_map)))
1172	return;
1173
1174	bm1 = forbidden_pages_map;
1175	bm2 = free_pages_map;
1176	forbidden_pages_map = NULL;
1177	free_pages_map = NULL;
1178	memory_bm_free(bm1, PG_UNSAFE_CLEAR);
1179	kfree(bm1);
1180	memory_bm_free(bm2, PG_UNSAFE_CLEAR);
1181	kfree(bm2);
1182
1183	pr_debug("Basic memory bitmaps freed\n");
1184	}
1185
1186	static void clear_or_poison_free_page(struct page *page)
1187	{
1188	if (page_poisoning_enabled_static())
1189	__kernel_poison_pages(page, 1);
1190	else if (want_init_on_free())
1191	clear_highpage(page);
1192	}
1193
1194	void clear_or_poison_free_pages(void)
1195	{
1196	struct memory_bitmap *bm = free_pages_map;
1197	unsigned long pfn;
1198
1199	if (WARN_ON(!(free_pages_map)))
1200	return;
1201
1202	if (page_poisoning_enabled() \|\| want_init_on_free()) {
1203	memory_bm_position_reset(bm);
1204	pfn = memory_bm_next_pfn(bm);
1205	while (pfn != BM_END_OF_MAP) {
1206	if (pfn_valid(pfn))
1207	clear_or_poison_free_page(pfn_to_page(pfn));
1208
1209	pfn = memory_bm_next_pfn(bm);
1210	}
1211	memory_bm_position_reset(bm);
1212	pr_info("free pages cleared after restore\n");
1213	}
1214	}
1215
1216	/**
1217	* snapshot_additional_pages - Estimate the number of extra pages needed.
1218	* @zone: Memory zone to carry out the computation for.
1219	*
1220	* Estimate the number of additional pages needed for setting up a hibernation
1221	* image data structures for @zone (usually, the returned value is greater than
1222	* the exact number).
1223	*/
1224	unsigned int snapshot_additional_pages(struct zone *zone)
1225	{
1226	unsigned int rtree, nodes;
1227
1228	rtree = nodes = DIV_ROUND_UP(zone->spanned_pages, BM_BITS_PER_BLOCK);
1229	rtree += DIV_ROUND_UP(rtree * sizeof(struct rtree_node),
1230	LINKED_PAGE_DATA_SIZE);
1231	while (nodes > 1) {
1232	nodes = DIV_ROUND_UP(nodes, BM_ENTRIES_PER_LEVEL);
1233	rtree += nodes;
1234	}
1235
1236	return 2 * rtree;
1237	}
1238
1239	/*
1240	* Touch the watchdog for every WD_PAGE_COUNT pages.
1241	*/
1242	#define WD_PAGE_COUNT (128*1024)
1243
1244	static void mark_free_pages(struct zone *zone)
1245	{
1246	unsigned long pfn, max_zone_pfn, page_count = WD_PAGE_COUNT;
1247	struct list_head *free_list;
1248	unsigned long flags;
1249	unsigned int order;
1250	struct page *page;
1251
1252	if (zone_is_empty(zone))
1253	return;
1254
1255	spin_lock_irqsave(&zone->lock, flags);
1256
1257	max_zone_pfn = zone_end_pfn(zone);
1258	for_each_valid_pfn(pfn, zone->zone_start_pfn, max_zone_pfn) {
1259	page = pfn_to_page(pfn);
1260
1261	if (!--page_count) {
1262	touch_nmi_watchdog();
1263	page_count = WD_PAGE_COUNT;
1264	}
1265
1266	if (page_zone(page) != zone)
1267	continue;
1268
1269	if (!swsusp_page_is_forbidden(page))
1270	swsusp_unset_page_free(page);
1271	}
1272
1273	for_each_free_list(free_list, zone, order) {
1274	list_for_each_entry(page, free_list, buddy_list) {
1275	unsigned long i;
1276
1277	pfn = page_to_pfn(page);
1278	for (i = 0; i < (1UL << order); i++) {
1279	if (!--page_count) {
1280	touch_nmi_watchdog();
1281	page_count = WD_PAGE_COUNT;
1282	}
1283	swsusp_set_page_free(pfn_to_page(pfn + i));
1284	}
1285	}
1286	}
1287	spin_unlock_irqrestore(&zone->lock, flags);
1288	}
1289
1290	#ifdef CONFIG_HIGHMEM
1291	/**
1292	* count_free_highmem_pages - Compute the total number of free highmem pages.
1293	*
1294	* The returned number is system-wide.
1295	*/
1296	static unsigned int count_free_highmem_pages(void)
1297	{
1298	struct zone *zone;
1299	unsigned int cnt = 0;
1300
1301	for_each_populated_zone(zone)
1302	if (is_highmem(zone))
1303	cnt += zone_page_state(zone, NR_FREE_PAGES);
1304
1305	return cnt;
1306	}
1307
1308	/*
1309	* saveable_highmem_page - Check if a highmem page is saveable.
1310	*
1311	* Determine whether a highmem page should be included in a hibernation image.
1312	*
1313	* We should save the page if it isn't Nosave or NosaveFree, or Reserved,
1314	* and it isn't part of a free chunk of pages.
1315	*/
1316	static struct page saveable_highmem_page(struct zone zone, unsigned long pfn)
1317	{
1318	struct page *page;
1319
1320	if (!pfn_valid(pfn))
1321	return NULL;
1322
1323	page = pfn_to_online_page(pfn);
1324	if (!page \|\| page_zone(page) != zone)
1325	return NULL;
1326
1327	BUG_ON(!PageHighMem(page));
1328
1329	if (swsusp_page_is_forbidden(page) \|\| swsusp_page_is_free(page))
1330	return NULL;
1331
1332	if (PageReserved(page) \|\| PageOffline(page))
1333	return NULL;
1334
1335	if (page_is_guard(page))
1336	return NULL;
1337
1338	return page;
1339	}
1340
1341	/**
1342	* count_highmem_pages - Compute the total number of saveable highmem pages.
1343	*/
1344	static unsigned int count_highmem_pages(void)
1345	{
1346	struct zone *zone;
1347	unsigned int n = 0;
1348
1349	for_each_populated_zone(zone) {
1350	unsigned long pfn, max_zone_pfn;
1351
1352	if (!is_highmem(zone))
1353	continue;
1354
1355	mark_free_pages(zone);
1356	max_zone_pfn = zone_end_pfn(zone);
1357	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1358	if (saveable_highmem_page(zone, pfn))
1359	n++;
1360	}
1361	return n;
1362	}
1363	#endif /* CONFIG_HIGHMEM */
1364
1365	/*
1366	* saveable_page - Check if the given page is saveable.
1367	*
1368	* Determine whether a non-highmem page should be included in a hibernation
1369	* image.
1370	*
1371	* We should save the page if it isn't Nosave, and is not in the range
1372	* of pages statically defined as 'unsaveable', and it isn't part of
1373	* a free chunk of pages.
1374	*/
1375	static struct page saveable_page(struct zone zone, unsigned long pfn)
1376	{
1377	struct page *page;
1378
1379	if (!pfn_valid(pfn))
1380	return NULL;
1381
1382	page = pfn_to_online_page(pfn);
1383	if (!page \|\| page_zone(page) != zone)
1384	return NULL;
1385
1386	BUG_ON(PageHighMem(page));
1387
1388	if (swsusp_page_is_forbidden(page) \|\| swsusp_page_is_free(page))
1389	return NULL;
1390
1391	if (PageOffline(page))
1392	return NULL;
1393
1394	if (PageReserved(page)
1395	&& (!kernel_page_present(page) \|\| pfn_is_nosave(pfn)))
1396	return NULL;
1397
1398	if (page_is_guard(page))
1399	return NULL;
1400
1401	return page;
1402	}
1403
1404	/**
1405	* count_data_pages - Compute the total number of saveable non-highmem pages.
1406	*/
1407	static unsigned int count_data_pages(void)
1408	{
1409	struct zone *zone;
1410	unsigned long pfn, max_zone_pfn;
1411	unsigned int n = 0;
1412
1413	for_each_populated_zone(zone) {
1414	if (is_highmem(zone))
1415	continue;
1416
1417	mark_free_pages(zone);
1418	max_zone_pfn = zone_end_pfn(zone);
1419	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1420	if (saveable_page(zone, pfn))
1421	n++;
1422	}
1423	return n;
1424	}
1425
1426	/*
1427	* This is needed, because copy_page and memcpy are not usable for copying
1428	* task structs. Returns true if the page was filled with only zeros,
1429	* otherwise false.
1430	*/
1431	static inline bool do_copy_page(long dst, long src)
1432	{
1433	long z = 0;
1434	int n;
1435
1436	for (n = PAGE_SIZE / sizeof(long); n; n--) {
1437	z \|= *src;
1438	dst++ = src++;
1439	}
1440	return !z;
1441	}
1442
1443	/*
1444	* safe_copy_page - Copy a page in a safe way.
1445	*
1446	* Check if the page we are going to copy is marked as present in the kernel
1447	* page tables. This always is the case if CONFIG_DEBUG_PAGEALLOC or
1448	* CONFIG_ARCH_HAS_SET_DIRECT_MAP is not set. In that case kernel_page_present()
1449	* always returns 'true'. Returns true if the page was entirely composed of
1450	* zeros, otherwise it will return false.
1451	*/
1452	static bool safe_copy_page(void dst, struct page s_page)
1453	{
1454	bool zeros_only;
1455
1456	if (kernel_page_present(s_page)) {
1457	zeros_only = do_copy_page(dst, page_address(s_page));
1458	} else {
1459	hibernate_map_page(s_page);
1460	zeros_only = do_copy_page(dst, page_address(s_page));
1461	hibernate_unmap_page(s_page);
1462	}
1463	return zeros_only;
1464	}
1465
1466	#ifdef CONFIG_HIGHMEM
1467	static inline struct page page_is_saveable(struct zone zone, unsigned long pfn)
1468	{
1469	return is_highmem(zone) ?
1470	saveable_highmem_page(zone, pfn) : saveable_page(zone, pfn);
1471	}
1472
1473	static bool copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
1474	{
1475	struct page s_page, d_page;
1476	void src, dst;
1477	bool zeros_only;
1478
1479	s_page = pfn_to_page(src_pfn);
1480	d_page = pfn_to_page(dst_pfn);
1481	if (PageHighMem(s_page)) {
1482	src = kmap_local_page(s_page);
1483	dst = kmap_local_page(d_page);
1484	zeros_only = do_copy_page(dst, src);
1485	kunmap_local(dst);
1486	kunmap_local(src);
1487	} else {
1488	if (PageHighMem(d_page)) {
1489	/*
1490	* The page pointed to by src may contain some kernel
1491	* data modified by kmap_atomic()
1492	*/
1493	zeros_only = safe_copy_page(buffer, s_page);
1494	dst = kmap_local_page(d_page);
1495	copy_page(dst, buffer);
1496	kunmap_local(dst);
1497	} else {
1498	zeros_only = safe_copy_page(page_address(d_page), s_page);
1499	}
1500	}
1501	return zeros_only;
1502	}
1503	#else
1504	#define page_is_saveable(zone, pfn) saveable_page(zone, pfn)
1505
1506	static inline int copy_data_page(unsigned long dst_pfn, unsigned long src_pfn)
1507	{
1508	return safe_copy_page(page_address(pfn_to_page(dst_pfn)),
1509	pfn_to_page(src_pfn));
1510	}
1511	#endif /* CONFIG_HIGHMEM */
1512
1513	/*
1514	* Copy data pages will copy all pages into pages pulled from the copy_bm.
1515	* If a page was entirely filled with zeros it will be marked in the zero_bm.
1516	*
1517	* Returns the number of pages copied.
1518	*/
1519	static unsigned long copy_data_pages(struct memory_bitmap *copy_bm,
1520	struct memory_bitmap *orig_bm,
1521	struct memory_bitmap *zero_bm)
1522	{
1523	unsigned long copied_pages = 0;
1524	struct zone *zone;
1525	unsigned long pfn, copy_pfn;
1526
1527	for_each_populated_zone(zone) {
1528	unsigned long max_zone_pfn;
1529
1530	mark_free_pages(zone);
1531	max_zone_pfn = zone_end_pfn(zone);
1532	for (pfn = zone->zone_start_pfn; pfn < max_zone_pfn; pfn++)
1533	if (page_is_saveable(zone, pfn))
1534	memory_bm_set_bit(orig_bm, pfn);
1535	}
1536	memory_bm_position_reset(orig_bm);
1537	memory_bm_position_reset(copy_bm);
1538	copy_pfn = memory_bm_next_pfn(copy_bm);
1539	for (;;) {
1540	pfn = memory_bm_next_pfn(orig_bm);
1541	if (unlikely(pfn == BM_END_OF_MAP))
1542	break;
1543	if (copy_data_page(copy_pfn, pfn)) {
1544	memory_bm_set_bit(zero_bm, pfn);
1545	/* Use this copy_pfn for a page that is not full of zeros */
1546	continue;
1547	}
1548	copied_pages++;
1549	copy_pfn = memory_bm_next_pfn(copy_bm);
1550	}
1551	return copied_pages;
1552	}
1553
1554	/* Total number of image pages */
1555	static unsigned int nr_copy_pages;
1556	/* Number of pages needed for saving the original pfns of the image pages */
1557	static unsigned int nr_meta_pages;
1558	/* Number of zero pages */
1559	static unsigned int nr_zero_pages;
1560
1561	/*
1562	* Numbers of normal and highmem page frames allocated for hibernation image
1563	* before suspending devices.
1564	*/
1565	static unsigned int alloc_normal, alloc_highmem;
1566	/*
1567	* Memory bitmap used for marking saveable pages (during hibernation) or
1568	* hibernation image pages (during restore)
1569	*/
1570	static struct memory_bitmap orig_bm;
1571	/*
1572	* Memory bitmap used during hibernation for marking allocated page frames that
1573	* will contain copies of saveable pages. During restore it is initially used
1574	* for marking hibernation image pages, but then the set bits from it are
1575	* duplicated in @orig_bm and it is released. On highmem systems it is next
1576	* used for marking "safe" highmem pages, but it has to be reinitialized for
1577	* this purpose.
1578	*/
1579	static struct memory_bitmap copy_bm;
1580
1581	/* Memory bitmap which tracks which saveable pages were zero filled. */
1582	static struct memory_bitmap zero_bm;
1583
1584	/**
1585	* swsusp_free - Free pages allocated for hibernation image.
1586	*
1587	* Image pages are allocated before snapshot creation, so they need to be
1588	* released after resume.
1589	*/
1590	void swsusp_free(void)
1591	{
1592	unsigned long fb_pfn, fr_pfn;
1593
1594	if (!forbidden_pages_map \|\| !free_pages_map)
1595	goto out;
1596
1597	memory_bm_position_reset(forbidden_pages_map);
1598	memory_bm_position_reset(free_pages_map);
1599
1600	loop:
1601	fr_pfn = memory_bm_next_pfn(free_pages_map);
1602	fb_pfn = memory_bm_next_pfn(forbidden_pages_map);
1603
1604	/*
1605	* Find the next bit set in both bitmaps. This is guaranteed to
1606	* terminate when fb_pfn == fr_pfn == BM_END_OF_MAP.
1607	*/
1608	do {
1609	if (fb_pfn < fr_pfn)
1610	fb_pfn = memory_bm_next_pfn(forbidden_pages_map);
1611	if (fr_pfn < fb_pfn)
1612	fr_pfn = memory_bm_next_pfn(free_pages_map);
1613	} while (fb_pfn != fr_pfn);
1614
1615	if (fr_pfn != BM_END_OF_MAP && pfn_valid(fr_pfn)) {
1616	struct page *page = pfn_to_page(fr_pfn);
1617
1618	memory_bm_clear_current(forbidden_pages_map);
1619	memory_bm_clear_current(free_pages_map);
1620	hibernate_restore_unprotect_page(page_address(page));
1621	__free_page(page);
1622	goto loop;
1623	}
1624
1625	out:
1626	nr_copy_pages = 0;
1627	nr_meta_pages = 0;
1628	nr_zero_pages = 0;
1629	restore_pblist = NULL;
1630	buffer = NULL;
1631	alloc_normal = 0;
1632	alloc_highmem = 0;
1633	hibernate_restore_protection_end();
1634	}
1635
1636	/* Helper functions used for the shrinking of memory. */
1637
1638	#define GFP_IMAGE (GFP_KERNEL \| __GFP_NOWARN)
1639
1640	/**
1641	* preallocate_image_pages - Allocate a number of pages for hibernation image.
1642	* @nr_pages: Number of page frames to allocate.
1643	* @mask: GFP flags to use for the allocation.
1644	*
1645	* Return value: Number of page frames actually allocated
1646	*/
1647	static unsigned long preallocate_image_pages(unsigned long nr_pages, gfp_t mask)
1648	{
1649	unsigned long nr_alloc = 0;
1650
1651	while (nr_pages > 0) {
1652	struct page *page;
1653
1654	page = alloc_image_page(mask);
1655	if (!page)
1656	break;
1657	memory_bm_set_bit(&copy_bm, page_to_pfn(page));
1658	if (PageHighMem(page))
1659	alloc_highmem++;
1660	else
1661	alloc_normal++;
1662	nr_pages--;
1663	nr_alloc++;
1664	}
1665
1666	return nr_alloc;
1667	}
1668
1669	static unsigned long preallocate_image_memory(unsigned long nr_pages,
1670	unsigned long avail_normal)
1671	{
1672	unsigned long alloc;
1673
1674	if (avail_normal <= alloc_normal)
1675	return 0;
1676
1677	alloc = avail_normal - alloc_normal;
1678	if (nr_pages < alloc)
1679	alloc = nr_pages;
1680
1681	return preallocate_image_pages(alloc, GFP_IMAGE);
1682	}
1683
1684	#ifdef CONFIG_HIGHMEM
1685	static unsigned long preallocate_image_highmem(unsigned long nr_pages)
1686	{
1687	return preallocate_image_pages(nr_pages, GFP_IMAGE \| __GFP_HIGHMEM);
1688	}
1689
1690	/*
1691	* __fraction - Compute (an approximation of) x * (multiplier / base).
1692	*/
1693	static unsigned long __fraction(u64 x, u64 multiplier, u64 base)
1694	{
1695	return div64_u64(x * multiplier, base);
1696	}
1697
1698	static unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1699	unsigned long highmem,
1700	unsigned long total)
1701	{
1702	unsigned long alloc = __fraction(nr_pages, highmem, total);
1703
1704	return preallocate_image_pages(alloc, GFP_IMAGE \| __GFP_HIGHMEM);
1705	}
1706	#else /* CONFIG_HIGHMEM */
1707	static inline unsigned long preallocate_image_highmem(unsigned long nr_pages)
1708	{
1709	return 0;
1710	}
1711
1712	static inline unsigned long preallocate_highmem_fraction(unsigned long nr_pages,
1713	unsigned long highmem,
1714	unsigned long total)
1715	{
1716	return 0;
1717	}
1718	#endif /* CONFIG_HIGHMEM */
1719
1720	/**
1721	* free_unnecessary_pages - Release preallocated pages not needed for the image.
1722	*/
1723	static unsigned long free_unnecessary_pages(void)
1724	{
1725	unsigned long save, to_free_normal, to_free_highmem, free;
1726
1727	save = count_data_pages();
1728	if (alloc_normal >= save) {
1729	to_free_normal = alloc_normal - save;
1730	save = 0;
1731	} else {
1732	to_free_normal = 0;
1733	save -= alloc_normal;
1734	}
1735	save += count_highmem_pages();
1736	if (alloc_highmem >= save) {
1737	to_free_highmem = alloc_highmem - save;
1738	} else {
1739	to_free_highmem = 0;
1740	save -= alloc_highmem;
1741	if (to_free_normal > save)
1742	to_free_normal -= save;
1743	else
1744	to_free_normal = 0;
1745	}
1746	free = to_free_normal + to_free_highmem;
1747
1748	memory_bm_position_reset(&copy_bm);
1749
1750	while (to_free_normal > 0 \|\| to_free_highmem > 0) {
1751	unsigned long pfn = memory_bm_next_pfn(&copy_bm);
1752	struct page *page = pfn_to_page(pfn);
1753
1754	if (PageHighMem(page)) {
1755	if (!to_free_highmem)
1756	continue;
1757	to_free_highmem--;
1758	alloc_highmem--;
1759	} else {
1760	if (!to_free_normal)
1761	continue;
1762	to_free_normal--;
1763	alloc_normal--;
1764	}
1765	memory_bm_clear_bit(&copy_bm, pfn);
1766	swsusp_unset_page_forbidden(page);
1767	swsusp_unset_page_free(page);
1768	__free_page(page);
1769	}
1770
1771	return free;
1772	}
1773
1774	/**
1775	* minimum_image_size - Estimate the minimum acceptable size of an image.
1776	* @saveable: Number of saveable pages in the system.
1777	*
1778	* We want to avoid attempting to free too much memory too hard, so estimate the
1779	* minimum acceptable size of a hibernation image to use as the lower limit for
1780	* preallocating memory.
1781	*
1782	* We assume that the minimum image size should be proportional to
1783	*
1784	* [number of saveable pages] - [number of pages that can be freed in theory]
1785	*
1786	* where the second term is the sum of (1) reclaimable slab pages, (2) active
1787	* and (3) inactive anonymous pages, (4) active and (5) inactive file pages.
1788	*/
1789	static unsigned long minimum_image_size(unsigned long saveable)
1790	{
1791	unsigned long size;
1792
1793	size = global_node_page_state_pages(NR_SLAB_RECLAIMABLE_B)
1794	+ global_node_page_state(NR_ACTIVE_ANON)
1795	+ global_node_page_state(NR_INACTIVE_ANON)
1796	+ global_node_page_state(NR_ACTIVE_FILE)
1797	+ global_node_page_state(NR_INACTIVE_FILE);
1798
1799	return saveable <= size ? 0 : saveable - size;
1800	}
1801
1802	/**
1803	* hibernate_preallocate_memory - Preallocate memory for hibernation image.
1804	*
1805	* To create a hibernation image it is necessary to make a copy of every page
1806	* frame in use. We also need a number of page frames to be free during
1807	* hibernation for allocations made while saving the image and for device
1808	* drivers, in case they need to allocate memory from their hibernation
1809	* callbacks (these two numbers are given by PAGES_FOR_IO (which is a rough
1810	* estimate) and reserved_size divided by PAGE_SIZE (which is tunable through
1811	* /sys/power/reserved_size, respectively). To make this happen, we compute the
1812	* total number of available page frames and allocate at least
1813	*
1814	* ([page frames total] - PAGES_FOR_IO - [metadata pages]) / 2
1815	* - 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE)
1816	*
1817	* of them, which corresponds to the maximum size of a hibernation image.
1818	*
1819	* If image_size is set below the number following from the above formula,
1820	* the preallocation of memory is continued until the total number of saveable
1821	* pages in the system is below the requested image size or the minimum
1822	* acceptable image size returned by minimum_image_size(), whichever is greater.
1823	*/
1824	int hibernate_preallocate_memory(void)
1825	{
1826	struct zone *zone;
1827	unsigned long saveable, size, max_size, count, highmem, pages = 0;
1828	unsigned long alloc, save_highmem, pages_highmem, avail_normal;
1829	ktime_t start, stop;
1830	int error;
1831
1832	pr_info("Preallocating image memory\n");
1833	start = ktime_get();
1834
1835	error = memory_bm_create(&orig_bm, GFP_IMAGE, PG_ANY);
1836	if (error) {
1837	pr_err("Cannot allocate original bitmap\n");
1838	goto err_out;
1839	}
1840
1841	error = memory_bm_create(&copy_bm, GFP_IMAGE, PG_ANY);
1842	if (error) {
1843	pr_err("Cannot allocate copy bitmap\n");
1844	goto err_out;
1845	}
1846
1847	error = memory_bm_create(&zero_bm, GFP_IMAGE, PG_ANY);
1848	if (error) {
1849	pr_err("Cannot allocate zero bitmap\n");
1850	goto err_out;
1851	}
1852
1853	alloc_normal = 0;
1854	alloc_highmem = 0;
1855	nr_zero_pages = 0;
1856
1857	/* Count the number of saveable data pages. */
1858	save_highmem = count_highmem_pages();
1859	saveable = count_data_pages();
1860
1861	/*
1862	* Compute the total number of page frames we can use (count) and the
1863	* number of pages needed for image metadata (size).
1864	*/
1865	count = saveable;
1866	saveable += save_highmem;
1867	highmem = save_highmem;
1868	size = 0;
1869	for_each_populated_zone(zone) {
1870	size += snapshot_additional_pages(zone);
1871	if (is_highmem(zone))
1872	highmem += zone_page_state(zone, NR_FREE_PAGES);
1873	else
1874	count += zone_page_state(zone, NR_FREE_PAGES);
1875	}
1876	avail_normal = count;
1877	count += highmem;
1878	count -= totalreserve_pages;
1879
1880	/* Compute the maximum number of saveable pages to leave in memory. */
1881	max_size = (count - (size + PAGES_FOR_IO)) / 2
1882	- 2 * DIV_ROUND_UP(reserved_size, PAGE_SIZE);
1883	/* Compute the desired number of image pages specified by image_size. */
1884	size = DIV_ROUND_UP(image_size, PAGE_SIZE);
1885	if (size > max_size)
1886	size = max_size;
1887	/*
1888	* If the desired number of image pages is at least as large as the
1889	* current number of saveable pages in memory, allocate page frames for
1890	* the image and we're done.
1891	*/
1892	if (size >= saveable) {
1893	pages = preallocate_image_highmem(save_highmem);
1894	pages += preallocate_image_memory(saveable - pages, avail_normal);
1895	goto out;
1896	}
1897
1898	/* Estimate the minimum size of the image. */
1899	pages = minimum_image_size(saveable);
1900	/*
1901	* To avoid excessive pressure on the normal zone, leave room in it to
1902	* accommodate an image of the minimum size (unless it's already too
1903	* small, in which case don't preallocate pages from it at all).
1904	*/
1905	if (avail_normal > pages)
1906	avail_normal -= pages;
1907	else
1908	avail_normal = 0;
1909	if (size < pages)
1910	size = min_t(unsigned long, pages, max_size);
1911
1912	/*
1913	* Let the memory management subsystem know that we're going to need a
1914	* large number of page frames to allocate and make it free some memory.
1915	* NOTE: If this is not done, performance will be hurt badly in some
1916	* test cases.
1917	*/
1918	shrink_all_memory(saveable - size);
1919
1920	/*
1921	* The number of saveable pages in memory was too high, so apply some
1922	* pressure to decrease it. First, make room for the largest possible
1923	* image and fail if that doesn't work. Next, try to decrease the size
1924	* of the image as much as indicated by 'size' using allocations from
1925	* highmem and non-highmem zones separately.
1926	*/
1927	pages_highmem = preallocate_image_highmem(highmem / 2);
1928	alloc = count - max_size;
1929	if (alloc > pages_highmem)
1930	alloc -= pages_highmem;
1931	else
1932	alloc = 0;
1933	pages = preallocate_image_memory(alloc, avail_normal);
1934	if (pages < alloc) {
1935	/* We have exhausted non-highmem pages, try highmem. */
1936	alloc -= pages;
1937	pages += pages_highmem;
1938	pages_highmem = preallocate_image_highmem(alloc);
1939	if (pages_highmem < alloc) {
1940	pr_err("Image allocation is %lu pages short\n",
1941	alloc - pages_highmem);
1942	goto err_out;
1943	}
1944	pages += pages_highmem;
1945	/*
1946	* size is the desired number of saveable pages to leave in
1947	* memory, so try to preallocate (all memory - size) pages.
1948	*/
1949	alloc = (count - pages) - size;
1950	pages += preallocate_image_highmem(alloc);
1951	} else {
1952	/*
1953	* There are approximately max_size saveable pages at this point
1954	* and we want to reduce this number down to size.
1955	*/
1956	alloc = max_size - size;
1957	size = preallocate_highmem_fraction(alloc, highmem, count);
1958	pages_highmem += size;
1959	alloc -= size;
1960	size = preallocate_image_memory(alloc, avail_normal);
1961	pages_highmem += preallocate_image_highmem(alloc - size);
1962	pages += pages_highmem + size;
1963	}
1964
1965	/*
1966	* We only need as many page frames for the image as there are saveable
1967	* pages in memory, but we have allocated more. Release the excessive
1968	* ones now.
1969	*/
1970	pages -= free_unnecessary_pages();
1971
1972	out:
1973	stop = ktime_get();
1974	pr_info("Allocated %lu pages for snapshot\n", pages);
1975	swsusp_show_speed(start, stop, pages, "Allocated");
1976
1977	return 0;
1978
1979	err_out:
1980	swsusp_free();
1981	return -ENOMEM;
1982	}
1983
1984	#ifdef CONFIG_HIGHMEM
1985	/*
1986	* count_pages_for_highmem - Count non-highmem pages needed for copying highmem.
1987	*
1988	* Compute the number of non-highmem pages that will be necessary for creating
1989	* copies of highmem pages.
1990	*/
1991	static unsigned int count_pages_for_highmem(unsigned int nr_highmem)
1992	{
1993	unsigned int free_highmem = count_free_highmem_pages() + alloc_highmem;
1994
1995	if (free_highmem >= nr_highmem)
1996	nr_highmem = 0;
1997	else
1998	nr_highmem -= free_highmem;
1999
2000	return nr_highmem;
2001	}
2002	#else
2003	static unsigned int count_pages_for_highmem(unsigned int nr_highmem) { return 0; }
2004	#endif /* CONFIG_HIGHMEM */
2005
2006	/*
2007	* enough_free_mem - Check if there is enough free memory for the image.
2008	*/
2009	static int enough_free_mem(unsigned int nr_pages, unsigned int nr_highmem)
2010	{
2011	struct zone *zone;
2012	unsigned int free = alloc_normal;
2013
2014	for_each_populated_zone(zone)
2015	if (!is_highmem(zone))
2016	free += zone_page_state(zone, NR_FREE_PAGES);
2017
2018	nr_pages += count_pages_for_highmem(nr_highmem);
2019	pr_debug("Normal pages needed: %u + %u, available pages: %u\n",
2020	nr_pages, PAGES_FOR_IO, free);
2021
2022	return free > nr_pages + PAGES_FOR_IO;
2023	}
2024
2025	#ifdef CONFIG_HIGHMEM
2026	/*
2027	* get_highmem_buffer - Allocate a buffer for highmem pages.
2028	*
2029	* If there are some highmem pages in the hibernation image, we may need a
2030	* buffer to copy them and/or load their data.
2031	*/
2032	static inline int get_highmem_buffer(int safe_needed)
2033	{
2034	buffer = get_image_page(GFP_ATOMIC, safe_needed);
2035	return buffer ? 0 : -ENOMEM;
2036	}
2037
2038	/*
2039	* alloc_highmem_pages - Allocate some highmem pages for the image.
2040	*
2041	* Try to allocate as many pages as needed, but if the number of free highmem
2042	* pages is less than that, allocate them all.
2043	*/
2044	static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm,
2045	unsigned int nr_highmem)
2046	{
2047	unsigned int to_alloc = count_free_highmem_pages();
2048
2049	if (to_alloc > nr_highmem)
2050	to_alloc = nr_highmem;
2051
2052	nr_highmem -= to_alloc;
2053	while (to_alloc-- > 0) {
2054	struct page *page;
2055
2056	page = alloc_image_page(__GFP_HIGHMEM\|__GFP_KSWAPD_RECLAIM);
2057	memory_bm_set_bit(bm, page_to_pfn(page));
2058	}
2059	return nr_highmem;
2060	}
2061	#else
2062	static inline int get_highmem_buffer(int safe_needed) { return 0; }
2063
2064	static inline unsigned int alloc_highmem_pages(struct memory_bitmap *bm,
2065	unsigned int n) { return 0; }
2066	#endif /* CONFIG_HIGHMEM */
2067
2068	/*
2069	* swsusp_alloc - Allocate memory for hibernation image.
2070	*
2071	* We first try to allocate as many highmem pages as there are
2072	* saveable highmem pages in the system. If that fails, we allocate
2073	* non-highmem pages for the copies of the remaining highmem ones.
2074	*
2075	* In this approach it is likely that the copies of highmem pages will
2076	* also be located in the high memory, because of the way in which
2077	* copy_data_pages() works.
2078	*/
2079	static int swsusp_alloc(struct memory_bitmap *copy_bm,
2080	unsigned int nr_pages, unsigned int nr_highmem)
2081	{
2082	if (nr_highmem > 0) {
2083	if (get_highmem_buffer(PG_ANY))
2084	goto err_out;
2085	if (nr_highmem > alloc_highmem) {
2086	nr_highmem -= alloc_highmem;
2087	nr_pages += alloc_highmem_pages(copy_bm, nr_highmem);
2088	}
2089	}
2090	if (nr_pages > alloc_normal) {
2091	nr_pages -= alloc_normal;
2092	while (nr_pages-- > 0) {
2093	struct page *page;
2094
2095	page = alloc_image_page(GFP_ATOMIC);
2096	if (!page)
2097	goto err_out;
2098	memory_bm_set_bit(copy_bm, page_to_pfn(page));
2099	}
2100	}
2101
2102	return 0;
2103
2104	err_out:
2105	swsusp_free();
2106	return -ENOMEM;
2107	}
2108
2109	asmlinkage __visible int swsusp_save(void)
2110	{
2111	unsigned int nr_pages, nr_highmem;
2112
2113	pm_deferred_pr_dbg("Creating image\n");
2114
2115	drain_local_pages(NULL);
2116	nr_pages = count_data_pages();
2117	nr_highmem = count_highmem_pages();
2118	pm_deferred_pr_dbg("Need to copy %u pages\n", nr_pages + nr_highmem);
2119
2120	if (!enough_free_mem(nr_pages, nr_highmem)) {
2121	pm_deferred_pr_dbg("Not enough free memory for image creation\n");
2122	return -ENOMEM;
2123	}
2124
2125	if (swsusp_alloc(&copy_bm, nr_pages, nr_highmem))
2126	return -ENOMEM;
2127
2128	/*
2129	* During allocating of suspend pagedir, new cold pages may appear.
2130	* Kill them.
2131	*/
2132	drain_local_pages(NULL);
2133	nr_copy_pages = copy_data_pages(&copy_bm, &orig_bm, &zero_bm);
2134
2135	/*
2136	* End of critical section. From now on, we can write to memory,
2137	* but we should not touch disk. This specially means we must _not_
2138	* touch swap space! Except we must write out our image of course.
2139	*/
2140	nr_pages += nr_highmem;
2141	/* We don't actually copy the zero pages */
2142	nr_zero_pages = nr_pages - nr_copy_pages;
2143	nr_meta_pages = DIV_ROUND_UP(nr_pages * sizeof(long), PAGE_SIZE);
2144
2145	pm_deferred_pr_dbg("Image created (%d pages copied, %d zero pages)\n",
2146	nr_copy_pages, nr_zero_pages);
2147
2148	return 0;
2149	}
2150
2151	#ifndef CONFIG_ARCH_HIBERNATION_HEADER
2152	static int init_header_complete(struct swsusp_info *info)
2153	{
2154	memcpy(&info->uts, init_utsname(), sizeof(struct new_utsname));
2155	info->version_code = LINUX_VERSION_CODE;
2156	return 0;
2157	}
2158
2159	static const char check_image_kernel(struct swsusp_info info)
2160	{
2161	if (info->version_code != LINUX_VERSION_CODE)
2162	return "kernel version";
2163	if (strcmp(info->uts.sysname, init_utsname()->sysname))
2164	return "system type";
2165	if (strcmp(info->uts.release, init_utsname()->release))
2166	return "kernel release";
2167	if (strcmp(info->uts.version, init_utsname()->version))
2168	return "version";
2169	if (strcmp(info->uts.machine, init_utsname()->machine))
2170	return "machine";
2171	return NULL;
2172	}
2173	#endif /* CONFIG_ARCH_HIBERNATION_HEADER */
2174
2175	unsigned long snapshot_get_image_size(void)
2176	{
2177	return nr_copy_pages + nr_meta_pages + 1;
2178	}
2179
2180	static int init_header(struct swsusp_info *info)
2181	{
2182	memset(info, 0, sizeof(struct swsusp_info));
2183	info->num_physpages = get_num_physpages();
2184	info->image_pages = nr_copy_pages;
2185	info->pages = snapshot_get_image_size();
2186	info->size = info->pages;
2187	info->size <<= PAGE_SHIFT;
2188	return init_header_complete(info);
2189	}
2190
2191	#define ENCODED_PFN_ZERO_FLAG ((unsigned long)1 << (BITS_PER_LONG - 1))
2192	#define ENCODED_PFN_MASK (~ENCODED_PFN_ZERO_FLAG)
2193
2194	/**
2195	* pack_pfns - Prepare PFNs for saving.
2196	* @bm: Memory bitmap.
2197	* @buf: Memory buffer to store the PFNs in.
2198	* @zero_bm: Memory bitmap containing PFNs of zero pages.
2199	*
2200	* PFNs corresponding to set bits in @bm are stored in the area of memory
2201	* pointed to by @buf (1 page at a time). Pages which were filled with only
2202	* zeros will have the highest bit set in the packed format to distinguish
2203	* them from PFNs which will be contained in the image file.
2204	*/
2205	static inline void pack_pfns(unsigned long buf, struct memory_bitmap bm,
2206	struct memory_bitmap *zero_bm)
2207	{
2208	int j;
2209
2210	for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
2211	buf[j] = memory_bm_next_pfn(bm);
2212	if (unlikely(buf[j] == BM_END_OF_MAP))
2213	break;
2214	if (memory_bm_test_bit(zero_bm, buf[j]))
2215	buf[j] \|= ENCODED_PFN_ZERO_FLAG;
2216	}
2217	}
2218
2219	/**
2220	* snapshot_read_next - Get the address to read the next image page from.
2221	* @handle: Snapshot handle to be used for the reading.
2222	*
2223	* On the first call, @handle should point to a zeroed snapshot_handle
2224	* structure. The structure gets populated then and a pointer to it should be
2225	* passed to this function every next time.
2226	*
2227	* On success, the function returns a positive number. Then, the caller
2228	* is allowed to read up to the returned number of bytes from the memory
2229	* location computed by the data_of() macro.
2230	*
2231	* The function returns 0 to indicate the end of the data stream condition,
2232	* and negative numbers are returned on errors. If that happens, the structure
2233	* pointed to by @handle is not updated and should not be used any more.
2234	*/
2235	int snapshot_read_next(struct snapshot_handle *handle)
2236	{
2237	if (handle->cur > nr_meta_pages + nr_copy_pages)
2238	return 0;
2239
2240	if (!buffer) {
2241	/* This makes the buffer be freed by swsusp_free() */
2242	buffer = get_image_page(GFP_ATOMIC, PG_ANY);
2243	if (!buffer)
2244	return -ENOMEM;
2245	}
2246	if (!handle->cur) {
2247	int error;
2248
2249	error = init_header((struct swsusp_info *)buffer);
2250	if (error)
2251	return error;
2252	handle->buffer = buffer;
2253	memory_bm_position_reset(&orig_bm);
2254	memory_bm_position_reset(&copy_bm);
2255	} else if (handle->cur <= nr_meta_pages) {
2256	clear_page(buffer);
2257	pack_pfns(buffer, &orig_bm, &zero_bm);
2258	} else {
2259	struct page *page;
2260
2261	page = pfn_to_page(memory_bm_next_pfn(&copy_bm));
2262	if (PageHighMem(page)) {
2263	/*
2264	* Highmem pages are copied to the buffer,
2265	* because we can't return with a kmapped
2266	* highmem page (we may not be called again).
2267	*/
2268	void *kaddr;
2269
2270	kaddr = kmap_local_page(page);
2271	copy_page(buffer, kaddr);
2272	kunmap_local(kaddr);
2273	handle->buffer = buffer;
2274	} else {
2275	handle->buffer = page_address(page);
2276	}
2277	}
2278	handle->cur++;
2279	return PAGE_SIZE;
2280	}
2281
2282	static void duplicate_memory_bitmap(struct memory_bitmap *dst,
2283	struct memory_bitmap *src)
2284	{
2285	unsigned long pfn;
2286
2287	memory_bm_position_reset(src);
2288	pfn = memory_bm_next_pfn(src);
2289	while (pfn != BM_END_OF_MAP) {
2290	memory_bm_set_bit(dst, pfn);
2291	pfn = memory_bm_next_pfn(src);
2292	}
2293	}
2294
2295	/*
2296	* mark_unsafe_pages - Mark pages that were used before hibernation.
2297	*
2298	* Mark the pages that cannot be used for storing the image during restoration,
2299	* because they conflict with the pages that had been used before hibernation.
2300	*/
2301	static void mark_unsafe_pages(struct memory_bitmap *bm)
2302	{
2303	unsigned long pfn;
2304
2305	/* Clear the "free"/"unsafe" bit for all PFNs */
2306	memory_bm_position_reset(free_pages_map);
2307	pfn = memory_bm_next_pfn(free_pages_map);
2308	while (pfn != BM_END_OF_MAP) {
2309	memory_bm_clear_current(free_pages_map);
2310	pfn = memory_bm_next_pfn(free_pages_map);
2311	}
2312
2313	/* Mark pages that correspond to the "original" PFNs as "unsafe" */
2314	duplicate_memory_bitmap(free_pages_map, bm);
2315
2316	allocated_unsafe_pages = 0;
2317	}
2318
2319	static int check_header(struct swsusp_info *info)
2320	{
2321	const char *reason;
2322
2323	reason = check_image_kernel(info);
2324	if (!reason && info->num_physpages != get_num_physpages())
2325	reason = "memory size";
2326	if (reason) {
2327	pr_err("Image mismatch: %s\n", reason);
2328	return -EPERM;
2329	}
2330	return 0;
2331	}
2332
2333	/*
2334	* load_header - Check the image header and copy the data from it.
2335	*/
2336	static int load_header(struct swsusp_info *info)
2337	{
2338	int error;
2339
2340	restore_pblist = NULL;
2341	error = check_header(info);
2342	if (!error) {
2343	nr_copy_pages = info->image_pages;
2344	nr_meta_pages = info->pages - info->image_pages - 1;
2345	}
2346	return error;
2347	}
2348
2349	/**
2350	* unpack_orig_pfns - Set bits corresponding to given PFNs in a memory bitmap.
2351	* @bm: Memory bitmap.
2352	* @buf: Area of memory containing the PFNs.
2353	* @zero_bm: Memory bitmap with the zero PFNs marked.
2354	*
2355	* For each element of the array pointed to by @buf (1 page at a time), set the
2356	* corresponding bit in @bm. If the page was originally populated with only
2357	* zeros then a corresponding bit will also be set in @zero_bm.
2358	*/
2359	static int unpack_orig_pfns(unsigned long buf, struct memory_bitmap bm,
2360	struct memory_bitmap *zero_bm)
2361	{
2362	unsigned long decoded_pfn;
2363	bool zero;
2364	int j;
2365
2366	for (j = 0; j < PAGE_SIZE / sizeof(long); j++) {
2367	if (unlikely(buf[j] == BM_END_OF_MAP))
2368	break;
2369
2370	zero = !!(buf[j] & ENCODED_PFN_ZERO_FLAG);
2371	decoded_pfn = buf[j] & ENCODED_PFN_MASK;
2372	if (pfn_valid(decoded_pfn) && memory_bm_pfn_present(bm, decoded_pfn)) {
2373	memory_bm_set_bit(bm, decoded_pfn);
2374	if (zero) {
2375	memory_bm_set_bit(zero_bm, decoded_pfn);
2376	nr_zero_pages++;
2377	}
2378	} else {
2379	if (!pfn_valid(decoded_pfn))
2380	pr_err(FW_BUG "Memory map mismatch at 0x%llx after hibernation\n",
2381	(unsigned long long)PFN_PHYS(decoded_pfn));
2382	return -EFAULT;
2383	}
2384	}
2385
2386	return 0;
2387	}
2388
2389	#ifdef CONFIG_HIGHMEM
2390	/*
2391	* struct highmem_pbe is used for creating the list of highmem pages that
2392	* should be restored atomically during the resume from disk, because the page
2393	* frames they have occupied before the suspend are in use.
2394	*/
2395	struct highmem_pbe {
2396	struct page copy_page; / data is here now */
2397	struct page orig_page; / data was here before the suspend */
2398	struct highmem_pbe *next;
2399	};
2400
2401	/*
2402	* List of highmem PBEs needed for restoring the highmem pages that were
2403	* allocated before the suspend and included in the suspend image, but have
2404	* also been allocated by the "resume" kernel, so their contents cannot be
2405	* written directly to their "original" page frames.
2406	*/
2407	static struct highmem_pbe *highmem_pblist;
2408
2409	/**
2410	* count_highmem_image_pages - Compute the number of highmem pages in the image.
2411	* @bm: Memory bitmap.
2412	*
2413	* The bits in @bm that correspond to image pages are assumed to be set.
2414	*/
2415	static unsigned int count_highmem_image_pages(struct memory_bitmap *bm)
2416	{
2417	unsigned long pfn;
2418	unsigned int cnt = 0;
2419
2420	memory_bm_position_reset(bm);
2421	pfn = memory_bm_next_pfn(bm);
2422	while (pfn != BM_END_OF_MAP) {
2423	if (PageHighMem(pfn_to_page(pfn)))
2424	cnt++;
2425
2426	pfn = memory_bm_next_pfn(bm);
2427	}
2428	return cnt;
2429	}
2430
2431	static unsigned int safe_highmem_pages;
2432
2433	static struct memory_bitmap *safe_highmem_bm;
2434
2435	/**
2436	* prepare_highmem_image - Allocate memory for loading highmem data from image.
2437	* @bm: Pointer to an uninitialized memory bitmap structure.
2438	* @nr_highmem_p: Pointer to the number of highmem image pages.
2439	*
2440	* Try to allocate as many highmem pages as there are highmem image pages
2441	* (@nr_highmem_p points to the variable containing the number of highmem image
2442	* pages). The pages that are "safe" (ie. will not be overwritten when the
2443	* hibernation image is restored entirely) have the corresponding bits set in
2444	* @bm (it must be uninitialized).
2445	*
2446	* NOTE: This function should not be called if there are no highmem image pages.
2447	*/
2448	static int prepare_highmem_image(struct memory_bitmap *bm,
2449	unsigned int *nr_highmem_p)
2450	{
2451	unsigned int to_alloc;
2452
2453	if (memory_bm_create(bm, GFP_ATOMIC, PG_SAFE))
2454	return -ENOMEM;
2455
2456	if (get_highmem_buffer(PG_SAFE))
2457	return -ENOMEM;
2458
2459	to_alloc = count_free_highmem_pages();
2460	if (to_alloc > *nr_highmem_p)
2461	to_alloc = *nr_highmem_p;
2462	else
2463	*nr_highmem_p = to_alloc;
2464
2465	safe_highmem_pages = 0;
2466	while (to_alloc-- > 0) {
2467	struct page *page;
2468
2469	page = alloc_page(__GFP_HIGHMEM);
2470	if (!swsusp_page_is_free(page)) {
2471	/* The page is "safe", set its bit the bitmap */
2472	memory_bm_set_bit(bm, page_to_pfn(page));
2473	safe_highmem_pages++;
2474	}
2475	/* Mark the page as allocated */
2476	swsusp_set_page_forbidden(page);
2477	swsusp_set_page_free(page);
2478	}
2479	memory_bm_position_reset(bm);
2480	safe_highmem_bm = bm;
2481	return 0;
2482	}
2483
2484	static struct page *last_highmem_page;
2485
2486	/*
2487	* get_highmem_page_buffer - Prepare a buffer to store a highmem image page.
2488	*
2489	* For a given highmem image page get a buffer that suspend_write_next() should
2490	* return to its caller to write to.
2491	*
2492	* If the page is to be saved to its "original" page frame or a copy of
2493	* the page is to be made in the highmem, @buffer is returned. Otherwise,
2494	* the copy of the page is to be made in normal memory, so the address of
2495	* the copy is returned.
2496	*
2497	* If @buffer is returned, the caller of suspend_write_next() will write
2498	* the page's contents to @buffer, so they will have to be copied to the
2499	* right location on the next call to suspend_write_next() and it is done
2500	* with the help of copy_last_highmem_page(). For this purpose, if
2501	* @buffer is returned, @last_highmem_page is set to the page to which
2502	* the data will have to be copied from @buffer.
2503	*/
2504	static void get_highmem_page_buffer(struct page page,
2505	struct chain_allocator *ca)
2506	{
2507	struct highmem_pbe *pbe;
2508	void *kaddr;
2509
2510	if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page)) {
2511	/*
2512	* We have allocated the "original" page frame and we can
2513	* use it directly to store the loaded page.
2514	*/
2515	last_highmem_page = page;
2516	return buffer;
2517	}
2518	/*
2519	* The "original" page frame has not been allocated and we have to
2520	* use a "safe" page frame to store the loaded page.
2521	*/
2522	pbe = chain_alloc(ca, sizeof(struct highmem_pbe));
2523	if (!pbe) {
2524	swsusp_free();
2525	return ERR_PTR(-ENOMEM);
2526	}
2527	pbe->orig_page = page;
2528	if (safe_highmem_pages > 0) {
2529	struct page *tmp;
2530
2531	/* Copy of the page will be stored in high memory */
2532	kaddr = buffer;
2533	tmp = pfn_to_page(memory_bm_next_pfn(safe_highmem_bm));
2534	safe_highmem_pages--;
2535	last_highmem_page = tmp;
2536	pbe->copy_page = tmp;
2537	} else {
2538	/* Copy of the page will be stored in normal memory */
2539	kaddr = __get_safe_page(ca->gfp_mask);
2540	if (!kaddr)
2541	return ERR_PTR(-ENOMEM);
2542	pbe->copy_page = virt_to_page(kaddr);
2543	}
2544	pbe->next = highmem_pblist;
2545	highmem_pblist = pbe;
2546	return kaddr;
2547	}
2548
2549	/**
2550	* copy_last_highmem_page - Copy most the most recent highmem image page.
2551	*
2552	* Copy the contents of a highmem image from @buffer, where the caller of
2553	* snapshot_write_next() has stored them, to the right location represented by
2554	* @last_highmem_page .
2555	*/
2556	static void copy_last_highmem_page(void)
2557	{
2558	if (last_highmem_page) {
2559	void *dst;
2560
2561	dst = kmap_local_page(last_highmem_page);
2562	copy_page(dst, buffer);
2563	kunmap_local(dst);
2564	last_highmem_page = NULL;
2565	}
2566	}
2567
2568	static inline int last_highmem_page_copied(void)
2569	{
2570	return !last_highmem_page;
2571	}
2572
2573	static inline void free_highmem_data(void)
2574	{
2575	if (safe_highmem_bm)
2576	memory_bm_free(safe_highmem_bm, PG_UNSAFE_CLEAR);
2577
2578	if (buffer)
2579	free_image_page(buffer, PG_UNSAFE_CLEAR);
2580	}
2581	#else
2582	static unsigned int count_highmem_image_pages(struct memory_bitmap *bm) { return 0; }
2583
2584	static inline int prepare_highmem_image(struct memory_bitmap *bm,
2585	unsigned int *nr_highmem_p) { return 0; }
2586
2587	static inline void get_highmem_page_buffer(struct page page,
2588	struct chain_allocator *ca)
2589	{
2590	return ERR_PTR(-EINVAL);
2591	}
2592
2593	static inline void copy_last_highmem_page(void) {}
2594	static inline int last_highmem_page_copied(void) { return 1; }
2595	static inline void free_highmem_data(void) {}
2596	#endif /* CONFIG_HIGHMEM */
2597
2598	#define PBES_PER_LINKED_PAGE (LINKED_PAGE_DATA_SIZE / sizeof(struct pbe))
2599
2600	/**
2601	* prepare_image - Make room for loading hibernation image.
2602	* @new_bm: Uninitialized memory bitmap structure.
2603	* @bm: Memory bitmap with unsafe pages marked.
2604	* @zero_bm: Memory bitmap containing the zero pages.
2605	*
2606	* Use @bm to mark the pages that will be overwritten in the process of
2607	* restoring the system memory state from the suspend image ("unsafe" pages)
2608	* and allocate memory for the image.
2609	*
2610	* The idea is to allocate a new memory bitmap first and then allocate
2611	* as many pages as needed for image data, but without specifying what those
2612	* pages will be used for just yet. Instead, we mark them all as allocated and
2613	* create a lists of "safe" pages to be used later. On systems with high
2614	* memory a list of "safe" highmem pages is created too.
2615	*
2616	* Because it was not known which pages were unsafe when @zero_bm was created,
2617	* make a copy of it and recreate it within safe pages.
2618	*/
2619	static int prepare_image(struct memory_bitmap new_bm, struct memory_bitmap bm,
2620	struct memory_bitmap *zero_bm)
2621	{
2622	unsigned int nr_pages, nr_highmem;
2623	struct memory_bitmap tmp;
2624	struct linked_page *lp;
2625	int error;
2626
2627	/* If there is no highmem, the buffer will not be necessary */
2628	free_image_page(buffer, PG_UNSAFE_CLEAR);
2629	buffer = NULL;
2630
2631	nr_highmem = count_highmem_image_pages(bm);
2632	mark_unsafe_pages(bm);
2633
2634	error = memory_bm_create(new_bm, GFP_ATOMIC, PG_SAFE);
2635	if (error)
2636	goto Free;
2637
2638	duplicate_memory_bitmap(new_bm, bm);
2639	memory_bm_free(bm, PG_UNSAFE_KEEP);
2640
2641	/* Make a copy of zero_bm so it can be created in safe pages */
2642	error = memory_bm_create(&tmp, GFP_ATOMIC, PG_SAFE);
2643	if (error)
2644	goto Free;
2645
2646	duplicate_memory_bitmap(&tmp, zero_bm);
2647	memory_bm_free(zero_bm, PG_UNSAFE_KEEP);
2648
2649	/* Recreate zero_bm in safe pages */
2650	error = memory_bm_create(zero_bm, GFP_ATOMIC, PG_SAFE);
2651	if (error)
2652	goto Free;
2653
2654	duplicate_memory_bitmap(zero_bm, &tmp);
2655	memory_bm_free(&tmp, PG_UNSAFE_CLEAR);
2656	/* At this point zero_bm is in safe pages and it can be used for restoring. */
2657
2658	if (nr_highmem > 0) {
2659	error = prepare_highmem_image(bm, &nr_highmem);
2660	if (error)
2661	goto Free;
2662	}
2663	/*
2664	* Reserve some safe pages for potential later use.
2665	*
2666	* NOTE: This way we make sure there will be enough safe pages for the
2667	* chain_alloc() in get_buffer(). It is a bit wasteful, but
2668	* nr_copy_pages cannot be greater than 50% of the memory anyway.
2669	*
2670	* nr_copy_pages cannot be less than allocated_unsafe_pages too.
2671	*/
2672	nr_pages = (nr_zero_pages + nr_copy_pages) - nr_highmem - allocated_unsafe_pages;
2673	nr_pages = DIV_ROUND_UP(nr_pages, PBES_PER_LINKED_PAGE);
2674	while (nr_pages > 0) {
2675	lp = get_image_page(GFP_ATOMIC, PG_SAFE);
2676	if (!lp) {
2677	error = -ENOMEM;
2678	goto Free;
2679	}
2680	lp->next = safe_pages_list;
2681	safe_pages_list = lp;
2682	nr_pages--;
2683	}
2684	/* Preallocate memory for the image */
2685	nr_pages = (nr_zero_pages + nr_copy_pages) - nr_highmem - allocated_unsafe_pages;
2686	while (nr_pages > 0) {
2687	lp = (struct linked_page *)get_zeroed_page(GFP_ATOMIC);
2688	if (!lp) {
2689	error = -ENOMEM;
2690	goto Free;
2691	}
2692	if (!swsusp_page_is_free(virt_to_page(lp))) {
2693	/* The page is "safe", add it to the list */
2694	lp->next = safe_pages_list;
2695	safe_pages_list = lp;
2696	}
2697	/* Mark the page as allocated */
2698	swsusp_set_page_forbidden(virt_to_page(lp));
2699	swsusp_set_page_free(virt_to_page(lp));
2700	nr_pages--;
2701	}
2702	return 0;
2703
2704	Free:
2705	swsusp_free();
2706	return error;
2707	}
2708
2709	/*
2710	* get_buffer - Get the address to store the next image data page.
2711	*
2712	* Get the address that snapshot_write_next() should return to its caller to
2713	* write to.
2714	*/
2715	static void get_buffer(struct memory_bitmap bm, struct chain_allocator *ca)
2716	{
2717	struct pbe *pbe;
2718	struct page *page;
2719	unsigned long pfn = memory_bm_next_pfn(bm);
2720
2721	if (pfn == BM_END_OF_MAP)
2722	return ERR_PTR(-EFAULT);
2723
2724	page = pfn_to_page(pfn);
2725	if (PageHighMem(page))
2726	return get_highmem_page_buffer(page, ca);
2727
2728	if (swsusp_page_is_forbidden(page) && swsusp_page_is_free(page))
2729	/*
2730	* We have allocated the "original" page frame and we can
2731	* use it directly to store the loaded page.
2732	*/
2733	return page_address(page);
2734
2735	/*
2736	* The "original" page frame has not been allocated and we have to
2737	* use a "safe" page frame to store the loaded page.
2738	*/
2739	pbe = chain_alloc(ca, sizeof(struct pbe));
2740	if (!pbe) {
2741	swsusp_free();
2742	return ERR_PTR(-ENOMEM);
2743	}
2744	pbe->orig_address = page_address(page);
2745	pbe->address = __get_safe_page(ca->gfp_mask);
2746	if (!pbe->address)
2747	return ERR_PTR(-ENOMEM);
2748	pbe->next = restore_pblist;
2749	restore_pblist = pbe;
2750	return pbe->address;
2751	}
2752
2753	/**
2754	* snapshot_write_next - Get the address to store the next image page.
2755	* @handle: Snapshot handle structure to guide the writing.
2756	*
2757	* On the first call, @handle should point to a zeroed snapshot_handle
2758	* structure. The structure gets populated then and a pointer to it should be
2759	* passed to this function every next time.
2760	*
2761	* On success, the function returns a positive number. Then, the caller
2762	* is allowed to write up to the returned number of bytes to the memory
2763	* location computed by the data_of() macro.
2764	*
2765	* The function returns 0 to indicate the "end of file" condition. Negative
2766	* numbers are returned on errors, in which cases the structure pointed to by
2767	* @handle is not updated and should not be used any more.
2768	*/
2769	int snapshot_write_next(struct snapshot_handle *handle)
2770	{
2771	static struct chain_allocator ca;
2772	int error;
2773
2774	next:
2775	/* Check if we have already loaded the entire image */
2776	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages)
2777	return 0;
2778
2779	if (!handle->cur) {
2780	if (!buffer)
2781	/* This makes the buffer be freed by swsusp_free() */
2782	buffer = get_image_page(GFP_ATOMIC, PG_ANY);
2783
2784	if (!buffer)
2785	return -ENOMEM;
2786
2787	handle->buffer = buffer;
2788	} else if (handle->cur == 1) {
2789	error = load_header(buffer);
2790	if (error)
2791	return error;
2792
2793	safe_pages_list = NULL;
2794
2795	error = memory_bm_create(&copy_bm, GFP_ATOMIC, PG_ANY);
2796	if (error)
2797	return error;
2798
2799	error = memory_bm_create(&zero_bm, GFP_ATOMIC, PG_ANY);
2800	if (error) {
2801	memory_bm_free(&copy_bm, PG_UNSAFE_CLEAR);
2802	return error;
2803	}
2804	nr_zero_pages = 0;
2805
2806	hibernate_restore_protection_begin();
2807	} else if (handle->cur <= nr_meta_pages + 1) {
2808	error = unpack_orig_pfns(buffer, &copy_bm, &zero_bm);
2809	if (error)
2810	return error;
2811
2812	if (handle->cur == nr_meta_pages + 1) {
2813	error = prepare_image(&orig_bm, &copy_bm, &zero_bm);
2814	if (error)
2815	return error;
2816
2817	chain_init(&ca, GFP_ATOMIC, PG_SAFE);
2818	memory_bm_position_reset(&orig_bm);
2819	memory_bm_position_reset(&zero_bm);
2820	restore_pblist = NULL;
2821	handle->buffer = get_buffer(&orig_bm, &ca);
2822	if (IS_ERR(handle->buffer))
2823	return PTR_ERR(handle->buffer);
2824	}
2825	} else {
2826	copy_last_highmem_page();
2827	error = hibernate_restore_protect_page(handle->buffer);
2828	if (error)
2829	return error;
2830	handle->buffer = get_buffer(&orig_bm, &ca);
2831	if (IS_ERR(handle->buffer))
2832	return PTR_ERR(handle->buffer);
2833	}
2834	handle->sync_read = (handle->buffer == buffer);
2835	handle->cur++;
2836
2837	/* Zero pages were not included in the image, memset it and move on. */
2838	if (handle->cur > nr_meta_pages + 1 &&
2839	memory_bm_test_bit(&zero_bm, memory_bm_get_current(&orig_bm))) {
2840	memset(handle->buffer, 0, PAGE_SIZE);
2841	goto next;
2842	}
2843
2844	return PAGE_SIZE;
2845	}
2846
2847	/*
2848	* snapshot_write_finalize - Complete the loading of a hibernation image.
2849	*
2850	* Must be called after the last call to snapshot_write_next() in case the last
2851	* page in the image happens to be a highmem page and its contents should be
2852	* stored in highmem. Additionally, it recycles bitmap memory that's not
2853	* necessary any more.
2854	*/
2855	int snapshot_write_finalize(struct snapshot_handle *handle)
2856	{
2857	int error;
2858
2859	/*
2860	* Call snapshot_write_next() to drain any trailing zero pages,
2861	* but make sure we're in the data page region first.
2862	* This function can return PAGE_SIZE if the kernel was expecting
2863	* another copy page. Return -ENODATA in that situation.
2864	*/
2865	if (handle->cur > nr_meta_pages + 1) {
2866	error = snapshot_write_next(handle);
2867	if (error)
2868	return error > 0 ? -ENODATA : error;
2869	}
2870	copy_last_highmem_page();
2871	error = hibernate_restore_protect_page(handle->buffer);
2872	/* Do that only if we have loaded the image entirely */
2873	if (handle->cur > 1 && handle->cur > nr_meta_pages + nr_copy_pages + nr_zero_pages) {
2874	memory_bm_recycle(&orig_bm);
2875	free_highmem_data();
2876	}
2877	return error;
2878	}
2879
2880	int snapshot_image_loaded(struct snapshot_handle *handle)
2881	{
2882	return !(!nr_copy_pages \|\| !last_highmem_page_copied() \|\|
2883	handle->cur <= nr_meta_pages + nr_copy_pages + nr_zero_pages);
2884	}
2885
2886	#ifdef CONFIG_HIGHMEM
2887	/* Assumes that @buf is ready and points to a "safe" page */
2888	static inline void swap_two_pages_data(struct page p1, struct page p2,
2889	void *buf)
2890	{
2891	void kaddr1, kaddr2;
2892
2893	kaddr1 = kmap_local_page(p1);
2894	kaddr2 = kmap_local_page(p2);
2895	copy_page(buf, kaddr1);
2896	copy_page(kaddr1, kaddr2);
2897	copy_page(kaddr2, buf);
2898	kunmap_local(kaddr2);
2899	kunmap_local(kaddr1);
2900	}
2901
2902	/**
2903	* restore_highmem - Put highmem image pages into their original locations.
2904	*
2905	* For each highmem page that was in use before hibernation and is included in
2906	* the image, and also has been allocated by the "restore" kernel, swap its
2907	* current contents with the previous (ie. "before hibernation") ones.
2908	*
2909	* If the restore eventually fails, we can call this function once again and
2910	* restore the highmem state as seen by the restore kernel.
2911	*/
2912	int restore_highmem(void)
2913	{
2914	struct highmem_pbe *pbe = highmem_pblist;
2915	void *buf;
2916
2917	if (!pbe)
2918	return 0;
2919
2920	buf = get_image_page(GFP_ATOMIC, PG_SAFE);
2921	if (!buf)
2922	return -ENOMEM;
2923
2924	while (pbe) {
2925	swap_two_pages_data(pbe->copy_page, pbe->orig_page, buf);
2926	pbe = pbe->next;
2927	}
2928	free_image_page(buf, PG_UNSAFE_CLEAR);
2929	return 0;
2930	}
2931	#endif /* CONFIG_HIGHMEM */