| 1 | |
| 2 | |
| 3 | * linux/mm/mmzone.c
|
| 4 | *
|
| 5 | * management codes for pgdats, zones and page flags
|
| 6 |
|
| 7 |
|
| 8 |
|
| 9 | #include <linux/stddef.h>
|
| 10 | #include <linux/mm.h>
|
| 11 | #include <linux/mmzone.h>
|
| 12 |
|
| 13 | struct pglist_data *first_online_pgdat(void)
|
| 14 | {
|
| 15 | return NODE_DATA(first_online_node);
|
| 16 | }
|
| 17 |
|
| 18 | struct pglist_data *next_online_pgdat(struct pglist_data *pgdat)
|
| 19 | {
|
| 20 | int nid = next_online_node(pgdat->node_id);
|
| 21 |
|
| 22 | if (nid == MAX_NUMNODES)
|
| 23 | return NULL;
|
| 24 | return NODE_DATA(nid);
|
| 25 | }
|
| 26 |
|
| 27 | |
| 28 | * next_zone - helper magic for for_each_zone()
|
| 29 |
|
| 30 | struct zone *next_zone(struct zone *zone)
|
| 31 | {
|
| 32 | pg_data_t *pgdat = zone->zone_pgdat;
|
| 33 |
|
| 34 | if (zone < pgdat->node_zones + MAX_NR_ZONES - 1)
|
| 35 | zone++;
|
| 36 | else {
|
| 37 | pgdat = next_online_pgdat(pgdat);
|
| 38 | if (pgdat)
|
| 39 | zone = pgdat->node_zones;
|
| 40 | else
|
| 41 | zone = NULL;
|
| 42 | }
|
| 43 | return zone;
|
| 44 | }
|
| 45 |
|
| 46 | static inline int zref_in_nodemask(struct zoneref *zref,
|
| 47 | const nodemask_t *nodes)
|
| 48 | {
|
| 49 | #ifdef CONFIG_NUMA
|
| 50 | return node_isset(zonelist_node_idx(zref), *nodes);
|
| 51 | #else
|
| 52 | return 1;
|
| 53 | #endif
|
| 54 | }
|
| 55 |
|
| 56 |
|
| 57 | struct zoneref *__next_zones_zonelist(struct zoneref *z,
|
| 58 | enum zone_type highest_zoneidx,
|
| 59 | const nodemask_t *nodes)
|
| 60 | {
|
| 61 | |
| 62 | * Find the next suitable zone to use for the allocation.
|
| 63 | * Only filter based on nodemask if it's set
|
| 64 |
|
| 65 | if (unlikely(nodes == NULL))
|
| 66 | while (zonelist_zone_idx(z) > highest_zoneidx)
|
| 67 | z++;
|
| 68 | else
|
| 69 | while (zonelist_zone_idx(z) > highest_zoneidx ||
|
| 70 | (zonelist_zone(z) && !zref_in_nodemask(z, nodes)))
|
| 71 | z++;
|
| 72 |
|
| 73 | return z;
|
| 74 | }
|
| 75 |
|
| 76 | void lruvec_init(struct lruvec *lruvec)
|
| 77 | {
|
| 78 | enum lru_list lru;
|
| 79 |
|
| 80 | memset(lruvec, 0, sizeof(struct lruvec));
|
| 81 | spin_lock_init(&lruvec->lru_lock);
|
| 82 | spin_lock_init(&lruvec->cost_lock);
|
| 83 | zswap_lruvec_state_init(lruvec);
|
| 84 |
|
| 85 | for_each_lru(lru)
|
| 86 | INIT_LIST_HEAD(&lruvec->lists[lru]);
|
| 87 | |
| 88 | * The "Unevictable LRU" is imaginary: though its size is maintained,
|
| 89 | * it is never scanned, and unevictable pages are not threaded on it
|
| 90 | * (so that their lru fields can be reused to hold mlock_count).
|
| 91 | * Poison its list head, so that any operations on it would crash.
|
| 92 |
|
| 93 | list_del(&lruvec->lists[LRU_UNEVICTABLE]);
|
| 94 |
|
| 95 | lru_gen_init_lruvec(lruvec);
|
| 96 | }
|
| 97 |
|
| 98 | #if defined(CONFIG_NUMA_BALANCING) && !defined(LAST_CPUPID_NOT_IN_PAGE_FLAGS)
|
| 99 | int folio_xchg_last_cpupid(struct folio *folio, int cpupid)
|
| 100 | {
|
| 101 | unsigned long old_flags, flags;
|
| 102 | int last_cpupid;
|
| 103 |
|
| 104 | old_flags = READ_ONCE(folio->flags.f);
|
| 105 | do {
|
| 106 | flags = old_flags;
|
| 107 | last_cpupid = (flags >> LAST_CPUPID_PGSHIFT) & LAST_CPUPID_MASK;
|
| 108 |
|
| 109 | flags &= ~(LAST_CPUPID_MASK << LAST_CPUPID_PGSHIFT);
|
| 110 | flags |= (cpupid & LAST_CPUPID_MASK) << LAST_CPUPID_PGSHIFT;
|
| 111 | } while (unlikely(!try_cmpxchg(&folio->flags.f, &old_flags, flags)));
|
| 112 |
|
| 113 | return last_cpupid;
|
| 114 | }
|
| 115 | #endif
|