projects/linux·blob master · refs

255 lines · 6.1 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * memory tiering: migrate cold pages in node 0 and hot pages in node 1 to node
4 * 1 and node 0, respectively. Adjust the hotness/coldness threshold aiming
5 * resulting 99.6 % node 0 utilization ratio.
6 */
7
8#define pr_fmt(fmt) "damon_sample_mtier: " fmt
9
10#include <linux/damon.h>
11#include <linux/init.h>
12#include <linux/kernel.h>
13#include <linux/module.h>
14
15#ifdef MODULE_PARAM_PREFIX
16#undef MODULE_PARAM_PREFIX
17#endif
18#define MODULE_PARAM_PREFIX "damon_sample_mtier."
19
20static unsigned long node0_start_addr __read_mostly;
21module_param(node0_start_addr, ulong, 0600);
22
23static unsigned long node0_end_addr __read_mostly;
24module_param(node0_end_addr, ulong, 0600);
25
26static unsigned long node1_start_addr __read_mostly;
27module_param(node1_start_addr, ulong, 0600);
28
29static unsigned long node1_end_addr __read_mostly;
30module_param(node1_end_addr, ulong, 0600);
31
32static unsigned long node0_mem_used_bp __read_mostly = 9970;
33module_param(node0_mem_used_bp, ulong, 0600);
34
35static unsigned long node0_mem_free_bp __read_mostly = 50;
36module_param(node0_mem_free_bp, ulong, 0600);
37
38static int damon_sample_mtier_enable_store(
39 const char *val, const struct kernel_param *kp);
40
41static const struct kernel_param_ops enabled_param_ops = {
42 .set = damon_sample_mtier_enable_store,
43 .get = param_get_bool,
44};
45
46static bool enabled __read_mostly;
47module_param_cb(enabled, &enabled_param_ops, &enabled, 0600);
48MODULE_PARM_DESC(enabled, "Enable or disable DAMON_SAMPLE_MTIER");
49
50static bool detect_node_addresses __read_mostly;
51module_param(detect_node_addresses, bool, 0600);
52
53static struct damon_ctx *ctxs[2];
54
55struct region_range {
56 phys_addr_t start;
57 phys_addr_t end;
58};
59
60static int nid_to_phys(int target_node, struct region_range *range)
61{
62 if (!node_online(target_node)) {
63 pr_err("NUMA node %d is not online\n", target_node);
64 return -EINVAL;
65 }
66
67 range->start = PFN_PHYS(node_start_pfn(target_node));
68 range->end = PFN_PHYS(node_end_pfn(target_node));
69
70 return 0;
71}
72
73static struct damon_ctx *damon_sample_mtier_build_ctx(bool promote)
74{
75 struct damon_ctx *ctx;
76 struct damon_attrs attrs;
77 struct damon_target *target;
78 struct damos *scheme;
79 struct damos_quota_goal *quota_goal;
80 struct damos_filter *filter;
81 struct region_range addr;
82 struct damon_addr_range range;
83 int ret;
84
85 ctx = damon_new_ctx();
86 if (!ctx)
87 return NULL;
88 attrs = (struct damon_attrs) {
89 .sample_interval = 5 * USEC_PER_MSEC,
90 .aggr_interval = 100 * USEC_PER_MSEC,
91 .ops_update_interval = 60 * USEC_PER_MSEC * MSEC_PER_SEC,
92 .min_nr_regions = 10,
93 .max_nr_regions = 1000,
94 };
95
96 /*
97 * auto-tune sampling and aggregation interval aiming 4% DAMON-observed
98 * accesses ratio, keeping sampling interval in [5ms, 10s] range.
99 */
100 attrs.intervals_goal = (struct damon_intervals_goal) {
101 .access_bp = 400, .aggrs = 3,
102 .min_sample_us = 5000, .max_sample_us = 10000000,
103 };
104 if (damon_set_attrs(ctx, &attrs))
105 goto free_out;
106 if (damon_select_ops(ctx, DAMON_OPS_PADDR))
107 goto free_out;
108
109 target = damon_new_target();
110 if (!target)
111 goto free_out;
112 damon_add_target(ctx, target);
113
114 if (detect_node_addresses) {
115 ret = promote ? nid_to_phys(1, &addr) : nid_to_phys(0, &addr);
116 if (ret)
117 goto free_out;
118 } else {
119 addr.start = promote ? node1_start_addr : node0_start_addr;
120 addr.end = promote ? node1_end_addr : node0_end_addr;
121 }
122
123 if (addr.start >= addr.end)
124 goto free_out;
125
126 range.start = addr.start;
127 range.end = addr.end;
128
129 ret = damon_set_regions(target, &range, 1, DAMON_MIN_REGION_SZ);
130 if (ret)
131 goto free_out;
132
133 scheme = damon_new_scheme(
134 /* access pattern */
135 &(struct damos_access_pattern) {
136 .min_sz_region = PAGE_SIZE,
137 .max_sz_region = ULONG_MAX,
138 .min_nr_accesses = promote ? 1 : 0,
139 .max_nr_accesses = promote ? UINT_MAX : 0,
140 .min_age_region = 0,
141 .max_age_region = UINT_MAX},
142 /* action */
143 promote ? DAMOS_MIGRATE_HOT : DAMOS_MIGRATE_COLD,
144 1000000, /* apply interval (1s) */
145 &(struct damos_quota){
146 /* 200 MiB per sec by most */
147 .reset_interval = 1000,
148 .sz = 200 * 1024 * 1024,
149 /* ignore size of region when prioritizing */
150 .weight_sz = 0,
151 .weight_nr_accesses = 100,
152 .weight_age = 100,
153 },
154 &(struct damos_watermarks){},
155 promote ? 0 : 1); /* migrate target node id */
156 if (!scheme)
157 goto free_out;
158 damon_set_schemes(ctx, &scheme, 1);
159 /* zero target value causes division by zero in damos_quota_store() */
160 if (!node0_mem_used_bp || !node0_mem_free_bp)
161 goto free_out;
162 quota_goal = damos_new_quota_goal(
163 promote ? DAMOS_QUOTA_NODE_MEM_USED_BP :
164 DAMOS_QUOTA_NODE_MEM_FREE_BP,
165 promote ? node0_mem_used_bp : node0_mem_free_bp);
166 if (!quota_goal)
167 goto free_out;
168 quota_goal->nid = 0;
169 damos_add_quota_goal(&scheme->quota, quota_goal);
170 filter = damos_new_filter(DAMOS_FILTER_TYPE_YOUNG, true, promote);
171 if (!filter)
172 goto free_out;
173 damos_add_filter(scheme, filter);
174 return ctx;
175free_out:
176 damon_destroy_ctx(ctx);
177 return NULL;
178}
179
180static int damon_sample_mtier_start(void)
181{
182 struct damon_ctx *ctx;
183 int err;
184
185 ctx = damon_sample_mtier_build_ctx(true);
186 if (!ctx)
187 return -ENOMEM;
188 ctxs[0] = ctx;
189 ctx = damon_sample_mtier_build_ctx(false);
190 if (!ctx) {
191 damon_destroy_ctx(ctxs[0]);
192 return -ENOMEM;
193 }
194 ctxs[1] = ctx;
195 err = damon_start(ctxs, 2, true);
196 if (!err)
197 return 0;
198
199 damon_destroy_ctx(ctxs[0]);
200 damon_destroy_ctx(ctxs[1]);
201 return err;
202}
203
204static void damon_sample_mtier_stop(void)
205{
206 damon_stop(ctxs, 2);
207 damon_destroy_ctx(ctxs[0]);
208 damon_destroy_ctx(ctxs[1]);
209}
210
211static int damon_sample_mtier_enable_store(
212 const char *val, const struct kernel_param *kp)
213{
214 bool is_enabled = enabled;
215 int err;
216
217 err = kstrtobool(val, &enabled);
218 if (err)
219 return err;
220
221 if (enabled == is_enabled)
222 return 0;
223
224 if (!damon_initialized())
225 return 0;
226
227 if (enabled) {
228 err = damon_sample_mtier_start();
229 if (err)
230 enabled = false;
231 return err;
232 }
233 damon_sample_mtier_stop();
234 return 0;
235}
236
237static int __init damon_sample_mtier_init(void)
238{
239 int err = 0;
240
241 if (!damon_initialized()) {
242 if (enabled)
243 enabled = false;
244 return -ENOMEM;
245 }
246
247 if (enabled) {
248 err = damon_sample_mtier_start();
249 if (err)
250 enabled = false;
251 }
252 return 0;
253}
254
255module_init(damon_sample_mtier_init);