| 1 | |
| 2 | |
| 3 | * Data Access Monitor
|
| 4 |
|
| 5 |
|
| 6 | #define pr_fmt(fmt) "damon: " fmt
|
| 7 |
|
| 8 | #include <linux/damon.h>
|
| 9 | #include <linux/delay.h>
|
| 10 | #include <linux/kthread.h>
|
| 11 | #include <linux/memcontrol.h>
|
| 12 | #include <linux/mm.h>
|
| 13 | #include <linux/psi.h>
|
| 14 | #include <linux/sched.h>
|
| 15 | #include <linux/slab.h>
|
| 16 | #include <linux/string.h>
|
| 17 | #include <linux/string_choices.h>
|
| 18 |
|
| 19 |
|
| 20 | #include "ops-common.h"
|
| 21 |
|
| 22 | #define CREATE_TRACE_POINTS
|
| 23 | #include <trace/events/damon.h>
|
| 24 |
|
| 25 | static DEFINE_MUTEX(damon_lock);
|
| 26 | static int nr_running_ctxs;
|
| 27 | static bool running_exclusive_ctxs;
|
| 28 |
|
| 29 | static DEFINE_MUTEX(damon_ops_lock);
|
| 30 | static struct damon_operations damon_registered_ops[NR_DAMON_OPS];
|
| 31 |
|
| 32 | static struct kmem_cache *damon_region_cache __ro_after_init;
|
| 33 |
|
| 34 |
|
| 35 | static bool __damon_is_registered_ops(enum damon_ops_id id)
|
| 36 | {
|
| 37 | struct damon_operations empty_ops = {};
|
| 38 |
|
| 39 | if (!memcmp(&empty_ops, &damon_registered_ops[id], sizeof(empty_ops)))
|
| 40 | return false;
|
| 41 | return true;
|
| 42 | }
|
| 43 |
|
| 44 | |
| 45 | * damon_is_registered_ops() - Check if a given damon_operations is registered.
|
| 46 | * @id: Id of the damon_operations to check if registered.
|
| 47 | *
|
| 48 | * Return: true if the ops is set, false otherwise.
|
| 49 |
|
| 50 | bool damon_is_registered_ops(enum damon_ops_id id)
|
| 51 | {
|
| 52 | bool registered;
|
| 53 |
|
| 54 | if (id >= NR_DAMON_OPS)
|
| 55 | return false;
|
| 56 | mutex_lock(&damon_ops_lock);
|
| 57 | registered = __damon_is_registered_ops(id);
|
| 58 | mutex_unlock(&damon_ops_lock);
|
| 59 | return registered;
|
| 60 | }
|
| 61 |
|
| 62 | |
| 63 | * damon_register_ops() - Register a monitoring operations set to DAMON.
|
| 64 | * @ops: monitoring operations set to register.
|
| 65 | *
|
| 66 | * This function registers a monitoring operations set of valid &struct
|
| 67 | * damon_operations->id so that others can find and use them later.
|
| 68 | *
|
| 69 | * Return: 0 on success, negative error code otherwise.
|
| 70 |
|
| 71 | int damon_register_ops(struct damon_operations *ops)
|
| 72 | {
|
| 73 | int err = 0;
|
| 74 |
|
| 75 | if (ops->id >= NR_DAMON_OPS)
|
| 76 | return -EINVAL;
|
| 77 |
|
| 78 | mutex_lock(&damon_ops_lock);
|
| 79 |
|
| 80 | if (__damon_is_registered_ops(ops->id))
|
| 81 | err = -EINVAL;
|
| 82 | else
|
| 83 | damon_registered_ops[ops->id] = *ops;
|
| 84 | mutex_unlock(&damon_ops_lock);
|
| 85 | return err;
|
| 86 | }
|
| 87 |
|
| 88 | |
| 89 | * damon_select_ops() - Select a monitoring operations to use with the context.
|
| 90 | * @ctx: monitoring context to use the operations.
|
| 91 | * @id: id of the registered monitoring operations to select.
|
| 92 | *
|
| 93 | * This function finds registered monitoring operations set of @id and make
|
| 94 | * @ctx to use it.
|
| 95 | *
|
| 96 | * Return: 0 on success, negative error code otherwise.
|
| 97 |
|
| 98 | int damon_select_ops(struct damon_ctx *ctx, enum damon_ops_id id)
|
| 99 | {
|
| 100 | int err = 0;
|
| 101 |
|
| 102 | if (id >= NR_DAMON_OPS)
|
| 103 | return -EINVAL;
|
| 104 |
|
| 105 | mutex_lock(&damon_ops_lock);
|
| 106 | if (!__damon_is_registered_ops(id))
|
| 107 | err = -EINVAL;
|
| 108 | else
|
| 109 | ctx->ops = damon_registered_ops[id];
|
| 110 | mutex_unlock(&damon_ops_lock);
|
| 111 | return err;
|
| 112 | }
|
| 113 |
|
| 114 | struct damon_filter *damon_new_filter(enum damon_filter_type type,
|
| 115 | bool matching, bool allow)
|
| 116 | {
|
| 117 | struct damon_filter *filter;
|
| 118 |
|
| 119 | filter = kmalloc_obj(*filter);
|
| 120 | if (!filter)
|
| 121 | return NULL;
|
| 122 | filter->type = type;
|
| 123 | filter->matching = matching;
|
| 124 | filter->allow = allow;
|
| 125 | INIT_LIST_HEAD(&filter->list);
|
| 126 | return filter;
|
| 127 | }
|
| 128 |
|
| 129 | void damon_add_filter(struct damon_probe *p, struct damon_filter *f)
|
| 130 | {
|
| 131 | list_add_tail(&f->list, &p->filters);
|
| 132 | }
|
| 133 |
|
| 134 | static void damon_del_filter(struct damon_filter *f)
|
| 135 | {
|
| 136 | list_del(&f->list);
|
| 137 | }
|
| 138 |
|
| 139 | static void damon_free_filter(struct damon_filter *f)
|
| 140 | {
|
| 141 | kfree(f);
|
| 142 | }
|
| 143 |
|
| 144 | void damon_destroy_filter(struct damon_filter *f)
|
| 145 | {
|
| 146 | damon_del_filter(f);
|
| 147 | damon_free_filter(f);
|
| 148 | }
|
| 149 |
|
| 150 | static struct damon_filter *damon_nth_filter(int n, struct damon_probe *p)
|
| 151 | {
|
| 152 | struct damon_filter *f;
|
| 153 | int i = 0;
|
| 154 |
|
| 155 | damon_for_each_filter(f, p) {
|
| 156 | if (i++ == n)
|
| 157 | return f;
|
| 158 | }
|
| 159 | return NULL;
|
| 160 | }
|
| 161 |
|
| 162 | struct damon_probe *damon_new_probe(void)
|
| 163 | {
|
| 164 | struct damon_probe *p;
|
| 165 |
|
| 166 | p = kmalloc_obj(*p);
|
| 167 | if (!p)
|
| 168 | return NULL;
|
| 169 | p->weight = 0;
|
| 170 | INIT_LIST_HEAD(&p->filters);
|
| 171 | INIT_LIST_HEAD(&p->list);
|
| 172 | return p;
|
| 173 | }
|
| 174 |
|
| 175 | void damon_add_probe(struct damon_ctx *ctx, struct damon_probe *probe)
|
| 176 | {
|
| 177 | list_add_tail(&probe->list, &ctx->probes);
|
| 178 | }
|
| 179 |
|
| 180 | static void damon_del_probe(struct damon_probe *p)
|
| 181 | {
|
| 182 | list_del(&p->list);
|
| 183 | }
|
| 184 |
|
| 185 | static void damon_free_probe(struct damon_probe *p)
|
| 186 | {
|
| 187 | struct damon_filter *f, *next;
|
| 188 |
|
| 189 | damon_for_each_filter_safe(f, next, p)
|
| 190 | damon_free_filter(f);
|
| 191 | kfree(p);
|
| 192 | }
|
| 193 |
|
| 194 | static void damon_destroy_probe(struct damon_probe *p)
|
| 195 | {
|
| 196 | damon_del_probe(p);
|
| 197 | damon_free_probe(p);
|
| 198 | }
|
| 199 |
|
| 200 | static struct damon_probe *damon_nth_probe(int n, struct damon_ctx *ctx)
|
| 201 | {
|
| 202 | struct damon_probe *p;
|
| 203 | int i = 0;
|
| 204 |
|
| 205 | damon_for_each_probe(p, ctx) {
|
| 206 | if (i++ == n)
|
| 207 | return p;
|
| 208 | }
|
| 209 | return NULL;
|
| 210 | }
|
| 211 |
|
| 212 | static bool damon_has_probe_weights(struct damon_ctx *c)
|
| 213 | {
|
| 214 | struct damon_probe *p;
|
| 215 |
|
| 216 | damon_for_each_probe(p, c) {
|
| 217 | if (p->weight)
|
| 218 | return true;
|
| 219 | }
|
| 220 | return false;
|
| 221 | }
|
| 222 |
|
| 223 | |
| 224 | * damon_mvsum() - Returns pseudo moving sum value for a time window.
|
| 225 | * @current_nr: The value of the current aggregation window.
|
| 226 | * @last_nr: The value of the last aggregation window.
|
| 227 | * @left_window_bp: Left time of the current aggregation window.
|
| 228 | *
|
| 229 | * This function calculates a pseudo moving sum value of a counter that is
|
| 230 | * aggregated for each time window. @current_nr is the value of the counter
|
| 231 | * that aggregated so far (maybe not yet complete), from the beginning of the
|
| 232 | * current aggregation time window. @last_nr is the value of the counter that
|
| 233 | * has completely aggregated in the last aggregation time window.
|
| 234 | * @left_window_bp represents how much time is left for the current aggregation
|
| 235 | * time window in bp (1/10,000). For example, the aggregation time window is
|
| 236 | * for every 10 seconds and 7 seconds has passed since the beginning of the
|
| 237 | * current window, this parameter will be 3000 ((10 - 7) / 10 * 10000).
|
| 238 | *
|
| 239 | * The logic assumes the aggregation in the last phase was made in a single
|
| 240 | * speed. Based on the assumption, the value from the last window that needs
|
| 241 | * to be added to the current value is calculated as a portion of the last
|
| 242 | * value based on the remaining time window.
|
| 243 |
|
| 244 | static unsigned long damon_mvsum(unsigned long current_nr,
|
| 245 | unsigned long last_nr, unsigned long left_window_bp)
|
| 246 | {
|
| 247 | return current_nr + mult_frac(last_nr, left_window_bp, 10000);
|
| 248 | }
|
| 249 |
|
| 250 | |
| 251 | * damon_nr_accesses_mvsum() - Returns moving sum access frequency score.
|
| 252 | * @r: Region to get the access frequency of.
|
| 253 | * @ctx: DAMON context of @r.
|
| 254 | *
|
| 255 | * This function returns for how many sampling iterations in the last
|
| 256 | * aggregation interval (&damon_attrs->aggr_interval) the region was found to
|
| 257 | * be accessed. Hence the value can be interpreted as the relative access
|
| 258 | * frequency score of the region (@r). The value is calculated as a pseudo
|
| 259 | * moving sum, and hence it is not an exact value but just a best-effort
|
| 260 | * reasonable estimation.
|
| 261 | *
|
| 262 | * Return: the pseudo moving sum access frequency score.
|
| 263 |
|
| 264 | unsigned int damon_nr_accesses_mvsum(struct damon_region *r,
|
| 265 | struct damon_ctx *ctx)
|
| 266 | {
|
| 267 | unsigned long sample_interval, aggr_interval;
|
| 268 | unsigned long window_len, left_window, left_window_bp;
|
| 269 |
|
| 270 | sample_interval = ctx->attrs.sample_interval ? : 1;
|
| 271 | aggr_interval = ctx->attrs.aggr_interval ? : 1;
|
| 272 | window_len = aggr_interval / sample_interval;
|
| 273 | if (time_after_eq(ctx->passed_sample_intervals,
|
| 274 | ctx->next_aggregation_sis))
|
| 275 | left_window = 0;
|
| 276 | else
|
| 277 | left_window = ctx->next_aggregation_sis -
|
| 278 | ctx->passed_sample_intervals;
|
| 279 | left_window_bp = mult_frac(left_window, 10000, window_len);
|
| 280 |
|
| 281 | if (left_window_bp == 10000)
|
| 282 | return r->last_nr_accesses;
|
| 283 |
|
| 284 | return damon_mvsum(r->nr_accesses, r->last_nr_accesses,
|
| 285 | left_window_bp);
|
| 286 | }
|
| 287 |
|
| 288 | unsigned char damon_probe_hits_mvsum(int probe_idx, struct damon_region *r,
|
| 289 | struct damon_ctx *ctx)
|
| 290 | {
|
| 291 | unsigned long sample_interval, aggr_interval;
|
| 292 | unsigned long window_len, left_window, left_window_bp;
|
| 293 |
|
| 294 | sample_interval = ctx->attrs.sample_interval ? : 1;
|
| 295 | aggr_interval = ctx->attrs.aggr_interval ? : 1;
|
| 296 | window_len = aggr_interval / sample_interval;
|
| 297 | if (time_after_eq(ctx->passed_sample_intervals,
|
| 298 | ctx->next_aggregation_sis))
|
| 299 | left_window = 0;
|
| 300 | else
|
| 301 | left_window = ctx->next_aggregation_sis -
|
| 302 | ctx->passed_sample_intervals;
|
| 303 | left_window_bp = mult_frac(left_window, 10000, window_len);
|
| 304 |
|
| 305 | if (left_window_bp == 10000)
|
| 306 | return r->last_probe_hits[probe_idx];
|
| 307 |
|
| 308 | return damon_mvsum(r->probe_hits[probe_idx],
|
| 309 | r->last_probe_hits[probe_idx], left_window_bp);
|
| 310 | }
|
| 311 |
|
| 312 | #ifdef CONFIG_DAMON_DEBUG_SANITY
|
| 313 | static void damon_verify_new_region(unsigned long start, unsigned long end)
|
| 314 | {
|
| 315 | WARN_ONCE(start >= end, "start %lu >= end %lu\n", start, end);
|
| 316 | }
|
| 317 | #else
|
| 318 | static void damon_verify_new_region(unsigned long start, unsigned long end)
|
| 319 | {
|
| 320 | }
|
| 321 | #endif
|
| 322 |
|
| 323 | |
| 324 | * Construct a damon_region struct
|
| 325 | *
|
| 326 | * Returns the pointer to the new struct if success, or NULL otherwise
|
| 327 |
|
| 328 | struct damon_region *damon_new_region(unsigned long start, unsigned long end)
|
| 329 | {
|
| 330 | struct damon_region *region;
|
| 331 | int i;
|
| 332 |
|
| 333 | damon_verify_new_region(start, end);
|
| 334 | region = kmem_cache_alloc(damon_region_cache, GFP_KERNEL);
|
| 335 | if (!region)
|
| 336 | return NULL;
|
| 337 |
|
| 338 | region->ar.start = start;
|
| 339 | region->ar.end = end;
|
| 340 | region->nr_accesses = 0;
|
| 341 | for (i = 0; i < DAMON_MAX_PROBES; i++) {
|
| 342 | region->probe_hits[i] = 0;
|
| 343 | region->last_probe_hits[i] = 0;
|
| 344 | }
|
| 345 | INIT_LIST_HEAD(®ion->list);
|
| 346 |
|
| 347 | region->age = 0;
|
| 348 | region->last_nr_accesses = 0;
|
| 349 |
|
| 350 | return region;
|
| 351 | }
|
| 352 |
|
| 353 | static void damon_add_region(struct damon_region *r, struct damon_target *t)
|
| 354 | {
|
| 355 | list_add_tail(&r->list, &t->regions_list);
|
| 356 | t->nr_regions++;
|
| 357 | }
|
| 358 |
|
| 359 | |
| 360 | * Add a region between two other regions
|
| 361 |
|
| 362 | static inline void damon_insert_region(struct damon_region *r,
|
| 363 | struct damon_region *prev, struct damon_region *next,
|
| 364 | struct damon_target *t)
|
| 365 | {
|
| 366 | __list_add(&r->list, &prev->list, &next->list);
|
| 367 | t->nr_regions++;
|
| 368 | }
|
| 369 |
|
| 370 | #ifdef CONFIG_DAMON_DEBUG_SANITY
|
| 371 | static void damon_verify_del_region(struct damon_target *t)
|
| 372 | {
|
| 373 | WARN_ONCE(t->nr_regions == 0, "t->nr_regions == 0\n");
|
| 374 | }
|
| 375 | #else
|
| 376 | static void damon_verify_del_region(struct damon_target *t)
|
| 377 | {
|
| 378 | }
|
| 379 | #endif
|
| 380 |
|
| 381 | static void damon_del_region(struct damon_region *r, struct damon_target *t)
|
| 382 | {
|
| 383 | damon_verify_del_region(t);
|
| 384 |
|
| 385 | list_del(&r->list);
|
| 386 | t->nr_regions--;
|
| 387 | }
|
| 388 |
|
| 389 | static void damon_free_region(struct damon_region *r)
|
| 390 | {
|
| 391 | kmem_cache_free(damon_region_cache, r);
|
| 392 | }
|
| 393 |
|
| 394 | static void damon_destroy_region(struct damon_region *r,
|
| 395 | struct damon_target *t)
|
| 396 | {
|
| 397 | damon_del_region(r, t);
|
| 398 | damon_free_region(r);
|
| 399 | }
|
| 400 |
|
| 401 | static bool damon_is_last_region(struct damon_region *r,
|
| 402 | struct damon_target *t)
|
| 403 | {
|
| 404 | return list_is_last(&r->list, &t->regions_list);
|
| 405 | }
|
| 406 |
|
| 407 | |
| 408 | * damon_probe_hits_wsum() - Returns probe hits weighted sum of a region.
|
| 409 | * @r: region to get the weighted sum of.
|
| 410 | * @last: if the request is for last-window aggregated probe hits.
|
| 411 | * @ctx: context of &r.
|
| 412 | *
|
| 413 | * Return: the weighted sum of probe hits of the region.
|
| 414 |
|
| 415 | unsigned int damon_probe_hits_wsum(struct damon_region *r, bool last,
|
| 416 | struct damon_ctx *ctx)
|
| 417 | {
|
| 418 | struct damon_probe *probe;
|
| 419 | unsigned int sum = 0;
|
| 420 | int i = 0;
|
| 421 |
|
| 422 | damon_for_each_probe(probe, ctx) {
|
| 423 | if (last)
|
| 424 | sum += r->last_probe_hits[i++] * probe->weight;
|
| 425 | else
|
| 426 | sum += r->probe_hits[i++] * probe->weight;
|
| 427 | }
|
| 428 | return sum;
|
| 429 | }
|
| 430 |
|
| 431 | |
| 432 | * Check whether a region is intersecting an address range
|
| 433 | *
|
| 434 | * Returns true if it is.
|
| 435 |
|
| 436 | static bool damon_intersect(struct damon_region *r,
|
| 437 | struct damon_addr_range *re)
|
| 438 | {
|
| 439 | return !(r->ar.end <= re->start || re->end <= r->ar.start);
|
| 440 | }
|
| 441 |
|
| 442 | |
| 443 | * Fill holes in regions with new regions.
|
| 444 |
|
| 445 | static int damon_fill_regions_holes(struct damon_region *first,
|
| 446 | struct damon_region *last, struct damon_target *t)
|
| 447 | {
|
| 448 | struct damon_region *r = first;
|
| 449 |
|
| 450 | damon_for_each_region_from(r, t) {
|
| 451 | struct damon_region *next, *newr;
|
| 452 |
|
| 453 | if (r == last)
|
| 454 | break;
|
| 455 | next = damon_next_region(r);
|
| 456 | if (r->ar.end != next->ar.start) {
|
| 457 | newr = damon_new_region(r->ar.end, next->ar.start);
|
| 458 | if (!newr)
|
| 459 | return -ENOMEM;
|
| 460 | damon_insert_region(newr, r, next, t);
|
| 461 | }
|
| 462 | }
|
| 463 | return 0;
|
| 464 | }
|
| 465 |
|
| 466 | |
| 467 | * damon_set_regions() - Set regions of a target for given address ranges.
|
| 468 | * @t: the given target.
|
| 469 | * @ranges: array of new monitoring target ranges.
|
| 470 | * @nr_ranges: length of @ranges.
|
| 471 | * @min_region_sz: minimum region size.
|
| 472 | *
|
| 473 | * This function adds new regions to, or modify existing regions of a
|
| 474 | * monitoring target to fit in specific ranges.
|
| 475 | *
|
| 476 | * Return: 0 if success, or negative error code otherwise.
|
| 477 |
|
| 478 | int damon_set_regions(struct damon_target *t, struct damon_addr_range *ranges,
|
| 479 | unsigned int nr_ranges, unsigned long min_region_sz)
|
| 480 | {
|
| 481 | struct damon_region *r, *next;
|
| 482 | unsigned int i;
|
| 483 | unsigned long last_end;
|
| 484 | int err;
|
| 485 |
|
| 486 | for (i = 0; i < nr_ranges; i++) {
|
| 487 | unsigned long start, end;
|
| 488 |
|
| 489 | start = ALIGN_DOWN(ranges[i].start, min_region_sz);
|
| 490 | end = ALIGN(ranges[i].end, min_region_sz);
|
| 491 | if (start >= end)
|
| 492 | return -EINVAL;
|
| 493 | if (i > 0 && last_end > start)
|
| 494 | return -EINVAL;
|
| 495 | last_end = end;
|
| 496 | }
|
| 497 |
|
| 498 |
|
| 499 | damon_for_each_region_safe(r, next, t) {
|
| 500 | for (i = 0; i < nr_ranges; i++) {
|
| 501 | if (damon_intersect(r, &ranges[i]))
|
| 502 | break;
|
| 503 | }
|
| 504 | if (i == nr_ranges)
|
| 505 | damon_destroy_region(r, t);
|
| 506 | }
|
| 507 |
|
| 508 | if (!damon_nr_regions(t)) {
|
| 509 | for (i = 0; i < nr_ranges; i++) {
|
| 510 | r = damon_new_region(
|
| 511 | ALIGN_DOWN(ranges[i].start,
|
| 512 | min_region_sz),
|
| 513 | ALIGN(ranges[i].end, min_region_sz));
|
| 514 | if (!r)
|
| 515 | return -ENOMEM;
|
| 516 | damon_add_region(r, t);
|
| 517 | }
|
| 518 | return 0;
|
| 519 | }
|
| 520 |
|
| 521 | r = damon_first_region(t);
|
| 522 |
|
| 523 | for (i = 0; i < nr_ranges; i++) {
|
| 524 | struct damon_region *first = NULL, *last, *newr;
|
| 525 | struct damon_addr_range *range;
|
| 526 | bool insert_before_r = false;
|
| 527 |
|
| 528 | range = &ranges[i];
|
| 529 |
|
| 530 | damon_for_each_region_from(r, t) {
|
| 531 | if (damon_intersect(r, range)) {
|
| 532 | if (!first)
|
| 533 | first = r;
|
| 534 | last = r;
|
| 535 | }
|
| 536 | if (r->ar.start >= range->end) {
|
| 537 | insert_before_r = true;
|
| 538 | break;
|
| 539 | }
|
| 540 | }
|
| 541 | if (!first) {
|
| 542 |
|
| 543 | newr = damon_new_region(
|
| 544 | ALIGN_DOWN(range->start,
|
| 545 | min_region_sz),
|
| 546 | ALIGN(range->end, min_region_sz));
|
| 547 | if (!newr)
|
| 548 | return -ENOMEM;
|
| 549 | if (insert_before_r)
|
| 550 | damon_insert_region(newr, damon_prev_region(r),
|
| 551 | r, t);
|
| 552 | else
|
| 553 | damon_add_region(newr, t);
|
| 554 | } else {
|
| 555 |
|
| 556 | first->ar.start = ALIGN_DOWN(range->start,
|
| 557 | min_region_sz);
|
| 558 | last->ar.end = ALIGN(range->end, min_region_sz);
|
| 559 |
|
| 560 |
|
| 561 | err = damon_fill_regions_holes(first, last, t);
|
| 562 | if (err)
|
| 563 | return err;
|
| 564 | }
|
| 565 | }
|
| 566 | return 0;
|
| 567 | }
|
| 568 |
|
| 569 | struct damos_filter *damos_new_filter(enum damos_filter_type type,
|
| 570 | bool matching, bool allow)
|
| 571 | {
|
| 572 | struct damos_filter *filter;
|
| 573 |
|
| 574 | filter = kmalloc_obj(*filter);
|
| 575 | if (!filter)
|
| 576 | return NULL;
|
| 577 | filter->type = type;
|
| 578 | filter->matching = matching;
|
| 579 | filter->allow = allow;
|
| 580 | INIT_LIST_HEAD(&filter->list);
|
| 581 | return filter;
|
| 582 | }
|
| 583 |
|
| 584 | |
| 585 | * damos_filter_for_ops() - Return if the filter is ops-handled one.
|
| 586 | * @type: type of the filter.
|
| 587 | *
|
| 588 | * Return: true if the filter of @type needs to be handled by ops layer, false
|
| 589 | * otherwise.
|
| 590 |
|
| 591 | bool damos_filter_for_ops(enum damos_filter_type type)
|
| 592 | {
|
| 593 | switch (type) {
|
| 594 | case DAMOS_FILTER_TYPE_ADDR:
|
| 595 | case DAMOS_FILTER_TYPE_TARGET:
|
| 596 | return false;
|
| 597 | default:
|
| 598 | break;
|
| 599 | }
|
| 600 | return true;
|
| 601 | }
|
| 602 |
|
| 603 | void damos_add_filter(struct damos *s, struct damos_filter *f)
|
| 604 | {
|
| 605 | if (damos_filter_for_ops(f->type))
|
| 606 | list_add_tail(&f->list, &s->ops_filters);
|
| 607 | else
|
| 608 | list_add_tail(&f->list, &s->core_filters);
|
| 609 | }
|
| 610 |
|
| 611 | static void damos_del_filter(struct damos_filter *f)
|
| 612 | {
|
| 613 | list_del(&f->list);
|
| 614 | }
|
| 615 |
|
| 616 | static void damos_free_filter(struct damos_filter *f)
|
| 617 | {
|
| 618 | kfree(f);
|
| 619 | }
|
| 620 |
|
| 621 | void damos_destroy_filter(struct damos_filter *f)
|
| 622 | {
|
| 623 | damos_del_filter(f);
|
| 624 | damos_free_filter(f);
|
| 625 | }
|
| 626 |
|
| 627 | struct damos_quota_goal *damos_new_quota_goal(
|
| 628 | enum damos_quota_goal_metric metric,
|
| 629 | unsigned long target_value)
|
| 630 | {
|
| 631 | struct damos_quota_goal *goal;
|
| 632 |
|
| 633 | goal = kmalloc_obj(*goal);
|
| 634 | if (!goal)
|
| 635 | return NULL;
|
| 636 | goal->metric = metric;
|
| 637 | goal->target_value = target_value;
|
| 638 | INIT_LIST_HEAD(&goal->list);
|
| 639 | return goal;
|
| 640 | }
|
| 641 |
|
| 642 | void damos_add_quota_goal(struct damos_quota *q, struct damos_quota_goal *g)
|
| 643 | {
|
| 644 | list_add_tail(&g->list, &q->goals);
|
| 645 | }
|
| 646 |
|
| 647 | static void damos_del_quota_goal(struct damos_quota_goal *g)
|
| 648 | {
|
| 649 | list_del(&g->list);
|
| 650 | }
|
| 651 |
|
| 652 | static void damos_free_quota_goal(struct damos_quota_goal *g)
|
| 653 | {
|
| 654 | kfree(g);
|
| 655 | }
|
| 656 |
|
| 657 | void damos_destroy_quota_goal(struct damos_quota_goal *g)
|
| 658 | {
|
| 659 | damos_del_quota_goal(g);
|
| 660 | damos_free_quota_goal(g);
|
| 661 | }
|
| 662 |
|
| 663 | static bool damos_quota_goals_empty(struct damos_quota *q)
|
| 664 | {
|
| 665 | return list_empty(&q->goals);
|
| 666 | }
|
| 667 |
|
| 668 |
|
| 669 | static struct damos_quota *damos_quota_init(struct damos_quota *quota)
|
| 670 | {
|
| 671 | quota->esz = 0;
|
| 672 | quota->total_charged_sz = 0;
|
| 673 | quota->total_charged_ns = 0;
|
| 674 | quota->charged_sz = 0;
|
| 675 | quota->charged_from = 0;
|
| 676 | quota->charge_target_from = NULL;
|
| 677 | quota->charge_addr_from = 0;
|
| 678 | quota->esz_bp = 0;
|
| 679 | return quota;
|
| 680 | }
|
| 681 |
|
| 682 | struct damos *damon_new_scheme(struct damos_access_pattern *pattern,
|
| 683 | enum damos_action action,
|
| 684 | unsigned long apply_interval_us,
|
| 685 | struct damos_quota *quota,
|
| 686 | struct damos_watermarks *wmarks,
|
| 687 | int target_nid)
|
| 688 | {
|
| 689 | struct damos *scheme;
|
| 690 |
|
| 691 | scheme = kmalloc_obj(*scheme);
|
| 692 | if (!scheme)
|
| 693 | return NULL;
|
| 694 | scheme->pattern = *pattern;
|
| 695 | scheme->action = action;
|
| 696 | scheme->apply_interval_us = apply_interval_us;
|
| 697 | |
| 698 | * next_apply_sis will be set when kdamond starts. While kdamond is
|
| 699 | * running, it will also updated when it is added to the DAMON context,
|
| 700 | * or damon_attrs are updated.
|
| 701 |
|
| 702 | scheme->next_apply_sis = 0;
|
| 703 | scheme->walk_completed = false;
|
| 704 | INIT_LIST_HEAD(&scheme->core_filters);
|
| 705 | INIT_LIST_HEAD(&scheme->ops_filters);
|
| 706 | scheme->stat = (struct damos_stat){};
|
| 707 | scheme->max_nr_snapshots = 0;
|
| 708 | scheme->last_applied = NULL;
|
| 709 | INIT_LIST_HEAD(&scheme->list);
|
| 710 |
|
| 711 | scheme->quota = *(damos_quota_init(quota));
|
| 712 |
|
| 713 | INIT_LIST_HEAD(&scheme->quota.goals);
|
| 714 |
|
| 715 | scheme->wmarks = *wmarks;
|
| 716 | scheme->wmarks.activated = true;
|
| 717 |
|
| 718 | scheme->migrate_dests = (struct damos_migrate_dests){};
|
| 719 | scheme->target_nid = target_nid;
|
| 720 |
|
| 721 | return scheme;
|
| 722 | }
|
| 723 |
|
| 724 | static void damos_set_next_apply_sis(struct damos *s, struct damon_ctx *ctx)
|
| 725 | {
|
| 726 | unsigned long sample_interval = ctx->attrs.sample_interval ?
|
| 727 | ctx->attrs.sample_interval : 1;
|
| 728 | unsigned long apply_interval = s->apply_interval_us ?
|
| 729 | s->apply_interval_us : ctx->attrs.aggr_interval;
|
| 730 |
|
| 731 | s->next_apply_sis = ctx->passed_sample_intervals +
|
| 732 | apply_interval / sample_interval;
|
| 733 | }
|
| 734 |
|
| 735 | void damon_add_scheme(struct damon_ctx *ctx, struct damos *s)
|
| 736 | {
|
| 737 | list_add_tail(&s->list, &ctx->schemes);
|
| 738 | damos_set_next_apply_sis(s, ctx);
|
| 739 | }
|
| 740 |
|
| 741 | static void damon_del_scheme(struct damos *s)
|
| 742 | {
|
| 743 | list_del(&s->list);
|
| 744 | }
|
| 745 |
|
| 746 | static void damon_free_scheme(struct damos *s)
|
| 747 | {
|
| 748 | kfree(s);
|
| 749 | }
|
| 750 |
|
| 751 | void damon_destroy_scheme(struct damos *s)
|
| 752 | {
|
| 753 | struct damos_quota_goal *g, *g_next;
|
| 754 | struct damos_filter *f, *next;
|
| 755 |
|
| 756 | damos_for_each_quota_goal_safe(g, g_next, &s->quota)
|
| 757 | damos_destroy_quota_goal(g);
|
| 758 |
|
| 759 | damos_for_each_core_filter_safe(f, next, s)
|
| 760 | damos_destroy_filter(f);
|
| 761 |
|
| 762 | damos_for_each_ops_filter_safe(f, next, s)
|
| 763 | damos_destroy_filter(f);
|
| 764 |
|
| 765 | kfree(s->migrate_dests.node_id_arr);
|
| 766 | kfree(s->migrate_dests.weight_arr);
|
| 767 | damon_del_scheme(s);
|
| 768 | damon_free_scheme(s);
|
| 769 | }
|
| 770 |
|
| 771 | |
| 772 | * Construct a damon_target struct
|
| 773 | *
|
| 774 | * Returns the pointer to the new struct if success, or NULL otherwise
|
| 775 |
|
| 776 | struct damon_target *damon_new_target(void)
|
| 777 | {
|
| 778 | struct damon_target *t;
|
| 779 |
|
| 780 | t = kmalloc_obj(*t);
|
| 781 | if (!t)
|
| 782 | return NULL;
|
| 783 |
|
| 784 | t->pid = NULL;
|
| 785 | t->nr_regions = 0;
|
| 786 | INIT_LIST_HEAD(&t->regions_list);
|
| 787 | INIT_LIST_HEAD(&t->list);
|
| 788 | t->obsolete = false;
|
| 789 |
|
| 790 | return t;
|
| 791 | }
|
| 792 |
|
| 793 | void damon_add_target(struct damon_ctx *ctx, struct damon_target *t)
|
| 794 | {
|
| 795 | list_add_tail(&t->list, &ctx->adaptive_targets);
|
| 796 | }
|
| 797 |
|
| 798 | bool damon_targets_empty(struct damon_ctx *ctx)
|
| 799 | {
|
| 800 | return list_empty(&ctx->adaptive_targets);
|
| 801 | }
|
| 802 |
|
| 803 | static void damon_del_target(struct damon_target *t)
|
| 804 | {
|
| 805 | list_del(&t->list);
|
| 806 | }
|
| 807 |
|
| 808 | void damon_free_target(struct damon_target *t)
|
| 809 | {
|
| 810 | struct damon_region *r, *next;
|
| 811 |
|
| 812 | damon_for_each_region_safe(r, next, t)
|
| 813 | damon_free_region(r);
|
| 814 | kfree(t);
|
| 815 | }
|
| 816 |
|
| 817 | void damon_destroy_target(struct damon_target *t, struct damon_ctx *ctx)
|
| 818 | {
|
| 819 |
|
| 820 | if (ctx && ctx->ops.cleanup_target)
|
| 821 | ctx->ops.cleanup_target(t);
|
| 822 |
|
| 823 | damon_del_target(t);
|
| 824 | damon_free_target(t);
|
| 825 | }
|
| 826 |
|
| 827 | unsigned int damon_nr_regions(struct damon_target *t)
|
| 828 | {
|
| 829 | return t->nr_regions;
|
| 830 | }
|
| 831 |
|
| 832 | struct damon_ctx *damon_new_ctx(void)
|
| 833 | {
|
| 834 | struct damon_ctx *ctx;
|
| 835 |
|
| 836 | ctx = kzalloc_obj(*ctx);
|
| 837 | if (!ctx)
|
| 838 | return NULL;
|
| 839 |
|
| 840 | init_completion(&ctx->kdamond_started);
|
| 841 |
|
| 842 | ctx->attrs.sample_interval = 5 * 1000;
|
| 843 | ctx->attrs.aggr_interval = 100 * 1000;
|
| 844 | ctx->attrs.ops_update_interval = 60 * 1000 * 1000;
|
| 845 |
|
| 846 | ctx->passed_sample_intervals = 0;
|
| 847 |
|
| 848 | ctx->next_aggregation_sis = 0;
|
| 849 | ctx->next_ops_update_sis = 0;
|
| 850 |
|
| 851 | mutex_init(&ctx->kdamond_lock);
|
| 852 | INIT_LIST_HEAD(&ctx->call_controls);
|
| 853 | mutex_init(&ctx->call_controls_lock);
|
| 854 | mutex_init(&ctx->walk_control_lock);
|
| 855 |
|
| 856 | ctx->attrs.min_nr_regions = 10;
|
| 857 | ctx->attrs.max_nr_regions = 1000;
|
| 858 |
|
| 859 | INIT_LIST_HEAD(&ctx->probes);
|
| 860 |
|
| 861 | ctx->addr_unit = 1;
|
| 862 | ctx->min_region_sz = DAMON_MIN_REGION_SZ;
|
| 863 |
|
| 864 | INIT_LIST_HEAD(&ctx->adaptive_targets);
|
| 865 | INIT_LIST_HEAD(&ctx->schemes);
|
| 866 |
|
| 867 | prandom_seed_state(&ctx->rnd_state, get_random_u64());
|
| 868 |
|
| 869 | return ctx;
|
| 870 | }
|
| 871 |
|
| 872 | static void damon_destroy_targets(struct damon_ctx *ctx)
|
| 873 | {
|
| 874 | struct damon_target *t, *next_t;
|
| 875 |
|
| 876 | damon_for_each_target_safe(t, next_t, ctx)
|
| 877 | damon_destroy_target(t, ctx);
|
| 878 | }
|
| 879 |
|
| 880 | void damon_destroy_ctx(struct damon_ctx *ctx)
|
| 881 | {
|
| 882 | struct damos *s, *next_s;
|
| 883 | struct damon_probe *p, *next_p;
|
| 884 |
|
| 885 | damon_destroy_targets(ctx);
|
| 886 |
|
| 887 | damon_for_each_scheme_safe(s, next_s, ctx)
|
| 888 | damon_destroy_scheme(s);
|
| 889 |
|
| 890 | damon_for_each_probe_safe(p, next_p, ctx)
|
| 891 | damon_destroy_probe(p);
|
| 892 |
|
| 893 | kfree(ctx);
|
| 894 | }
|
| 895 |
|
| 896 | static bool damon_attrs_equals(const struct damon_attrs *attrs1,
|
| 897 | const struct damon_attrs *attrs2)
|
| 898 | {
|
| 899 | const struct damon_intervals_goal *ig1 = &attrs1->intervals_goal;
|
| 900 | const struct damon_intervals_goal *ig2 = &attrs2->intervals_goal;
|
| 901 |
|
| 902 | return attrs1->sample_interval == attrs2->sample_interval &&
|
| 903 | attrs1->aggr_interval == attrs2->aggr_interval &&
|
| 904 | attrs1->ops_update_interval == attrs2->ops_update_interval &&
|
| 905 | attrs1->min_nr_regions == attrs2->min_nr_regions &&
|
| 906 | attrs1->max_nr_regions == attrs2->max_nr_regions &&
|
| 907 | ig1->access_bp == ig2->access_bp &&
|
| 908 | ig1->aggrs == ig2->aggrs &&
|
| 909 | ig1->min_sample_us == ig2->min_sample_us &&
|
| 910 | ig1->max_sample_us == ig2->max_sample_us;
|
| 911 | }
|
| 912 |
|
| 913 | static unsigned int damon_age_for_new_attrs(unsigned int age,
|
| 914 | struct damon_attrs *old_attrs, struct damon_attrs *new_attrs)
|
| 915 | {
|
| 916 | return age * old_attrs->aggr_interval / new_attrs->aggr_interval;
|
| 917 | }
|
| 918 |
|
| 919 |
|
| 920 | static unsigned int damon_sample_bp_to_count(
|
| 921 | unsigned int bp, struct damon_attrs *attrs)
|
| 922 | {
|
| 923 | return bp * damon_nr_samples_per_aggr(attrs) / 10000;
|
| 924 | }
|
| 925 |
|
| 926 |
|
| 927 | static unsigned int damon_sample_count_to_bp(
|
| 928 | unsigned int count, struct damon_attrs *attrs)
|
| 929 | {
|
| 930 | return mult_frac(count, 10000, damon_nr_samples_per_aggr(attrs));
|
| 931 | }
|
| 932 |
|
| 933 | static unsigned int damon_nr_samples_for_new_attrs(unsigned int nr,
|
| 934 | struct damon_attrs *old_attrs, struct damon_attrs *new_attrs)
|
| 935 | {
|
| 936 | return damon_sample_bp_to_count(
|
| 937 | damon_sample_count_to_bp(nr, old_attrs), new_attrs);
|
| 938 | }
|
| 939 |
|
| 940 | static void damon_update_probe_hits(struct damon_region *r,
|
| 941 | struct damon_attrs *old_attrs, struct damon_attrs *new_attrs,
|
| 942 | bool aggregating, struct damon_ctx *ctx)
|
| 943 | {
|
| 944 | struct damon_probe *p;
|
| 945 | int i = 0;
|
| 946 |
|
| 947 | damon_for_each_probe(p, ctx) {
|
| 948 | r->last_probe_hits[i] = damon_nr_samples_for_new_attrs(
|
| 949 | r->last_probe_hits[i], old_attrs, new_attrs);
|
| 950 | if (!aggregating)
|
| 951 | r->probe_hits[i] = damon_nr_samples_for_new_attrs(
|
| 952 | r->probe_hits[i], old_attrs,
|
| 953 | new_attrs);
|
| 954 | else
|
| 955 | r->probe_hits[i] = 0;
|
| 956 | i++;
|
| 957 | }
|
| 958 | }
|
| 959 |
|
| 960 | static void damon_update_monitoring_result(struct damon_region *r,
|
| 961 | struct damon_attrs *old_attrs, struct damon_attrs *new_attrs,
|
| 962 | bool aggregating, struct damon_ctx *ctx)
|
| 963 | {
|
| 964 | damon_update_probe_hits(r, old_attrs, new_attrs, aggregating, ctx);
|
| 965 |
|
| 966 | r->last_nr_accesses = damon_nr_samples_for_new_attrs(
|
| 967 | r->last_nr_accesses, old_attrs, new_attrs);
|
| 968 | if (!aggregating)
|
| 969 | r->nr_accesses = damon_nr_samples_for_new_attrs(
|
| 970 | r->nr_accesses, old_attrs, new_attrs);
|
| 971 | else
|
| 972 | |
| 973 | * if this is called in the middle of the aggregation, reset
|
| 974 | * the aggregations we made so far for this aggregation
|
| 975 | * interval. In other words, make the status like
|
| 976 | * kdamond_reset_aggregated() is called.
|
| 977 |
|
| 978 | r->nr_accesses = 0;
|
| 979 | r->age = damon_age_for_new_attrs(r->age, old_attrs, new_attrs);
|
| 980 | }
|
| 981 |
|
| 982 | |
| 983 | * region->nr_accesses is the number of sampling intervals in the last
|
| 984 | * aggregation interval that access to the region has found, and region->age is
|
| 985 | * the number of aggregation intervals that its access pattern has maintained.
|
| 986 | * For the reason, the real meaning of the two fields depend on current
|
| 987 | * sampling interval and aggregation interval. This function updates
|
| 988 | * ->nr_accesses and ->age of given damon_ctx's regions for new damon_attrs.
|
| 989 |
|
| 990 | static void damon_update_monitoring_results(struct damon_ctx *ctx,
|
| 991 | struct damon_attrs *new_attrs, bool aggregating)
|
| 992 | {
|
| 993 | struct damon_attrs *old_attrs = &ctx->attrs;
|
| 994 | struct damon_target *t;
|
| 995 | struct damon_region *r;
|
| 996 |
|
| 997 |
|
| 998 | if (!old_attrs->sample_interval || !old_attrs->aggr_interval ||
|
| 999 | !new_attrs->sample_interval ||
|
| 1000 | !new_attrs->aggr_interval)
|
| 1001 | return;
|
| 1002 |
|
| 1003 | damon_for_each_target(t, ctx)
|
| 1004 | damon_for_each_region(r, t)
|
| 1005 | damon_update_monitoring_result(r, old_attrs, new_attrs,
|
| 1006 | aggregating, ctx);
|
| 1007 | }
|
| 1008 |
|
| 1009 | |
| 1010 | * damon_valid_intervals_goal() - return if the intervals goal of @attrs is
|
| 1011 | * valid.
|
| 1012 |
|
| 1013 | static bool damon_valid_intervals_goal(struct damon_attrs *attrs)
|
| 1014 | {
|
| 1015 | struct damon_intervals_goal *goal = &attrs->intervals_goal;
|
| 1016 |
|
| 1017 |
|
| 1018 | if (!goal->aggrs)
|
| 1019 | return true;
|
| 1020 | if (goal->min_sample_us > goal->max_sample_us)
|
| 1021 | return false;
|
| 1022 | if (attrs->sample_interval < goal->min_sample_us ||
|
| 1023 | goal->max_sample_us < attrs->sample_interval)
|
| 1024 | return false;
|
| 1025 | return true;
|
| 1026 | }
|
| 1027 |
|
| 1028 | |
| 1029 | * damon_set_attrs() - Set attributes for the monitoring.
|
| 1030 | * @ctx: monitoring context
|
| 1031 | * @attrs: monitoring attributes
|
| 1032 | *
|
| 1033 | * This function updates monitoring results and next monitoring/damos operation
|
| 1034 | * schedules. Because those are periodically updated by kdamond, this should
|
| 1035 | * be called from a safe contexts. Such contexts include damon_ctx setup time
|
| 1036 | * while the kdamond is not yet started, and inside of kdamond_fn().
|
| 1037 | *
|
| 1038 | * In detail, all DAMON API callers directly call this function for initial
|
| 1039 | * setup of damon_ctx before calling damon_start(). Some of the API callers
|
| 1040 | * also indirectly call this function via damon_call() -> damon_commit() for
|
| 1041 | * online parameters updates. Finally, kdamond_fn() itself use this for
|
| 1042 | * applying auto-tuned monitoring intervals.
|
| 1043 | *
|
| 1044 | * Every time interval is in micro-seconds.
|
| 1045 | *
|
| 1046 | * Return: 0 on success, negative error code otherwise.
|
| 1047 |
|
| 1048 | int damon_set_attrs(struct damon_ctx *ctx, struct damon_attrs *attrs)
|
| 1049 | {
|
| 1050 | unsigned long sample_interval = attrs->sample_interval ?
|
| 1051 | attrs->sample_interval : 1;
|
| 1052 | struct damos *s;
|
| 1053 | bool aggregating = ctx->passed_sample_intervals <
|
| 1054 | ctx->next_aggregation_sis;
|
| 1055 |
|
| 1056 | if (!damon_valid_intervals_goal(attrs))
|
| 1057 | return -EINVAL;
|
| 1058 |
|
| 1059 | if (attrs->min_nr_regions < 3)
|
| 1060 | return -EINVAL;
|
| 1061 | if (attrs->min_nr_regions > attrs->max_nr_regions)
|
| 1062 | return -EINVAL;
|
| 1063 | if (attrs->sample_interval > attrs->aggr_interval)
|
| 1064 | return -EINVAL;
|
| 1065 |
|
| 1066 |
|
| 1067 | if (!attrs->aggr_samples)
|
| 1068 | attrs->aggr_samples = attrs->aggr_interval / sample_interval;
|
| 1069 |
|
| 1070 | ctx->next_aggregation_sis = ctx->passed_sample_intervals +
|
| 1071 | attrs->aggr_interval / sample_interval;
|
| 1072 | ctx->next_ops_update_sis = ctx->passed_sample_intervals +
|
| 1073 | attrs->ops_update_interval / sample_interval;
|
| 1074 | |
| 1075 | * next_intervals_tune_sis will be updated inside kdamond_fn().
|
| 1076 |
|
| 1077 |
|
| 1078 | damon_update_monitoring_results(ctx, attrs, aggregating);
|
| 1079 | ctx->attrs = *attrs;
|
| 1080 |
|
| 1081 | damon_for_each_scheme(s, ctx)
|
| 1082 | damos_set_next_apply_sis(s, ctx);
|
| 1083 |
|
| 1084 | return 0;
|
| 1085 | }
|
| 1086 |
|
| 1087 | |
| 1088 | * damon_set_schemes() - Set data access monitoring based operation schemes.
|
| 1089 | * @ctx: monitoring context
|
| 1090 | * @schemes: array of the schemes
|
| 1091 | * @nr_schemes: number of entries in @schemes
|
| 1092 | *
|
| 1093 | * This function should not be called while the kdamond of the context is
|
| 1094 | * running.
|
| 1095 |
|
| 1096 | void damon_set_schemes(struct damon_ctx *ctx, struct damos **schemes,
|
| 1097 | ssize_t nr_schemes)
|
| 1098 | {
|
| 1099 | struct damos *s, *next;
|
| 1100 | ssize_t i;
|
| 1101 |
|
| 1102 | damon_for_each_scheme_safe(s, next, ctx)
|
| 1103 | damon_destroy_scheme(s);
|
| 1104 | for (i = 0; i < nr_schemes; i++)
|
| 1105 | damon_add_scheme(ctx, schemes[i]);
|
| 1106 | }
|
| 1107 |
|
| 1108 | static struct damos_quota_goal *damos_nth_quota_goal(
|
| 1109 | int n, struct damos_quota *q)
|
| 1110 | {
|
| 1111 | struct damos_quota_goal *goal;
|
| 1112 | int i = 0;
|
| 1113 |
|
| 1114 | damos_for_each_quota_goal(goal, q) {
|
| 1115 | if (i++ == n)
|
| 1116 | return goal;
|
| 1117 | }
|
| 1118 | return NULL;
|
| 1119 | }
|
| 1120 |
|
| 1121 | static void damos_commit_quota_goal_union(
|
| 1122 | struct damos_quota_goal *dst, struct damos_quota_goal *src)
|
| 1123 | {
|
| 1124 | switch (dst->metric) {
|
| 1125 | case DAMOS_QUOTA_NODE_MEM_USED_BP:
|
| 1126 | case DAMOS_QUOTA_NODE_MEM_FREE_BP:
|
| 1127 | dst->nid = src->nid;
|
| 1128 | break;
|
| 1129 | case DAMOS_QUOTA_NODE_MEMCG_USED_BP:
|
| 1130 | case DAMOS_QUOTA_NODE_MEMCG_FREE_BP:
|
| 1131 | dst->nid = src->nid;
|
| 1132 | dst->memcg_id = src->memcg_id;
|
| 1133 | break;
|
| 1134 | default:
|
| 1135 | break;
|
| 1136 | }
|
| 1137 | }
|
| 1138 |
|
| 1139 | static void damos_commit_quota_goal(
|
| 1140 | struct damos_quota_goal *dst, struct damos_quota_goal *src)
|
| 1141 | {
|
| 1142 | dst->metric = src->metric;
|
| 1143 | dst->target_value = src->target_value;
|
| 1144 | if (dst->metric == DAMOS_QUOTA_USER_INPUT)
|
| 1145 | dst->current_value = src->current_value;
|
| 1146 |
|
| 1147 | damos_commit_quota_goal_union(dst, src);
|
| 1148 | }
|
| 1149 |
|
| 1150 | |
| 1151 | * damos_commit_quota_goals() - Commit DAMOS quota goals to another quota.
|
| 1152 | * @dst: The commit destination DAMOS quota.
|
| 1153 | * @src: The commit source DAMOS quota.
|
| 1154 | *
|
| 1155 | * Copies user-specified parameters for quota goals from @src to @dst. Users
|
| 1156 | * should use this function for quota goals-level parameters update of running
|
| 1157 | * DAMON contexts, instead of manual in-place updates.
|
| 1158 | *
|
| 1159 | * This function should be called from parameters-update safe context, like
|
| 1160 | * damon_call().
|
| 1161 |
|
| 1162 | int damos_commit_quota_goals(struct damos_quota *dst, struct damos_quota *src)
|
| 1163 | {
|
| 1164 | struct damos_quota_goal *dst_goal, *next, *src_goal, *new_goal;
|
| 1165 | int i = 0, j = 0;
|
| 1166 |
|
| 1167 | damos_for_each_quota_goal_safe(dst_goal, next, dst) {
|
| 1168 | src_goal = damos_nth_quota_goal(i++, src);
|
| 1169 | if (src_goal)
|
| 1170 | damos_commit_quota_goal(dst_goal, src_goal);
|
| 1171 | else
|
| 1172 | damos_destroy_quota_goal(dst_goal);
|
| 1173 | }
|
| 1174 | damos_for_each_quota_goal_safe(src_goal, next, src) {
|
| 1175 | if (j++ < i)
|
| 1176 | continue;
|
| 1177 | new_goal = damos_new_quota_goal(
|
| 1178 | src_goal->metric, src_goal->target_value);
|
| 1179 | if (!new_goal)
|
| 1180 | return -ENOMEM;
|
| 1181 | damos_commit_quota_goal(new_goal, src_goal);
|
| 1182 | damos_add_quota_goal(dst, new_goal);
|
| 1183 | }
|
| 1184 | return 0;
|
| 1185 | }
|
| 1186 |
|
| 1187 | static int damos_commit_quota(struct damos_quota *dst, struct damos_quota *src)
|
| 1188 | {
|
| 1189 | int err;
|
| 1190 |
|
| 1191 | dst->reset_interval = src->reset_interval;
|
| 1192 | dst->ms = src->ms;
|
| 1193 | dst->sz = src->sz;
|
| 1194 | err = damos_commit_quota_goals(dst, src);
|
| 1195 | if (err)
|
| 1196 | return err;
|
| 1197 | dst->goal_tuner = src->goal_tuner;
|
| 1198 | dst->fail_charge_num = src->fail_charge_num;
|
| 1199 | dst->fail_charge_denom = src->fail_charge_denom;
|
| 1200 | dst->weight_sz = src->weight_sz;
|
| 1201 | dst->weight_nr_accesses = src->weight_nr_accesses;
|
| 1202 | dst->weight_age = src->weight_age;
|
| 1203 | return 0;
|
| 1204 | }
|
| 1205 |
|
| 1206 | static struct damos_filter *damos_nth_core_filter(int n, struct damos *s)
|
| 1207 | {
|
| 1208 | struct damos_filter *filter;
|
| 1209 | int i = 0;
|
| 1210 |
|
| 1211 | damos_for_each_core_filter(filter, s) {
|
| 1212 | if (i++ == n)
|
| 1213 | return filter;
|
| 1214 | }
|
| 1215 | return NULL;
|
| 1216 | }
|
| 1217 |
|
| 1218 | static struct damos_filter *damos_nth_ops_filter(int n, struct damos *s)
|
| 1219 | {
|
| 1220 | struct damos_filter *filter;
|
| 1221 | int i = 0;
|
| 1222 |
|
| 1223 | damos_for_each_ops_filter(filter, s) {
|
| 1224 | if (i++ == n)
|
| 1225 | return filter;
|
| 1226 | }
|
| 1227 | return NULL;
|
| 1228 | }
|
| 1229 |
|
| 1230 | static void damos_commit_filter_arg(
|
| 1231 | struct damos_filter *dst, struct damos_filter *src)
|
| 1232 | {
|
| 1233 | switch (dst->type) {
|
| 1234 | case DAMOS_FILTER_TYPE_MEMCG:
|
| 1235 | dst->memcg_id = src->memcg_id;
|
| 1236 | break;
|
| 1237 | case DAMOS_FILTER_TYPE_ADDR:
|
| 1238 | dst->addr_range = src->addr_range;
|
| 1239 | break;
|
| 1240 | case DAMOS_FILTER_TYPE_TARGET:
|
| 1241 | dst->target_idx = src->target_idx;
|
| 1242 | break;
|
| 1243 | case DAMOS_FILTER_TYPE_HUGEPAGE_SIZE:
|
| 1244 | dst->sz_range = src->sz_range;
|
| 1245 | break;
|
| 1246 | default:
|
| 1247 | break;
|
| 1248 | }
|
| 1249 | }
|
| 1250 |
|
| 1251 | static void damos_commit_filter(
|
| 1252 | struct damos_filter *dst, struct damos_filter *src)
|
| 1253 | {
|
| 1254 | dst->type = src->type;
|
| 1255 | dst->matching = src->matching;
|
| 1256 | dst->allow = src->allow;
|
| 1257 | damos_commit_filter_arg(dst, src);
|
| 1258 | }
|
| 1259 |
|
| 1260 | static int damos_commit_core_filters(struct damos *dst, struct damos *src)
|
| 1261 | {
|
| 1262 | struct damos_filter *dst_filter, *next, *src_filter, *new_filter;
|
| 1263 | int i = 0, j = 0;
|
| 1264 |
|
| 1265 | damos_for_each_core_filter_safe(dst_filter, next, dst) {
|
| 1266 | src_filter = damos_nth_core_filter(i++, src);
|
| 1267 | if (src_filter)
|
| 1268 | damos_commit_filter(dst_filter, src_filter);
|
| 1269 | else
|
| 1270 | damos_destroy_filter(dst_filter);
|
| 1271 | }
|
| 1272 |
|
| 1273 | damos_for_each_core_filter_safe(src_filter, next, src) {
|
| 1274 | if (j++ < i)
|
| 1275 | continue;
|
| 1276 |
|
| 1277 | new_filter = damos_new_filter(
|
| 1278 | src_filter->type, src_filter->matching,
|
| 1279 | src_filter->allow);
|
| 1280 | if (!new_filter)
|
| 1281 | return -ENOMEM;
|
| 1282 | damos_commit_filter_arg(new_filter, src_filter);
|
| 1283 | damos_add_filter(dst, new_filter);
|
| 1284 | }
|
| 1285 | return 0;
|
| 1286 | }
|
| 1287 |
|
| 1288 | static int damos_commit_ops_filters(struct damos *dst, struct damos *src)
|
| 1289 | {
|
| 1290 | struct damos_filter *dst_filter, *next, *src_filter, *new_filter;
|
| 1291 | int i = 0, j = 0;
|
| 1292 |
|
| 1293 | damos_for_each_ops_filter_safe(dst_filter, next, dst) {
|
| 1294 | src_filter = damos_nth_ops_filter(i++, src);
|
| 1295 | if (src_filter)
|
| 1296 | damos_commit_filter(dst_filter, src_filter);
|
| 1297 | else
|
| 1298 | damos_destroy_filter(dst_filter);
|
| 1299 | }
|
| 1300 |
|
| 1301 | damos_for_each_ops_filter_safe(src_filter, next, src) {
|
| 1302 | if (j++ < i)
|
| 1303 | continue;
|
| 1304 |
|
| 1305 | new_filter = damos_new_filter(
|
| 1306 | src_filter->type, src_filter->matching,
|
| 1307 | src_filter->allow);
|
| 1308 | if (!new_filter)
|
| 1309 | return -ENOMEM;
|
| 1310 | damos_commit_filter_arg(new_filter, src_filter);
|
| 1311 | damos_add_filter(dst, new_filter);
|
| 1312 | }
|
| 1313 | return 0;
|
| 1314 | }
|
| 1315 |
|
| 1316 | |
| 1317 | * damos_filters_default_reject() - decide whether to reject memory that didn't
|
| 1318 | * match with any given filter.
|
| 1319 | * @filters: Given DAMOS filters of a group.
|
| 1320 |
|
| 1321 | static bool damos_filters_default_reject(struct list_head *filters)
|
| 1322 | {
|
| 1323 | struct damos_filter *last_filter;
|
| 1324 |
|
| 1325 | if (list_empty(filters))
|
| 1326 | return false;
|
| 1327 | last_filter = list_last_entry(filters, struct damos_filter, list);
|
| 1328 | return last_filter->allow;
|
| 1329 | }
|
| 1330 |
|
| 1331 | static void damos_set_filters_default_reject(struct damos *s)
|
| 1332 | {
|
| 1333 | if (!list_empty(&s->ops_filters))
|
| 1334 | s->core_filters_default_reject = false;
|
| 1335 | else
|
| 1336 | s->core_filters_default_reject =
|
| 1337 | damos_filters_default_reject(&s->core_filters);
|
| 1338 | s->ops_filters_default_reject =
|
| 1339 | damos_filters_default_reject(&s->ops_filters);
|
| 1340 | }
|
| 1341 |
|
| 1342 | static bool damon_valid_probe_params(struct damon_ctx *ctx)
|
| 1343 | {
|
| 1344 | unsigned long sample_interval;
|
| 1345 | unsigned char max_probe_hits;
|
| 1346 | struct damon_probe *probe;
|
| 1347 | unsigned int wsum, wsum_to_add;
|
| 1348 |
|
| 1349 | if (!damon_has_probe_weights(ctx))
|
| 1350 | return true;
|
| 1351 |
|
| 1352 | sample_interval = ctx->attrs.sample_interval ? : 1;
|
| 1353 | if (ctx->attrs.aggr_interval / sample_interval > U8_MAX)
|
| 1354 | return false;
|
| 1355 |
|
| 1356 |
|
| 1357 | max_probe_hits = damon_nr_samples_per_aggr(&ctx->attrs);
|
| 1358 | wsum = 0;
|
| 1359 | damon_for_each_probe(probe, ctx) {
|
| 1360 | if (probe->weight > UINT_MAX / max_probe_hits)
|
| 1361 | return false;
|
| 1362 | wsum_to_add = probe->weight * max_probe_hits;
|
| 1363 | if (UINT_MAX - wsum < wsum_to_add)
|
| 1364 | return false;
|
| 1365 | wsum += wsum_to_add;
|
| 1366 | }
|
| 1367 | return true;
|
| 1368 | }
|
| 1369 |
|
| 1370 | |
| 1371 | * damos_commit_dests() - Copy migration destinations from @src to @dst.
|
| 1372 | * @dst: Destination structure to update.
|
| 1373 | * @src: Source structure to copy from.
|
| 1374 | *
|
| 1375 | * If the number of destinations has changed, the old arrays in @dst are freed
|
| 1376 | * and new ones are allocated. On success, @dst contains a full copy of
|
| 1377 | * @src's arrays and count.
|
| 1378 | *
|
| 1379 | * On allocation failure, @dst is left in a partially torn-down state: its
|
| 1380 | * arrays may be NULL and @nr_dests may not reflect the actual allocation
|
| 1381 | * sizes. The structure remains safe to deallocate via damon_destroy_scheme(),
|
| 1382 | * but callers must not reuse @dst for further commits — it should be
|
| 1383 | * discarded.
|
| 1384 | *
|
| 1385 | * Return: 0 on success, -ENOMEM on allocation failure.
|
| 1386 |
|
| 1387 | static int damos_commit_dests(struct damos_migrate_dests *dst,
|
| 1388 | struct damos_migrate_dests *src)
|
| 1389 | {
|
| 1390 | if (dst->nr_dests != src->nr_dests) {
|
| 1391 | kfree(dst->node_id_arr);
|
| 1392 | kfree(dst->weight_arr);
|
| 1393 |
|
| 1394 | dst->node_id_arr = kmalloc_array(src->nr_dests,
|
| 1395 | sizeof(*dst->node_id_arr), GFP_KERNEL);
|
| 1396 | if (!dst->node_id_arr) {
|
| 1397 | dst->weight_arr = NULL;
|
| 1398 | return -ENOMEM;
|
| 1399 | }
|
| 1400 |
|
| 1401 | dst->weight_arr = kmalloc_array(src->nr_dests,
|
| 1402 | sizeof(*dst->weight_arr), GFP_KERNEL);
|
| 1403 | if (!dst->weight_arr) {
|
| 1404 |
|
| 1405 | return -ENOMEM;
|
| 1406 | }
|
| 1407 | }
|
| 1408 |
|
| 1409 | dst->nr_dests = src->nr_dests;
|
| 1410 | for (int i = 0; i < src->nr_dests; i++) {
|
| 1411 | dst->node_id_arr[i] = src->node_id_arr[i];
|
| 1412 | dst->weight_arr[i] = src->weight_arr[i];
|
| 1413 | }
|
| 1414 |
|
| 1415 | return 0;
|
| 1416 | }
|
| 1417 |
|
| 1418 | static int damos_commit_filters(struct damos *dst, struct damos *src)
|
| 1419 | {
|
| 1420 | int err;
|
| 1421 |
|
| 1422 | err = damos_commit_core_filters(dst, src);
|
| 1423 | if (err)
|
| 1424 | return err;
|
| 1425 | err = damos_commit_ops_filters(dst, src);
|
| 1426 | if (err)
|
| 1427 | return err;
|
| 1428 | damos_set_filters_default_reject(dst);
|
| 1429 | return 0;
|
| 1430 | }
|
| 1431 |
|
| 1432 | static struct damos *damon_nth_scheme(int n, struct damon_ctx *ctx)
|
| 1433 | {
|
| 1434 | struct damos *s;
|
| 1435 | int i = 0;
|
| 1436 |
|
| 1437 | damon_for_each_scheme(s, ctx) {
|
| 1438 | if (i++ == n)
|
| 1439 | return s;
|
| 1440 | }
|
| 1441 | return NULL;
|
| 1442 | }
|
| 1443 |
|
| 1444 | static int damos_commit(struct damos *dst, struct damos *src)
|
| 1445 | {
|
| 1446 | int err;
|
| 1447 |
|
| 1448 | dst->pattern = src->pattern;
|
| 1449 | dst->action = src->action;
|
| 1450 | dst->apply_interval_us = src->apply_interval_us;
|
| 1451 |
|
| 1452 | err = damos_commit_quota(&dst->quota, &src->quota);
|
| 1453 | if (err)
|
| 1454 | return err;
|
| 1455 |
|
| 1456 | dst->wmarks = src->wmarks;
|
| 1457 | dst->target_nid = src->target_nid;
|
| 1458 |
|
| 1459 | err = damos_commit_dests(&dst->migrate_dests, &src->migrate_dests);
|
| 1460 | if (err)
|
| 1461 | return err;
|
| 1462 |
|
| 1463 | err = damos_commit_filters(dst, src);
|
| 1464 | if (err)
|
| 1465 | return err;
|
| 1466 |
|
| 1467 | dst->max_nr_snapshots = src->max_nr_snapshots;
|
| 1468 | return 0;
|
| 1469 | }
|
| 1470 |
|
| 1471 | static int damon_commit_schemes(struct damon_ctx *dst, struct damon_ctx *src)
|
| 1472 | {
|
| 1473 | struct damos *dst_scheme, *next, *src_scheme, *new_scheme;
|
| 1474 | int i = 0, j = 0, err;
|
| 1475 |
|
| 1476 | damon_for_each_scheme_safe(dst_scheme, next, dst) {
|
| 1477 | src_scheme = damon_nth_scheme(i++, src);
|
| 1478 | if (src_scheme) {
|
| 1479 | err = damos_commit(dst_scheme, src_scheme);
|
| 1480 | if (err)
|
| 1481 | return err;
|
| 1482 | } else {
|
| 1483 | damon_destroy_scheme(dst_scheme);
|
| 1484 | }
|
| 1485 | }
|
| 1486 |
|
| 1487 | damon_for_each_scheme_safe(src_scheme, next, src) {
|
| 1488 | if (j++ < i)
|
| 1489 | continue;
|
| 1490 | new_scheme = damon_new_scheme(&src_scheme->pattern,
|
| 1491 | src_scheme->action,
|
| 1492 | src_scheme->apply_interval_us,
|
| 1493 | &src_scheme->quota, &src_scheme->wmarks,
|
| 1494 | NUMA_NO_NODE);
|
| 1495 | if (!new_scheme)
|
| 1496 | return -ENOMEM;
|
| 1497 | err = damos_commit(new_scheme, src_scheme);
|
| 1498 | if (err) {
|
| 1499 | damon_destroy_scheme(new_scheme);
|
| 1500 | return err;
|
| 1501 | }
|
| 1502 | damon_add_scheme(dst, new_scheme);
|
| 1503 | }
|
| 1504 | return 0;
|
| 1505 | }
|
| 1506 |
|
| 1507 | static struct damon_target *damon_nth_target(int n, struct damon_ctx *ctx)
|
| 1508 | {
|
| 1509 | struct damon_target *t;
|
| 1510 | int i = 0;
|
| 1511 |
|
| 1512 | damon_for_each_target(t, ctx) {
|
| 1513 | if (i++ == n)
|
| 1514 | return t;
|
| 1515 | }
|
| 1516 | return NULL;
|
| 1517 | }
|
| 1518 |
|
| 1519 | |
| 1520 | * The caller should ensure the regions of @src are
|
| 1521 | * 1. valid (end >= src) and
|
| 1522 | * 2. sorted by starting address.
|
| 1523 | *
|
| 1524 | * If @src has no region, @dst keeps current regions.
|
| 1525 |
|
| 1526 | static int damon_commit_target_regions(struct damon_target *dst,
|
| 1527 | struct damon_target *src, unsigned long src_min_region_sz)
|
| 1528 | {
|
| 1529 | struct damon_region *src_region, *prev = NULL;
|
| 1530 | struct damon_addr_range *ranges;
|
| 1531 | int i = 0, err;
|
| 1532 |
|
| 1533 | damon_for_each_region(src_region, src) {
|
| 1534 | if (!prev || prev->ar.end != src_region->ar.start)
|
| 1535 | i++;
|
| 1536 | prev = src_region;
|
| 1537 | }
|
| 1538 | if (!i)
|
| 1539 | return 0;
|
| 1540 |
|
| 1541 | ranges = kvmalloc_objs(*ranges, i, GFP_KERNEL | __GFP_NOWARN);
|
| 1542 | if (!ranges)
|
| 1543 | return -ENOMEM;
|
| 1544 | prev = NULL;
|
| 1545 | i = 0;
|
| 1546 | damon_for_each_region(src_region, src) {
|
| 1547 | if (!prev) {
|
| 1548 | ranges[i].start = src_region->ar.start;
|
| 1549 | } else if (prev->ar.end != src_region->ar.start) {
|
| 1550 | ranges[i++].end = prev->ar.end;
|
| 1551 | ranges[i].start = src_region->ar.start;
|
| 1552 | }
|
| 1553 | prev = src_region;
|
| 1554 | }
|
| 1555 | ranges[i++].end = damon_last_region(src)->ar.end;
|
| 1556 | err = damon_set_regions(dst, ranges, i, src_min_region_sz);
|
| 1557 | kvfree(ranges);
|
| 1558 | return err;
|
| 1559 | }
|
| 1560 |
|
| 1561 | static int damon_commit_target(
|
| 1562 | struct damon_target *dst, bool dst_has_pid,
|
| 1563 | struct damon_target *src, bool src_has_pid,
|
| 1564 | unsigned long src_min_region_sz)
|
| 1565 | {
|
| 1566 | int err;
|
| 1567 |
|
| 1568 | err = damon_commit_target_regions(dst, src, src_min_region_sz);
|
| 1569 | if (err)
|
| 1570 | return err;
|
| 1571 | if (dst_has_pid)
|
| 1572 | put_pid(dst->pid);
|
| 1573 | if (src_has_pid)
|
| 1574 | get_pid(src->pid);
|
| 1575 | dst->pid = src->pid;
|
| 1576 | return 0;
|
| 1577 | }
|
| 1578 |
|
| 1579 | |
| 1580 | * damon_revert_target_commits() - revert unsuccessful target commits.
|
| 1581 | * @dst: Commit destination context
|
| 1582 | * @failed: Commit failed destination target
|
| 1583 | * @src: Commit source context
|
| 1584 | *
|
| 1585 | * Revert target states that changed by damon_commit_target(), and cannot be
|
| 1586 | * cleaned up by the destination context's ops.cleanup_target().
|
| 1587 |
|
| 1588 | static void damon_revert_target_commits(struct damon_ctx *dst,
|
| 1589 | struct damon_target *failed, struct damon_ctx *src)
|
| 1590 | {
|
| 1591 | struct damon_target *target;
|
| 1592 |
|
| 1593 | if (!damon_target_has_pid(src))
|
| 1594 | return;
|
| 1595 | if (dst->ops.cleanup_target)
|
| 1596 | return;
|
| 1597 | damon_for_each_target(target, dst) {
|
| 1598 | if (target == failed)
|
| 1599 | return;
|
| 1600 | put_pid(target->pid);
|
| 1601 | }
|
| 1602 | }
|
| 1603 |
|
| 1604 | static int damon_commit_targets(
|
| 1605 | struct damon_ctx *dst, struct damon_ctx *src)
|
| 1606 | {
|
| 1607 | struct damon_target *dst_target, *next, *src_target, *new_target;
|
| 1608 | struct damon_target *failed;
|
| 1609 | int i = 0, j = 0, err;
|
| 1610 |
|
| 1611 | damon_for_each_target_safe(dst_target, next, dst) {
|
| 1612 | src_target = damon_nth_target(i++, src);
|
| 1613 | |
| 1614 | * If src target is obsolete, do not commit the parameters to
|
| 1615 | * the dst target, and further remove the dst target.
|
| 1616 |
|
| 1617 | if (src_target && !src_target->obsolete) {
|
| 1618 | err = damon_commit_target(
|
| 1619 | dst_target, damon_target_has_pid(dst),
|
| 1620 | src_target, damon_target_has_pid(src),
|
| 1621 | src->min_region_sz);
|
| 1622 | if (err) {
|
| 1623 | failed = dst_target;
|
| 1624 | goto out;
|
| 1625 | }
|
| 1626 | } else {
|
| 1627 | struct damos *s;
|
| 1628 |
|
| 1629 | damon_destroy_target(dst_target, dst);
|
| 1630 | damon_for_each_scheme(s, dst) {
|
| 1631 | if (s->quota.charge_target_from == dst_target) {
|
| 1632 | s->quota.charge_target_from = NULL;
|
| 1633 | s->quota.charge_addr_from = 0;
|
| 1634 | }
|
| 1635 | }
|
| 1636 | }
|
| 1637 | }
|
| 1638 |
|
| 1639 | failed = NULL;
|
| 1640 | damon_for_each_target_safe(src_target, next, src) {
|
| 1641 | if (j++ < i)
|
| 1642 | continue;
|
| 1643 |
|
| 1644 | if (src_target->obsolete) {
|
| 1645 | err = -EINVAL;
|
| 1646 | goto out;
|
| 1647 | }
|
| 1648 | new_target = damon_new_target();
|
| 1649 | if (!new_target) {
|
| 1650 | err = -ENOMEM;
|
| 1651 | goto out;
|
| 1652 | }
|
| 1653 | err = damon_commit_target(new_target, false,
|
| 1654 | src_target, damon_target_has_pid(src),
|
| 1655 | src->min_region_sz);
|
| 1656 | if (err) {
|
| 1657 | damon_destroy_target(new_target, NULL);
|
| 1658 | goto out;
|
| 1659 | }
|
| 1660 | damon_add_target(dst, new_target);
|
| 1661 | }
|
| 1662 | return 0;
|
| 1663 |
|
| 1664 | out:
|
| 1665 | damon_revert_target_commits(dst, failed, src);
|
| 1666 | return err;
|
| 1667 | }
|
| 1668 |
|
| 1669 | static void damon_commit_filter(struct damon_filter *dst,
|
| 1670 | struct damon_filter *src)
|
| 1671 | {
|
| 1672 | dst->type = src->type;
|
| 1673 | dst->matching = src->matching;
|
| 1674 | dst->allow = src->allow;
|
| 1675 | switch (dst->type) {
|
| 1676 | case DAMON_FILTER_TYPE_MEMCG:
|
| 1677 | dst->memcg_id = src->memcg_id;
|
| 1678 | break;
|
| 1679 | default:
|
| 1680 | break;
|
| 1681 | }
|
| 1682 | }
|
| 1683 |
|
| 1684 | static int damon_commit_filters(struct damon_probe *dst,
|
| 1685 | struct damon_probe *src)
|
| 1686 | {
|
| 1687 | struct damon_filter *dst_filter, *next, *src_filter, *new_filter;
|
| 1688 | int i = 0, j = 0;
|
| 1689 |
|
| 1690 | damon_for_each_filter_safe(dst_filter, next, dst) {
|
| 1691 | src_filter = damon_nth_filter(i++, src);
|
| 1692 | if (src_filter)
|
| 1693 | damon_commit_filter(dst_filter, src_filter);
|
| 1694 | else
|
| 1695 | damon_destroy_filter(dst_filter);
|
| 1696 | }
|
| 1697 |
|
| 1698 | damon_for_each_filter_safe(src_filter, next, src) {
|
| 1699 | if (j++ < i)
|
| 1700 | continue;
|
| 1701 |
|
| 1702 | new_filter = damon_new_filter(src_filter->type,
|
| 1703 | src_filter->matching, src_filter->allow);
|
| 1704 | if (!new_filter)
|
| 1705 | return -ENOMEM;
|
| 1706 | switch (src_filter->type) {
|
| 1707 | case DAMON_FILTER_TYPE_MEMCG:
|
| 1708 | new_filter->memcg_id = src_filter->memcg_id;
|
| 1709 | break;
|
| 1710 | default:
|
| 1711 | break;
|
| 1712 | }
|
| 1713 | damon_add_filter(dst, new_filter);
|
| 1714 | }
|
| 1715 | return 0;
|
| 1716 | }
|
| 1717 |
|
| 1718 | static int damon_commit_probes(struct damon_ctx *dst, struct damon_ctx *src)
|
| 1719 | {
|
| 1720 | struct damon_probe *dst_probe, *next, *src_probe, *new_probe;
|
| 1721 | int i = 0, j = 0, err;
|
| 1722 |
|
| 1723 | damon_for_each_probe_safe(dst_probe, next, dst) {
|
| 1724 | src_probe = damon_nth_probe(i++, src);
|
| 1725 | if (src_probe) {
|
| 1726 | dst_probe->weight = src_probe->weight;
|
| 1727 | err = damon_commit_filters(dst_probe, src_probe);
|
| 1728 | if (err)
|
| 1729 | return err;
|
| 1730 | } else {
|
| 1731 | damon_destroy_probe(dst_probe);
|
| 1732 | }
|
| 1733 | }
|
| 1734 |
|
| 1735 | damon_for_each_probe_safe(src_probe, next, src) {
|
| 1736 | if (j++ < i)
|
| 1737 | continue;
|
| 1738 |
|
| 1739 | new_probe = damon_new_probe();
|
| 1740 | if (!new_probe)
|
| 1741 | return -ENOMEM;
|
| 1742 | damon_add_probe(dst, new_probe);
|
| 1743 | new_probe->weight = src_probe->weight;
|
| 1744 | err = damon_commit_filters(new_probe, src_probe);
|
| 1745 | if (err)
|
| 1746 | return err;
|
| 1747 | }
|
| 1748 | return 0;
|
| 1749 | }
|
| 1750 |
|
| 1751 | static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
|
| 1752 | {
|
| 1753 | int err;
|
| 1754 | struct damos *scheme;
|
| 1755 | struct damos_quota_goal *goal;
|
| 1756 |
|
| 1757 | dst->maybe_corrupted = true;
|
| 1758 | if (!is_power_of_2(src->min_region_sz))
|
| 1759 | return -EINVAL;
|
| 1760 |
|
| 1761 |
|
| 1762 | if (src->ops.id != DAMON_OPS_PADDR) {
|
| 1763 | damon_for_each_scheme(scheme, src) {
|
| 1764 | struct damos_quota *quota = &scheme->quota;
|
| 1765 |
|
| 1766 | damos_for_each_quota_goal(goal, quota) {
|
| 1767 | if (goal->metric ==
|
| 1768 | DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP)
|
| 1769 | return -EINVAL;
|
| 1770 | }
|
| 1771 | }
|
| 1772 | }
|
| 1773 |
|
| 1774 | if (!damon_valid_probe_params(src))
|
| 1775 | return -EINVAL;
|
| 1776 |
|
| 1777 | err = damon_commit_schemes(dst, src);
|
| 1778 | if (err)
|
| 1779 | return err;
|
| 1780 | err = damon_commit_targets(dst, src);
|
| 1781 | if (err)
|
| 1782 | return err;
|
| 1783 | |
| 1784 | * schemes and targets should be updated first, since
|
| 1785 | * 1. damon_set_attrs() updates monitoring results of targets and
|
| 1786 | * next_apply_sis of schemes, and
|
| 1787 | * 2. ops update should be done after pid handling is done (target
|
| 1788 | * committing require putting pids).
|
| 1789 |
|
| 1790 | if (!damon_attrs_equals(&dst->attrs, &src->attrs)) {
|
| 1791 | err = damon_set_attrs(dst, &src->attrs);
|
| 1792 | if (err) {
|
| 1793 | damon_revert_target_commits(dst, NULL, src);
|
| 1794 | return err;
|
| 1795 | }
|
| 1796 | }
|
| 1797 | dst->pause = src->pause;
|
| 1798 | dst->ops = src->ops;
|
| 1799 | err = damon_commit_probes(dst, src);
|
| 1800 | if (err)
|
| 1801 | return err;
|
| 1802 | dst->addr_unit = src->addr_unit;
|
| 1803 | dst->min_region_sz = src->min_region_sz;
|
| 1804 |
|
| 1805 | dst->maybe_corrupted = false;
|
| 1806 | return 0;
|
| 1807 | }
|
| 1808 |
|
| 1809 | static struct damon_ctx *damon_new_test_ctx(struct damon_ctx *dst)
|
| 1810 | {
|
| 1811 | struct damon_ctx *test_ctx;
|
| 1812 | int err;
|
| 1813 |
|
| 1814 | test_ctx = damon_new_ctx();
|
| 1815 | if (!test_ctx)
|
| 1816 | return NULL;
|
| 1817 | err = __damon_commit_ctx(test_ctx, dst);
|
| 1818 | if (err) {
|
| 1819 | damon_destroy_ctx(test_ctx);
|
| 1820 | return NULL;
|
| 1821 | }
|
| 1822 | return test_ctx;
|
| 1823 | }
|
| 1824 |
|
| 1825 | |
| 1826 | * damon_commit_ctx() - Commit parameters of a DAMON context to another.
|
| 1827 | * @dst: The commit destination DAMON context.
|
| 1828 | * @src: The commit source DAMON context.
|
| 1829 | *
|
| 1830 | * This function copies user-specified parameters from @src to @dst and update
|
| 1831 | * the internal status and results accordingly. Users should use this function
|
| 1832 | * for context-level parameters update of running context, instead of manual
|
| 1833 | * in-place updates.
|
| 1834 | *
|
| 1835 | * This function should be called from parameters-update safe context, like
|
| 1836 | * damon_call().
|
| 1837 |
|
| 1838 | int damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src)
|
| 1839 | {
|
| 1840 | struct damon_ctx *test_ctx;
|
| 1841 | int err;
|
| 1842 |
|
| 1843 | test_ctx = damon_new_test_ctx(dst);
|
| 1844 | if (!test_ctx)
|
| 1845 | return -ENOMEM;
|
| 1846 | err = __damon_commit_ctx(test_ctx, src);
|
| 1847 | if (err)
|
| 1848 | goto out;
|
| 1849 | err = __damon_commit_ctx(dst, src);
|
| 1850 | out:
|
| 1851 | damon_destroy_ctx(test_ctx);
|
| 1852 | return err;
|
| 1853 | }
|
| 1854 |
|
| 1855 | |
| 1856 | * damon_nr_running_ctxs() - Return number of currently running contexts.
|
| 1857 |
|
| 1858 | int damon_nr_running_ctxs(void)
|
| 1859 | {
|
| 1860 | int nr_ctxs;
|
| 1861 |
|
| 1862 | mutex_lock(&damon_lock);
|
| 1863 | nr_ctxs = nr_running_ctxs;
|
| 1864 | mutex_unlock(&damon_lock);
|
| 1865 |
|
| 1866 | return nr_ctxs;
|
| 1867 | }
|
| 1868 |
|
| 1869 |
|
| 1870 | static unsigned long damon_region_sz_limit(struct damon_ctx *ctx)
|
| 1871 | {
|
| 1872 | struct damon_target *t;
|
| 1873 | struct damon_region *r;
|
| 1874 | unsigned long sz = 0;
|
| 1875 |
|
| 1876 | damon_for_each_target(t, ctx) {
|
| 1877 | damon_for_each_region(r, t)
|
| 1878 | sz += damon_sz_region(r);
|
| 1879 | }
|
| 1880 |
|
| 1881 | if (ctx->attrs.min_nr_regions)
|
| 1882 | sz /= ctx->attrs.min_nr_regions;
|
| 1883 | if (sz < ctx->min_region_sz)
|
| 1884 | sz = ctx->min_region_sz;
|
| 1885 |
|
| 1886 | return sz;
|
| 1887 | }
|
| 1888 |
|
| 1889 | static int damon_split_region_at(struct damon_target *t,
|
| 1890 | struct damon_region *r, unsigned long sz_r);
|
| 1891 |
|
| 1892 | |
| 1893 | * damon_apply_min_nr_regions() - Make effect of min_nr_regions parameter.
|
| 1894 | * @ctx: monitoring context.
|
| 1895 | *
|
| 1896 | * This function implement min_nr_regions (minimum number of damon_region
|
| 1897 | * objects in the given monitoring context) behavior. It first calculates
|
| 1898 | * maximum size of each region for enforcing the min_nr_regions as total size
|
| 1899 | * of the regions divided by the min_nr_regions. After that, this function
|
| 1900 | * splits regions to ensure all regions are equal to or smaller than the size
|
| 1901 | * limit. Finally, this function returns the maximum size limit.
|
| 1902 | *
|
| 1903 | * Returns: maximum size of each region for convincing min_nr_regions.
|
| 1904 |
|
| 1905 | static unsigned long damon_apply_min_nr_regions(struct damon_ctx *ctx)
|
| 1906 | {
|
| 1907 | unsigned long max_region_sz = damon_region_sz_limit(ctx);
|
| 1908 | struct damon_target *t;
|
| 1909 | struct damon_region *r, *next;
|
| 1910 |
|
| 1911 | max_region_sz = ALIGN(max_region_sz, ctx->min_region_sz);
|
| 1912 | damon_for_each_target(t, ctx) {
|
| 1913 | damon_for_each_region_safe(r, next, t) {
|
| 1914 | while (damon_sz_region(r) > max_region_sz) {
|
| 1915 | if (damon_split_region_at(t, r, max_region_sz))
|
| 1916 | goto out;
|
| 1917 | r = damon_next_region(r);
|
| 1918 | }
|
| 1919 | }
|
| 1920 | }
|
| 1921 | out:
|
| 1922 | return max_region_sz;
|
| 1923 | }
|
| 1924 |
|
| 1925 | static int kdamond_fn(void *data);
|
| 1926 |
|
| 1927 | |
| 1928 | * __damon_start() - Starts monitoring with given context.
|
| 1929 | * @ctx: monitoring context
|
| 1930 | *
|
| 1931 | * This function should be called while damon_lock is hold.
|
| 1932 | *
|
| 1933 | * Return: 0 on success, negative error code otherwise.
|
| 1934 |
|
| 1935 | static int __damon_start(struct damon_ctx *ctx)
|
| 1936 | {
|
| 1937 | int err = -EBUSY;
|
| 1938 |
|
| 1939 | mutex_lock(&ctx->kdamond_lock);
|
| 1940 | if (!ctx->kdamond) {
|
| 1941 | err = 0;
|
| 1942 | reinit_completion(&ctx->kdamond_started);
|
| 1943 | ctx->kdamond = kthread_run(kdamond_fn, ctx, "kdamond.%d",
|
| 1944 | nr_running_ctxs);
|
| 1945 | if (IS_ERR(ctx->kdamond)) {
|
| 1946 | err = PTR_ERR(ctx->kdamond);
|
| 1947 | ctx->kdamond = NULL;
|
| 1948 | } else {
|
| 1949 | wait_for_completion(&ctx->kdamond_started);
|
| 1950 | }
|
| 1951 | }
|
| 1952 | mutex_unlock(&ctx->kdamond_lock);
|
| 1953 |
|
| 1954 | return err;
|
| 1955 | }
|
| 1956 |
|
| 1957 | static int __damon_commit_ctx(struct damon_ctx *dst, struct damon_ctx *src);
|
| 1958 |
|
| 1959 | |
| 1960 | * damon_start() - Starts the monitorings for a given group of contexts.
|
| 1961 | * @ctxs: an array of the pointers for contexts to start monitoring
|
| 1962 | * @nr_ctxs: size of @ctxs
|
| 1963 | * @exclusive: exclusiveness of this contexts group
|
| 1964 | *
|
| 1965 | * This function starts a group of monitoring threads for a group of monitoring
|
| 1966 | * contexts. One thread per each context is created and run in parallel. The
|
| 1967 | * caller should handle synchronization between the threads by itself. If
|
| 1968 | * @exclusive is true and a group of threads that created by other
|
| 1969 | * 'damon_start()' call is currently running, this function does nothing but
|
| 1970 | * returns -EBUSY.
|
| 1971 | *
|
| 1972 | * Return: 0 on success, negative error code otherwise.
|
| 1973 |
|
| 1974 | int damon_start(struct damon_ctx **ctxs, int nr_ctxs, bool exclusive)
|
| 1975 | {
|
| 1976 | int i;
|
| 1977 | int err = 0;
|
| 1978 |
|
| 1979 | for (i = 0; i < nr_ctxs; i++) {
|
| 1980 | struct damon_ctx *test_ctx;
|
| 1981 |
|
| 1982 | test_ctx = damon_new_ctx();
|
| 1983 | if (!test_ctx)
|
| 1984 | return -ENOMEM;
|
| 1985 |
|
| 1986 | err = __damon_commit_ctx(test_ctx, ctxs[i]);
|
| 1987 | damon_destroy_ctx(test_ctx);
|
| 1988 | if (err)
|
| 1989 | return err;
|
| 1990 | }
|
| 1991 |
|
| 1992 | mutex_lock(&damon_lock);
|
| 1993 | if ((exclusive && nr_running_ctxs) ||
|
| 1994 | (!exclusive && running_exclusive_ctxs)) {
|
| 1995 | mutex_unlock(&damon_lock);
|
| 1996 | return -EBUSY;
|
| 1997 | }
|
| 1998 |
|
| 1999 | for (i = 0; i < nr_ctxs; i++) {
|
| 2000 | err = __damon_start(ctxs[i]);
|
| 2001 | if (err)
|
| 2002 | break;
|
| 2003 | nr_running_ctxs++;
|
| 2004 | }
|
| 2005 | if (exclusive && nr_running_ctxs)
|
| 2006 | running_exclusive_ctxs = true;
|
| 2007 | mutex_unlock(&damon_lock);
|
| 2008 |
|
| 2009 | if (i != nr_ctxs)
|
| 2010 | damon_stop(ctxs, i);
|
| 2011 |
|
| 2012 | return err;
|
| 2013 | }
|
| 2014 |
|
| 2015 | |
| 2016 | * __damon_stop() - Stops monitoring of a given context.
|
| 2017 | * @ctx: monitoring context
|
| 2018 |
|
| 2019 | static void __damon_stop(struct damon_ctx *ctx)
|
| 2020 | {
|
| 2021 | struct task_struct *tsk;
|
| 2022 |
|
| 2023 | mutex_lock(&ctx->kdamond_lock);
|
| 2024 | tsk = ctx->kdamond;
|
| 2025 | if (tsk) {
|
| 2026 | get_task_struct(tsk);
|
| 2027 | mutex_unlock(&ctx->kdamond_lock);
|
| 2028 | kthread_stop_put(tsk);
|
| 2029 | return;
|
| 2030 | }
|
| 2031 | mutex_unlock(&ctx->kdamond_lock);
|
| 2032 | }
|
| 2033 |
|
| 2034 | |
| 2035 | * damon_stop() - Stops the monitorings for a given group of contexts.
|
| 2036 | * @ctxs: an array of the pointers for contexts to stop monitoring
|
| 2037 | * @nr_ctxs: size of @ctxs
|
| 2038 |
|
| 2039 | void damon_stop(struct damon_ctx **ctxs, int nr_ctxs)
|
| 2040 | {
|
| 2041 | int i;
|
| 2042 |
|
| 2043 | for (i = 0; i < nr_ctxs; i++)
|
| 2044 |
|
| 2045 | __damon_stop(ctxs[i]);
|
| 2046 | }
|
| 2047 |
|
| 2048 | |
| 2049 | * damon_is_running() - Returns if a given DAMON context is running.
|
| 2050 | * @ctx: The DAMON context to see if running.
|
| 2051 | *
|
| 2052 | * Return: true if @ctx is running, false otherwise.
|
| 2053 |
|
| 2054 | bool damon_is_running(struct damon_ctx *ctx)
|
| 2055 | {
|
| 2056 | bool running;
|
| 2057 |
|
| 2058 | mutex_lock(&ctx->kdamond_lock);
|
| 2059 | running = ctx->kdamond != NULL;
|
| 2060 | mutex_unlock(&ctx->kdamond_lock);
|
| 2061 | return running;
|
| 2062 | }
|
| 2063 |
|
| 2064 | |
| 2065 | * damon_kdamond_pid() - Return pid of a given DAMON context's worker thread.
|
| 2066 | * @ctx: The DAMON context of the question.
|
| 2067 | *
|
| 2068 | * Return: pid if @ctx is running, negative error code otherwise.
|
| 2069 |
|
| 2070 | int damon_kdamond_pid(struct damon_ctx *ctx)
|
| 2071 | {
|
| 2072 | int pid = -EINVAL;
|
| 2073 |
|
| 2074 | mutex_lock(&ctx->kdamond_lock);
|
| 2075 | if (ctx->kdamond)
|
| 2076 | pid = ctx->kdamond->pid;
|
| 2077 | mutex_unlock(&ctx->kdamond_lock);
|
| 2078 | return pid;
|
| 2079 | }
|
| 2080 |
|
| 2081 | |
| 2082 | * damon_call() - Invoke a given function on DAMON worker thread (kdamond).
|
| 2083 | * @ctx: DAMON context to call the function for.
|
| 2084 | * @control: Control variable of the call request.
|
| 2085 | *
|
| 2086 | * Ask DAMON worker thread (kdamond) of @ctx to call a function with an
|
| 2087 | * argument data that respectively passed via &damon_call_control->fn and
|
| 2088 | * &damon_call_control->data of @control. If &damon_call_control->repeat of
|
| 2089 | * @control is unset, further wait until the kdamond finishes handling of the
|
| 2090 | * request. Otherwise, return as soon as the request is made.
|
| 2091 | *
|
| 2092 | * The kdamond executes the function with the argument in the main loop, just
|
| 2093 | * after a sampling of the iteration is finished. The function can hence
|
| 2094 | * safely access the internal data of the &struct damon_ctx without additional
|
| 2095 | * synchronization. The return value of the function will be saved in
|
| 2096 | * &damon_call_control->return_code.
|
| 2097 | *
|
| 2098 | * Note that this function should be called only after damon_start() with the
|
| 2099 | * @ctx has succeeded. Otherwise, this function could fall into an indefinite
|
| 2100 | * wait.
|
| 2101 | *
|
| 2102 | * When this function is failed, the @ctx is guaranteed to be stopped.
|
| 2103 | *
|
| 2104 | * Return: 0 on success, negative error code otherwise.
|
| 2105 |
|
| 2106 | int damon_call(struct damon_ctx *ctx, struct damon_call_control *control)
|
| 2107 | {
|
| 2108 | if (!control->repeat)
|
| 2109 | init_completion(&control->completion);
|
| 2110 | control->canceled = false;
|
| 2111 | INIT_LIST_HEAD(&control->list);
|
| 2112 |
|
| 2113 | mutex_lock(&ctx->call_controls_lock);
|
| 2114 | if (ctx->call_controls_obsolete) {
|
| 2115 | mutex_unlock(&ctx->call_controls_lock);
|
| 2116 | goto canceled;
|
| 2117 | }
|
| 2118 | list_add_tail(&control->list, &ctx->call_controls);
|
| 2119 | mutex_unlock(&ctx->call_controls_lock);
|
| 2120 | if (control->repeat)
|
| 2121 | return 0;
|
| 2122 | wait_for_completion(&control->completion);
|
| 2123 | if (control->canceled)
|
| 2124 | goto canceled;
|
| 2125 | return 0;
|
| 2126 |
|
| 2127 | canceled:
|
| 2128 | while (damon_is_running(ctx))
|
| 2129 | schedule_timeout_idle(msecs_to_jiffies(100));
|
| 2130 | return -ECANCELED;
|
| 2131 |
|
| 2132 | }
|
| 2133 |
|
| 2134 | |
| 2135 | * damos_walk() - Invoke a given functions while DAMOS walk regions.
|
| 2136 | * @ctx: DAMON context to call the functions for.
|
| 2137 | * @control: Control variable of the walk request.
|
| 2138 | *
|
| 2139 | * Ask DAMON worker thread (kdamond) of @ctx to call a function for each region
|
| 2140 | * that the kdamond will apply DAMOS action to, and wait until the kdamond
|
| 2141 | * finishes handling of the request.
|
| 2142 | *
|
| 2143 | * The kdamond executes the given function in the main loop, for each region
|
| 2144 | * just after it applied any DAMOS actions of @ctx to it. The invocation is
|
| 2145 | * made only within one &damos->apply_interval_us since damos_walk()
|
| 2146 | * invocation, for each scheme. The given callback function can hence safely
|
| 2147 | * access the internal data of &struct damon_ctx and &struct damon_region that
|
| 2148 | * each of the scheme will apply the action for next interval, without
|
| 2149 | * additional synchronizations against the kdamond. If every scheme of @ctx
|
| 2150 | * passed at least one &damos->apply_interval_us, kdamond marks the request as
|
| 2151 | * completed so that damos_walk() can wakeup and return.
|
| 2152 | *
|
| 2153 | * Note that this function should be called only after damon_start() with the
|
| 2154 | * @ctx has succeeded. Otherwise, this function could fall into an indefinite
|
| 2155 | * wait.
|
| 2156 | *
|
| 2157 | * Return: 0 on success, negative error code otherwise.
|
| 2158 |
|
| 2159 | int damos_walk(struct damon_ctx *ctx, struct damos_walk_control *control)
|
| 2160 | {
|
| 2161 | init_completion(&control->completion);
|
| 2162 | control->canceled = false;
|
| 2163 | mutex_lock(&ctx->walk_control_lock);
|
| 2164 | if (ctx->walk_control_obsolete) {
|
| 2165 | mutex_unlock(&ctx->walk_control_lock);
|
| 2166 | return -ECANCELED;
|
| 2167 | }
|
| 2168 | if (ctx->walk_control) {
|
| 2169 | mutex_unlock(&ctx->walk_control_lock);
|
| 2170 | return -EBUSY;
|
| 2171 | }
|
| 2172 | ctx->walk_control = control;
|
| 2173 | mutex_unlock(&ctx->walk_control_lock);
|
| 2174 | wait_for_completion(&control->completion);
|
| 2175 | if (control->canceled)
|
| 2176 | return -ECANCELED;
|
| 2177 | return 0;
|
| 2178 | }
|
| 2179 |
|
| 2180 | |
| 2181 | * Reset the aggregated monitoring results ('nr_accesses' of each region).
|
| 2182 |
|
| 2183 | static void kdamond_reset_aggregated(struct damon_ctx *c)
|
| 2184 | {
|
| 2185 | struct damon_target *t;
|
| 2186 | unsigned int ti = 0;
|
| 2187 | unsigned int nr_probes = 0;
|
| 2188 | struct damon_probe *probe;
|
| 2189 |
|
| 2190 | if (trace_damon_region_aggregated_enabled()) {
|
| 2191 | damon_for_each_probe(probe, c)
|
| 2192 | nr_probes++;
|
| 2193 | }
|
| 2194 |
|
| 2195 | damon_for_each_target(t, c) {
|
| 2196 | struct damon_region *r;
|
| 2197 |
|
| 2198 | damon_for_each_region(r, t) {
|
| 2199 | int i;
|
| 2200 |
|
| 2201 | trace_damon_aggregated(ti, r, damon_nr_regions(t));
|
| 2202 | trace_damon_region_aggregated(ti, r,
|
| 2203 | damon_nr_regions(t), nr_probes);
|
| 2204 | r->last_nr_accesses = r->nr_accesses;
|
| 2205 | r->nr_accesses = 0;
|
| 2206 | for (i = 0; i < DAMON_MAX_PROBES; i++) {
|
| 2207 | r->last_probe_hits[i] = r->probe_hits[i];
|
| 2208 | r->probe_hits[i] = 0;
|
| 2209 | }
|
| 2210 | }
|
| 2211 | ti++;
|
| 2212 | }
|
| 2213 | }
|
| 2214 |
|
| 2215 | static unsigned long damon_get_intervals_score(struct damon_ctx *c)
|
| 2216 | {
|
| 2217 | struct damon_target *t;
|
| 2218 | struct damon_region *r;
|
| 2219 | unsigned long sz_region, max_access_events = 0, access_events = 0;
|
| 2220 | unsigned long target_access_events;
|
| 2221 | unsigned long goal_bp = c->attrs.intervals_goal.access_bp;
|
| 2222 |
|
| 2223 | damon_for_each_target(t, c) {
|
| 2224 | damon_for_each_region(r, t) {
|
| 2225 | sz_region = damon_sz_region(r);
|
| 2226 | max_access_events += sz_region * c->attrs.aggr_samples;
|
| 2227 | access_events += sz_region * r->nr_accesses;
|
| 2228 | }
|
| 2229 | }
|
| 2230 | target_access_events = max_access_events * goal_bp / 10000;
|
| 2231 | target_access_events = target_access_events ? : 1;
|
| 2232 | return mult_frac(access_events, 10000, target_access_events);
|
| 2233 | }
|
| 2234 |
|
| 2235 | static unsigned long damon_feed_loop_next_input(unsigned long last_input,
|
| 2236 | unsigned long score);
|
| 2237 |
|
| 2238 | static unsigned long damon_get_intervals_adaptation_bp(struct damon_ctx *c)
|
| 2239 | {
|
| 2240 | unsigned long score_bp, adaptation_bp;
|
| 2241 |
|
| 2242 | score_bp = damon_get_intervals_score(c);
|
| 2243 | adaptation_bp = damon_feed_loop_next_input(100000000, score_bp) /
|
| 2244 | 10000;
|
| 2245 | |
| 2246 | * adaptation_bp ranges from 1 to 20,000. Avoid too rapid reduction of
|
| 2247 | * the intervals by rescaling [1,10,000] to [5000, 10,000].
|
| 2248 |
|
| 2249 | if (adaptation_bp <= 10000)
|
| 2250 | adaptation_bp = 5000 + adaptation_bp / 2;
|
| 2251 | return adaptation_bp;
|
| 2252 | }
|
| 2253 |
|
| 2254 | static noinline_for_stack void kdamond_tune_intervals(struct damon_ctx *c)
|
| 2255 | {
|
| 2256 | unsigned long adaptation_bp;
|
| 2257 | struct damon_attrs new_attrs;
|
| 2258 | struct damon_intervals_goal *goal;
|
| 2259 |
|
| 2260 | adaptation_bp = damon_get_intervals_adaptation_bp(c);
|
| 2261 | if (adaptation_bp == 10000)
|
| 2262 | return;
|
| 2263 |
|
| 2264 | new_attrs = c->attrs;
|
| 2265 | goal = &c->attrs.intervals_goal;
|
| 2266 | new_attrs.sample_interval = min(goal->max_sample_us,
|
| 2267 | c->attrs.sample_interval * adaptation_bp / 10000);
|
| 2268 | new_attrs.sample_interval = max(goal->min_sample_us,
|
| 2269 | new_attrs.sample_interval);
|
| 2270 | new_attrs.aggr_interval = new_attrs.sample_interval *
|
| 2271 | c->attrs.aggr_samples;
|
| 2272 | trace_damon_monitor_intervals_tune(new_attrs.sample_interval);
|
| 2273 | damon_set_attrs(c, &new_attrs);
|
| 2274 | }
|
| 2275 |
|
| 2276 | static bool __damos_valid_target(struct damon_region *r, struct damos *s,
|
| 2277 | struct damon_ctx *c)
|
| 2278 | {
|
| 2279 | unsigned long sz;
|
| 2280 | unsigned int nr_accesses = damon_nr_accesses_mvsum(r, c);
|
| 2281 |
|
| 2282 | sz = damon_sz_region(r);
|
| 2283 | return s->pattern.min_sz_region <= sz &&
|
| 2284 | sz <= s->pattern.max_sz_region &&
|
| 2285 | s->pattern.min_nr_accesses <= nr_accesses &&
|
| 2286 | nr_accesses <= s->pattern.max_nr_accesses &&
|
| 2287 | s->pattern.min_age_region <= r->age &&
|
| 2288 | r->age <= s->pattern.max_age_region;
|
| 2289 | }
|
| 2290 |
|
| 2291 | |
| 2292 | * damos_quota_is_set() - Return if the given quota is actually set.
|
| 2293 | * @quota: The quota to check.
|
| 2294 | *
|
| 2295 | * Returns true if the quota is set, false otherwise.
|
| 2296 |
|
| 2297 | static bool damos_quota_is_set(struct damos_quota *quota)
|
| 2298 | {
|
| 2299 | return quota->esz || quota->sz || quota->ms ||
|
| 2300 | !damos_quota_goals_empty(quota);
|
| 2301 | }
|
| 2302 |
|
| 2303 | static bool damos_valid_target(struct damon_ctx *c, struct damon_region *r,
|
| 2304 | struct damos *s)
|
| 2305 | {
|
| 2306 | bool ret = __damos_valid_target(r, s, c);
|
| 2307 |
|
| 2308 | if (!ret || !damos_quota_is_set(&s->quota) || !c->ops.get_scheme_score)
|
| 2309 | return ret;
|
| 2310 |
|
| 2311 | return c->ops.get_scheme_score(c, r, s) >= s->quota.min_score;
|
| 2312 | }
|
| 2313 |
|
| 2314 | |
| 2315 | * damos_skip_charged_region() - Check if the given region or starting part of
|
| 2316 | * it is already charged for the DAMOS quota.
|
| 2317 | * @t: The target of the region.
|
| 2318 | * @rp: The pointer to the region.
|
| 2319 | * @s: The scheme to be applied.
|
| 2320 | * @min_region_sz: minimum region size.
|
| 2321 | *
|
| 2322 | * If a quota of a scheme has exceeded in a quota charge window, the scheme's
|
| 2323 | * action would applied to only a part of the target access pattern fulfilling
|
| 2324 | * regions. To avoid applying the scheme action to only already applied
|
| 2325 | * regions, DAMON skips applying the scheme action to the regions that charged
|
| 2326 | * in the previous charge window.
|
| 2327 | *
|
| 2328 | * This function checks if a given region should be skipped or not for the
|
| 2329 | * reason. If only the starting part of the region has previously charged,
|
| 2330 | * this function splits the region into two so that the second one covers the
|
| 2331 | * area that not charged in the previous charge widnow, and return true. The
|
| 2332 | * caller can see the second one on the next iteration of the region walk.
|
| 2333 | * Note that this means the caller should use damon_for_each_region() instead
|
| 2334 | * of damon_for_each_region_safe(). If damon_for_each_region_safe() is used,
|
| 2335 | * the second region will just be ignored.
|
| 2336 | *
|
| 2337 | * Return: true if the region should be skipped, false otherwise.
|
| 2338 |
|
| 2339 | static bool damos_skip_charged_region(struct damon_target *t,
|
| 2340 | struct damon_region *r, struct damos *s,
|
| 2341 | unsigned long min_region_sz)
|
| 2342 | {
|
| 2343 | struct damos_quota *quota = &s->quota;
|
| 2344 | unsigned long sz_to_skip;
|
| 2345 | bool skip = false;
|
| 2346 |
|
| 2347 |
|
| 2348 | if (quota->charge_target_from) {
|
| 2349 | if (t != quota->charge_target_from)
|
| 2350 | return true;
|
| 2351 | if (quota->charge_addr_from &&
|
| 2352 | r->ar.end <= quota->charge_addr_from) {
|
| 2353 | skip = true;
|
| 2354 | goto out;
|
| 2355 | }
|
| 2356 |
|
| 2357 | if (quota->charge_addr_from && r->ar.start <
|
| 2358 | quota->charge_addr_from) {
|
| 2359 | sz_to_skip = ALIGN_DOWN(quota->charge_addr_from -
|
| 2360 | r->ar.start, min_region_sz);
|
| 2361 | if (!sz_to_skip) {
|
| 2362 | if (damon_sz_region(r) <= min_region_sz) {
|
| 2363 | skip = true;
|
| 2364 | goto out;
|
| 2365 | }
|
| 2366 | sz_to_skip = min_region_sz;
|
| 2367 | }
|
| 2368 | damon_split_region_at(t, r, sz_to_skip);
|
| 2369 | skip = true;
|
| 2370 | }
|
| 2371 | }
|
| 2372 | out:
|
| 2373 | if (r == damon_last_region(t)) {
|
| 2374 | quota->charge_target_from = NULL;
|
| 2375 | quota->charge_addr_from = 0;
|
| 2376 | }
|
| 2377 | return skip;
|
| 2378 | }
|
| 2379 |
|
| 2380 | static void damos_update_stat(struct damos *s,
|
| 2381 | unsigned long sz_tried, unsigned long sz_applied,
|
| 2382 | unsigned long sz_ops_filter_passed)
|
| 2383 | {
|
| 2384 | s->stat.nr_tried++;
|
| 2385 | s->stat.sz_tried += sz_tried;
|
| 2386 | if (sz_applied)
|
| 2387 | s->stat.nr_applied++;
|
| 2388 | s->stat.sz_applied += sz_applied;
|
| 2389 | s->stat.sz_ops_filter_passed += sz_ops_filter_passed;
|
| 2390 | }
|
| 2391 |
|
| 2392 | static bool damos_filter_match(struct damon_ctx *ctx, struct damon_target *t,
|
| 2393 | struct damon_region *r, struct damos_filter *filter,
|
| 2394 | unsigned long min_region_sz)
|
| 2395 | {
|
| 2396 | bool matched = false;
|
| 2397 | struct damon_target *ti;
|
| 2398 | int target_idx = 0;
|
| 2399 | unsigned long start, end;
|
| 2400 |
|
| 2401 | switch (filter->type) {
|
| 2402 | case DAMOS_FILTER_TYPE_TARGET:
|
| 2403 | damon_for_each_target(ti, ctx) {
|
| 2404 | if (ti == t)
|
| 2405 | break;
|
| 2406 | target_idx++;
|
| 2407 | }
|
| 2408 | matched = target_idx == filter->target_idx;
|
| 2409 | break;
|
| 2410 | case DAMOS_FILTER_TYPE_ADDR:
|
| 2411 | start = ALIGN_DOWN(filter->addr_range.start, min_region_sz);
|
| 2412 | end = ALIGN_DOWN(filter->addr_range.end, min_region_sz);
|
| 2413 |
|
| 2414 |
|
| 2415 | if (start <= r->ar.start && r->ar.end <= end) {
|
| 2416 | matched = true;
|
| 2417 | break;
|
| 2418 | }
|
| 2419 |
|
| 2420 | if (r->ar.end <= start || end <= r->ar.start) {
|
| 2421 | matched = false;
|
| 2422 | break;
|
| 2423 | }
|
| 2424 |
|
| 2425 | if (r->ar.start < start) {
|
| 2426 | damon_split_region_at(t, r, start - r->ar.start);
|
| 2427 | matched = false;
|
| 2428 | break;
|
| 2429 | }
|
| 2430 |
|
| 2431 | damon_split_region_at(t, r, end - r->ar.start);
|
| 2432 | matched = true;
|
| 2433 | break;
|
| 2434 | default:
|
| 2435 | return false;
|
| 2436 | }
|
| 2437 |
|
| 2438 | return matched == filter->matching;
|
| 2439 | }
|
| 2440 |
|
| 2441 | static bool damos_core_filter_out(struct damon_ctx *ctx, struct damon_target *t,
|
| 2442 | struct damon_region *r, struct damos *s)
|
| 2443 | {
|
| 2444 | struct damos_filter *filter;
|
| 2445 |
|
| 2446 | s->core_filters_allowed = false;
|
| 2447 | damos_for_each_core_filter(filter, s) {
|
| 2448 | if (damos_filter_match(ctx, t, r, filter, ctx->min_region_sz)) {
|
| 2449 | if (filter->allow)
|
| 2450 | s->core_filters_allowed = true;
|
| 2451 | return !filter->allow;
|
| 2452 | }
|
| 2453 | }
|
| 2454 | return s->core_filters_default_reject;
|
| 2455 | }
|
| 2456 |
|
| 2457 | |
| 2458 | * damos_walk_call_walk() - Call &damos_walk_control->walk_fn.
|
| 2459 | * @ctx: The context of &damon_ctx->walk_control.
|
| 2460 | * @t: The monitoring target of @r that @s will be applied.
|
| 2461 | * @r: The region of @t that @s will be applied.
|
| 2462 | * @s: The scheme of @ctx that will be applied to @r.
|
| 2463 | *
|
| 2464 | * This function is called from kdamond whenever it asked the operation set to
|
| 2465 | * apply a DAMOS scheme action to a region. If a DAMOS walk request is
|
| 2466 | * installed by damos_walk() and not yet uninstalled, invoke it.
|
| 2467 |
|
| 2468 | static void damos_walk_call_walk(struct damon_ctx *ctx, struct damon_target *t,
|
| 2469 | struct damon_region *r, struct damos *s,
|
| 2470 | unsigned long sz_filter_passed)
|
| 2471 | {
|
| 2472 | struct damos_walk_control *control;
|
| 2473 |
|
| 2474 | if (s->walk_completed)
|
| 2475 | return;
|
| 2476 |
|
| 2477 | control = ctx->walk_control;
|
| 2478 | if (!control)
|
| 2479 | return;
|
| 2480 |
|
| 2481 | control->walk_fn(control->data, ctx, t, r, s, sz_filter_passed);
|
| 2482 | }
|
| 2483 |
|
| 2484 | |
| 2485 | * damos_walk_complete() - Complete DAMOS walk request if all walks are done.
|
| 2486 | * @ctx: The context of &damon_ctx->walk_control.
|
| 2487 | * @s: A scheme of @ctx that all walks are now done.
|
| 2488 | *
|
| 2489 | * This function is called when kdamond finished applying the action of a DAMOS
|
| 2490 | * scheme to all regions that eligible for the given &damos->apply_interval_us.
|
| 2491 | * If every scheme of @ctx including @s now finished walking for at least one
|
| 2492 | * &damos->apply_interval_us, this function makrs the handling of the given
|
| 2493 | * DAMOS walk request is done, so that damos_walk() can wake up and return.
|
| 2494 |
|
| 2495 | static void damos_walk_complete(struct damon_ctx *ctx, struct damos *s)
|
| 2496 | {
|
| 2497 | struct damos *siter;
|
| 2498 | struct damos_walk_control *control;
|
| 2499 |
|
| 2500 | control = ctx->walk_control;
|
| 2501 | if (!control)
|
| 2502 | return;
|
| 2503 |
|
| 2504 | s->walk_completed = true;
|
| 2505 |
|
| 2506 | damon_for_each_scheme(siter, ctx) {
|
| 2507 | if (!siter->walk_completed)
|
| 2508 | return;
|
| 2509 | }
|
| 2510 | damon_for_each_scheme(siter, ctx)
|
| 2511 | siter->walk_completed = false;
|
| 2512 |
|
| 2513 | complete(&control->completion);
|
| 2514 | ctx->walk_control = NULL;
|
| 2515 | }
|
| 2516 |
|
| 2517 | |
| 2518 | * damos_walk_cancel() - Cancel the current DAMOS walk request.
|
| 2519 | * @ctx: The context of &damon_ctx->walk_control.
|
| 2520 | *
|
| 2521 | * This function is called when @ctx is deactivated by DAMOS watermarks, DAMOS
|
| 2522 | * walk is requested but there is no DAMOS scheme to walk for, or the kdamond
|
| 2523 | * is already out of the main loop and therefore gonna be terminated, and hence
|
| 2524 | * cannot continue the walks. This function therefore marks the walk request
|
| 2525 | * as canceled, so that damos_walk() can wake up and return.
|
| 2526 |
|
| 2527 | static void damos_walk_cancel(struct damon_ctx *ctx)
|
| 2528 | {
|
| 2529 | struct damos_walk_control *control;
|
| 2530 |
|
| 2531 | mutex_lock(&ctx->walk_control_lock);
|
| 2532 | control = ctx->walk_control;
|
| 2533 | mutex_unlock(&ctx->walk_control_lock);
|
| 2534 |
|
| 2535 | if (!control)
|
| 2536 | return;
|
| 2537 | control->canceled = true;
|
| 2538 | complete(&control->completion);
|
| 2539 | mutex_lock(&ctx->walk_control_lock);
|
| 2540 | ctx->walk_control = NULL;
|
| 2541 | mutex_unlock(&ctx->walk_control_lock);
|
| 2542 | }
|
| 2543 |
|
| 2544 | static void damos_charge_quota(struct damos_quota *quota,
|
| 2545 | unsigned long sz_region, unsigned long sz_applied)
|
| 2546 | {
|
| 2547 | |
| 2548 | * sz_applied could be bigger than sz_region, depending on ops
|
| 2549 | * implementation of the action, e.g., damos_pa_pageout(). Charge only
|
| 2550 | * the region size in the case.
|
| 2551 |
|
| 2552 | if (!quota->fail_charge_denom || sz_applied > sz_region)
|
| 2553 | quota->charged_sz += sz_region;
|
| 2554 | else
|
| 2555 | quota->charged_sz += sz_applied + mult_frac(
|
| 2556 | (sz_region - sz_applied),
|
| 2557 | quota->fail_charge_num,
|
| 2558 | quota->fail_charge_denom);
|
| 2559 | }
|
| 2560 |
|
| 2561 | static bool damos_quota_is_full(struct damos_quota *quota,
|
| 2562 | unsigned long min_region_sz)
|
| 2563 | {
|
| 2564 | if (!damos_quota_is_set(quota))
|
| 2565 | return false;
|
| 2566 | if (quota->charged_sz >= quota->esz)
|
| 2567 | return true;
|
| 2568 | |
| 2569 | * DAMOS action is applied per region, so <min_region_sz remaining
|
| 2570 | * quota means the quota is effectively full.
|
| 2571 |
|
| 2572 | return quota->esz - quota->charged_sz < min_region_sz;
|
| 2573 | }
|
| 2574 |
|
| 2575 | static void damos_apply_scheme(struct damon_ctx *c, struct damon_target *t,
|
| 2576 | struct damon_region *r, struct damos *s)
|
| 2577 | {
|
| 2578 | struct damos_quota *quota = &s->quota;
|
| 2579 | unsigned long sz = damon_sz_region(r);
|
| 2580 | struct timespec64 begin, end;
|
| 2581 | unsigned long sz_applied = 0;
|
| 2582 | unsigned long sz_ops_filter_passed = 0;
|
| 2583 | |
| 2584 | * We plan to support multiple context per kdamond, as DAMON sysfs
|
| 2585 | * implies with 'nr_contexts' file. Nevertheless, only single context
|
| 2586 | * per kdamond is supported for now. So, we can simply use '0' context
|
| 2587 | * index here.
|
| 2588 |
|
| 2589 | unsigned int cidx = 0;
|
| 2590 | struct damos *siter;
|
| 2591 | unsigned int sidx = 0;
|
| 2592 | struct damon_target *titer;
|
| 2593 | unsigned int tidx = 0, nr_accesses = 0;
|
| 2594 | bool do_trace = false;
|
| 2595 |
|
| 2596 |
|
| 2597 | if (trace_damos_before_apply_enabled()) {
|
| 2598 | damon_for_each_scheme(siter, c) {
|
| 2599 | if (siter == s)
|
| 2600 | break;
|
| 2601 | sidx++;
|
| 2602 | }
|
| 2603 | damon_for_each_target(titer, c) {
|
| 2604 | if (titer == t)
|
| 2605 | break;
|
| 2606 | tidx++;
|
| 2607 | }
|
| 2608 | nr_accesses = damon_nr_accesses_mvsum(r, c);
|
| 2609 | do_trace = true;
|
| 2610 | }
|
| 2611 |
|
| 2612 | if (c->ops.apply_scheme) {
|
| 2613 | if (damos_quota_is_set(quota) &&
|
| 2614 | quota->charged_sz + sz > quota->esz) {
|
| 2615 | sz = ALIGN_DOWN(quota->esz - quota->charged_sz,
|
| 2616 | c->min_region_sz);
|
| 2617 | if (!sz)
|
| 2618 | goto update_stat;
|
| 2619 | damon_split_region_at(t, r, sz);
|
| 2620 | }
|
| 2621 | if (damos_core_filter_out(c, t, r, s))
|
| 2622 | return;
|
| 2623 | ktime_get_coarse_ts64(&begin);
|
| 2624 | trace_damos_before_apply(cidx, sidx, tidx, r, nr_accesses,
|
| 2625 | damon_nr_regions(t), do_trace);
|
| 2626 | sz_applied = c->ops.apply_scheme(c, t, r, s,
|
| 2627 | &sz_ops_filter_passed);
|
| 2628 | damos_walk_call_walk(c, t, r, s, sz_ops_filter_passed);
|
| 2629 | ktime_get_coarse_ts64(&end);
|
| 2630 | quota->total_charged_ns += timespec64_to_ns(&end) -
|
| 2631 | timespec64_to_ns(&begin);
|
| 2632 | damos_charge_quota(quota, sz, sz_applied);
|
| 2633 | if (damos_quota_is_full(quota, c->min_region_sz)) {
|
| 2634 | quota->charge_target_from = t;
|
| 2635 | quota->charge_addr_from = r->ar.end;
|
| 2636 | }
|
| 2637 | }
|
| 2638 | if (s->action != DAMOS_STAT)
|
| 2639 | r->age = 0;
|
| 2640 |
|
| 2641 | update_stat:
|
| 2642 | damos_update_stat(s, sz, sz_applied, sz_ops_filter_passed);
|
| 2643 | }
|
| 2644 |
|
| 2645 | static void damon_do_apply_schemes(struct damon_ctx *c,
|
| 2646 | struct damon_target *t,
|
| 2647 | struct damon_region *r)
|
| 2648 | {
|
| 2649 | struct damos *s;
|
| 2650 |
|
| 2651 | damon_for_each_scheme(s, c) {
|
| 2652 | struct damos_quota *quota = &s->quota;
|
| 2653 |
|
| 2654 | if (time_before(c->passed_sample_intervals, s->next_apply_sis))
|
| 2655 | continue;
|
| 2656 |
|
| 2657 | if (!s->wmarks.activated)
|
| 2658 | continue;
|
| 2659 |
|
| 2660 |
|
| 2661 | if (damos_quota_is_full(quota, c->min_region_sz))
|
| 2662 | continue;
|
| 2663 |
|
| 2664 | if (damos_skip_charged_region(t, r, s, c->min_region_sz))
|
| 2665 | continue;
|
| 2666 |
|
| 2667 | if (s->max_nr_snapshots &&
|
| 2668 | s->max_nr_snapshots <= s->stat.nr_snapshots)
|
| 2669 | continue;
|
| 2670 |
|
| 2671 | if (damos_valid_target(c, r, s))
|
| 2672 | damos_apply_scheme(c, t, r, s);
|
| 2673 |
|
| 2674 | if (damon_is_last_region(r, t))
|
| 2675 | s->stat.nr_snapshots++;
|
| 2676 | }
|
| 2677 | }
|
| 2678 |
|
| 2679 | |
| 2680 | * damos_apply_target() - Apply DAMOS schemes to a given target.
|
| 2681 | * @c: monitoring context to apply its DAMOS schemes to..
|
| 2682 | * @t: monitoring target to apply the schemes to.
|
| 2683 | * @max_region_sz: maximum region size for @c.
|
| 2684 | *
|
| 2685 | * This function could split regions for keeping the quota. To minimize
|
| 2686 | * overhead from the split operations increased number of regions, this
|
| 2687 | * function will also merge regions after the schemes applying attempt is done,
|
| 2688 | * for each region. The merge operation is made only when it doesn't lose the
|
| 2689 | * monitoring information and not violating @max_region_sz.
|
| 2690 | *
|
| 2691 | * Hence, after this function is called, the total number of regions could
|
| 2692 | * be increased or reduced. The increase could make max_nr_regions temporarily
|
| 2693 | * be violated, until the next per-aggregation interval regions merge operation
|
| 2694 | * is executed. The decrease will not violate min_nr_regions though, since it
|
| 2695 | * keeps @max_region_sz.
|
| 2696 |
|
| 2697 | static void damos_apply_target(struct damon_ctx *c, struct damon_target *t,
|
| 2698 | unsigned long max_region_sz)
|
| 2699 | {
|
| 2700 | struct damon_region *r;
|
| 2701 |
|
| 2702 | damon_for_each_region(r, t) {
|
| 2703 | struct damon_region *prev_r;
|
| 2704 |
|
| 2705 | damon_do_apply_schemes(c, t, r);
|
| 2706 | |
| 2707 | * damon_do_apply_scheems() could split the region for the
|
| 2708 | * quota. Keeping the new slices is an overhead. Merge back
|
| 2709 | * the slices into the previous region if it doesn't lose any
|
| 2710 | * information and not violating the max_region_sz.
|
| 2711 |
|
| 2712 | if (damon_first_region(t) == r)
|
| 2713 | continue;
|
| 2714 | prev_r = damon_prev_region(r);
|
| 2715 | if (prev_r->ar.end != r->ar.start)
|
| 2716 | continue;
|
| 2717 | if (prev_r->age != r->age)
|
| 2718 | continue;
|
| 2719 | if (prev_r->last_nr_accesses != r->last_nr_accesses)
|
| 2720 | continue;
|
| 2721 | if (prev_r->nr_accesses != r->nr_accesses)
|
| 2722 | continue;
|
| 2723 | if (r->ar.end - prev_r->ar.start > max_region_sz)
|
| 2724 | continue;
|
| 2725 | prev_r->ar.end = r->ar.end;
|
| 2726 | damon_destroy_region(r, t);
|
| 2727 | r = prev_r;
|
| 2728 | }
|
| 2729 | }
|
| 2730 |
|
| 2731 | |
| 2732 | * damon_feed_loop_next_input() - get next input to achieve a target score.
|
| 2733 | * @last_input The last input.
|
| 2734 | * @score Current score that made with @last_input.
|
| 2735 | *
|
| 2736 | * Calculate next input to achieve the target score, based on the last input
|
| 2737 | * and current score. Assuming the input and the score are positively
|
| 2738 | * proportional, calculate how much compensation should be added to or
|
| 2739 | * subtracted from the last input as a proportion of the last input. Avoid
|
| 2740 | * next input always being zero by setting it non-zero always. In short form
|
| 2741 | * (assuming support of float and signed calculations), the algorithm is as
|
| 2742 | * below.
|
| 2743 | *
|
| 2744 | * next_input = max(last_input * ((goal - current) / goal + 1), 1)
|
| 2745 | *
|
| 2746 | * For simple implementation, we assume the target score is always 10,000. The
|
| 2747 | * caller should adjust @score for this.
|
| 2748 | *
|
| 2749 | * Returns next input that assumed to achieve the target score.
|
| 2750 |
|
| 2751 | static unsigned long damon_feed_loop_next_input(unsigned long last_input,
|
| 2752 | unsigned long score)
|
| 2753 | {
|
| 2754 | const unsigned long goal = 10000;
|
| 2755 |
|
| 2756 | const unsigned long min_input = 10000;
|
| 2757 | unsigned long score_goal_diff, compensation;
|
| 2758 | bool over_achieving = score > goal;
|
| 2759 |
|
| 2760 | if (score == goal)
|
| 2761 | return last_input;
|
| 2762 | if (score >= goal * 2)
|
| 2763 | return min_input;
|
| 2764 |
|
| 2765 | if (over_achieving)
|
| 2766 | score_goal_diff = score - goal;
|
| 2767 | else
|
| 2768 | score_goal_diff = goal - score;
|
| 2769 |
|
| 2770 | if (last_input < ULONG_MAX / score_goal_diff)
|
| 2771 | compensation = last_input * score_goal_diff / goal;
|
| 2772 | else
|
| 2773 | compensation = last_input / goal * score_goal_diff;
|
| 2774 |
|
| 2775 | if (over_achieving)
|
| 2776 | return max(last_input - compensation, min_input);
|
| 2777 | if (last_input < ULONG_MAX - compensation)
|
| 2778 | return last_input + compensation;
|
| 2779 | return ULONG_MAX;
|
| 2780 | }
|
| 2781 |
|
| 2782 | #ifdef CONFIG_PSI
|
| 2783 |
|
| 2784 | static u64 damos_get_some_mem_psi_total(void)
|
| 2785 | {
|
| 2786 | if (static_branch_likely(&psi_disabled))
|
| 2787 | return 0;
|
| 2788 | return div_u64(psi_system.total[PSI_AVGS][PSI_MEM * 2],
|
| 2789 | NSEC_PER_USEC);
|
| 2790 | }
|
| 2791 |
|
| 2792 | #else
|
| 2793 |
|
| 2794 | static inline u64 damos_get_some_mem_psi_total(void)
|
| 2795 | {
|
| 2796 | return 0;
|
| 2797 | }
|
| 2798 |
|
| 2799 | #endif
|
| 2800 |
|
| 2801 | #ifdef CONFIG_NUMA
|
| 2802 | static bool invalid_mem_node(int nid)
|
| 2803 | {
|
| 2804 | return nid < 0 || nid >= MAX_NUMNODES || !node_state(nid, N_MEMORY);
|
| 2805 | }
|
| 2806 |
|
| 2807 | static __kernel_ulong_t damos_get_node_mem_bp(
|
| 2808 | struct damos_quota_goal *goal)
|
| 2809 | {
|
| 2810 | struct sysinfo i;
|
| 2811 | __kernel_ulong_t numerator;
|
| 2812 |
|
| 2813 | if (invalid_mem_node(goal->nid)) {
|
| 2814 | if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP)
|
| 2815 | return 0;
|
| 2816 | else
|
| 2817 | return 10000;
|
| 2818 | }
|
| 2819 |
|
| 2820 | si_meminfo_node(&i, goal->nid);
|
| 2821 | if (goal->metric == DAMOS_QUOTA_NODE_MEM_USED_BP)
|
| 2822 | numerator = i.totalram - i.freeram;
|
| 2823 | else
|
| 2824 | numerator = i.freeram;
|
| 2825 | return mult_frac(numerator, 10000, i.totalram);
|
| 2826 | }
|
| 2827 |
|
| 2828 | static unsigned long damos_get_node_memcg_used_bp(
|
| 2829 | struct damos_quota_goal *goal)
|
| 2830 | {
|
| 2831 | struct mem_cgroup *memcg;
|
| 2832 | struct lruvec *lruvec;
|
| 2833 | unsigned long used_pages, numerator;
|
| 2834 | struct sysinfo i;
|
| 2835 |
|
| 2836 | if (invalid_mem_node(goal->nid)) {
|
| 2837 | if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP)
|
| 2838 | return 0;
|
| 2839 | else
|
| 2840 | return 10000;
|
| 2841 | }
|
| 2842 |
|
| 2843 | memcg = mem_cgroup_get_from_id(goal->memcg_id);
|
| 2844 | if (!memcg) {
|
| 2845 | if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP)
|
| 2846 | return 0;
|
| 2847 | else
|
| 2848 | return 10000;
|
| 2849 | }
|
| 2850 |
|
| 2851 | mem_cgroup_flush_stats(memcg);
|
| 2852 | lruvec = mem_cgroup_lruvec(memcg, NODE_DATA(goal->nid));
|
| 2853 | used_pages = lruvec_page_state(lruvec, NR_ACTIVE_ANON);
|
| 2854 | used_pages += lruvec_page_state(lruvec, NR_INACTIVE_ANON);
|
| 2855 | used_pages += lruvec_page_state(lruvec, NR_ACTIVE_FILE);
|
| 2856 | used_pages += lruvec_page_state(lruvec, NR_INACTIVE_FILE);
|
| 2857 |
|
| 2858 | mem_cgroup_put(memcg);
|
| 2859 |
|
| 2860 | si_meminfo_node(&i, goal->nid);
|
| 2861 | if (goal->metric == DAMOS_QUOTA_NODE_MEMCG_USED_BP)
|
| 2862 | numerator = used_pages;
|
| 2863 | else
|
| 2864 | numerator = i.totalram - used_pages;
|
| 2865 | return mult_frac(numerator, 10000, i.totalram);
|
| 2866 | }
|
| 2867 |
|
| 2868 | #ifdef CONFIG_DAMON_PADDR
|
| 2869 | |
| 2870 | * damos_calc_eligible_bytes() - Calculate raw eligible bytes per node.
|
| 2871 | * @c: The DAMON context.
|
| 2872 | * @s: The scheme.
|
| 2873 | * @nid: The target NUMA node id.
|
| 2874 | * @total: Output for total eligible bytes across all nodes.
|
| 2875 | *
|
| 2876 | * Iterates through each folio in eligible regions to accurately determine
|
| 2877 | * which node the memory resides on. Returns eligible bytes on the specified
|
| 2878 | * node and sets *total to the sum across all nodes.
|
| 2879 | *
|
| 2880 | * Note: This function requires damon_get_folio() from ops-common.c, which is
|
| 2881 | * only available when CONFIG_DAMON_PADDR is enabled. It also requires the
|
| 2882 | * context to be using PADDR operations for meaningful results.
|
| 2883 |
|
| 2884 | static phys_addr_t damos_calc_eligible_bytes(struct damon_ctx *c,
|
| 2885 | struct damos *s, int nid, phys_addr_t *total)
|
| 2886 | {
|
| 2887 | struct damon_target *t;
|
| 2888 | struct damon_region *r;
|
| 2889 | phys_addr_t total_eligible = 0;
|
| 2890 | phys_addr_t node_eligible = 0;
|
| 2891 |
|
| 2892 | damon_for_each_target(t, c) {
|
| 2893 | damon_for_each_region(r, t) {
|
| 2894 | phys_addr_t addr, end_addr;
|
| 2895 |
|
| 2896 | if (!__damos_valid_target(r, s, c))
|
| 2897 | continue;
|
| 2898 |
|
| 2899 |
|
| 2900 | addr = (phys_addr_t)r->ar.start * c->addr_unit;
|
| 2901 | end_addr = (phys_addr_t)r->ar.end * c->addr_unit;
|
| 2902 | while (addr < end_addr) {
|
| 2903 | struct folio *folio;
|
| 2904 | phys_addr_t folio_start, folio_end;
|
| 2905 | phys_addr_t overlap_start, overlap_end;
|
| 2906 | phys_addr_t counted;
|
| 2907 |
|
| 2908 | folio = damon_get_folio(PHYS_PFN(addr));
|
| 2909 | if (!folio) {
|
| 2910 | addr = PAGE_ALIGN_DOWN(addr +
|
| 2911 | PAGE_SIZE);
|
| 2912 | if (!addr)
|
| 2913 | break;
|
| 2914 | continue;
|
| 2915 | }
|
| 2916 |
|
| 2917 | |
| 2918 | * Calculate exact overlap between the region
|
| 2919 | * [addr, end_addr) and the folio range.
|
| 2920 | * The folio may start before addr if addr is
|
| 2921 | * in the middle of a large folio.
|
| 2922 |
|
| 2923 | folio_start = PFN_PHYS(folio_pfn(folio));
|
| 2924 | folio_end = folio_start + folio_size(folio);
|
| 2925 |
|
| 2926 | overlap_start = max(addr, folio_start);
|
| 2927 | overlap_end = min(end_addr, folio_end);
|
| 2928 |
|
| 2929 | if (overlap_end > overlap_start) {
|
| 2930 | counted = overlap_end - overlap_start;
|
| 2931 | total_eligible += counted;
|
| 2932 | if (folio_nid(folio) == nid)
|
| 2933 | node_eligible += counted;
|
| 2934 | }
|
| 2935 |
|
| 2936 |
|
| 2937 | addr = folio_end;
|
| 2938 | folio_put(folio);
|
| 2939 | }
|
| 2940 | cond_resched();
|
| 2941 | }
|
| 2942 | }
|
| 2943 |
|
| 2944 | *total = total_eligible;
|
| 2945 | return node_eligible;
|
| 2946 | }
|
| 2947 |
|
| 2948 | static unsigned long damos_get_node_eligible_mem_bp(struct damon_ctx *c,
|
| 2949 | struct damos *s, int nid)
|
| 2950 | {
|
| 2951 | phys_addr_t total_eligible = 0;
|
| 2952 | phys_addr_t node_eligible;
|
| 2953 |
|
| 2954 | if (c->ops.id != DAMON_OPS_PADDR)
|
| 2955 | return 0;
|
| 2956 |
|
| 2957 | if (nid < 0 || nid >= MAX_NUMNODES || !node_online(nid))
|
| 2958 | return 0;
|
| 2959 |
|
| 2960 | node_eligible = damos_calc_eligible_bytes(c, s, nid, &total_eligible);
|
| 2961 |
|
| 2962 | if (!(unsigned long)total_eligible)
|
| 2963 | return 0;
|
| 2964 |
|
| 2965 | return mult_frac((unsigned long)node_eligible, 10000,
|
| 2966 | (unsigned long)total_eligible);
|
| 2967 | }
|
| 2968 | #else
|
| 2969 | static unsigned long damos_get_node_eligible_mem_bp(struct damon_ctx *c,
|
| 2970 | struct damos *s, int nid)
|
| 2971 | {
|
| 2972 | return 0;
|
| 2973 | }
|
| 2974 | #endif
|
| 2975 | #else
|
| 2976 | static __kernel_ulong_t damos_get_node_mem_bp(
|
| 2977 | struct damos_quota_goal *goal)
|
| 2978 | {
|
| 2979 | return 0;
|
| 2980 | }
|
| 2981 |
|
| 2982 | static unsigned long damos_get_node_memcg_used_bp(
|
| 2983 | struct damos_quota_goal *goal)
|
| 2984 | {
|
| 2985 | return 0;
|
| 2986 | }
|
| 2987 |
|
| 2988 | static unsigned long damos_get_node_eligible_mem_bp(struct damon_ctx *c,
|
| 2989 | struct damos *s, int nid)
|
| 2990 | {
|
| 2991 | return 0;
|
| 2992 | }
|
| 2993 | #endif
|
| 2994 |
|
| 2995 | |
| 2996 | * Returns LRU-active or inactive memory to total LRU memory size ratio.
|
| 2997 |
|
| 2998 | static unsigned int damos_get_in_active_mem_bp(bool active_ratio)
|
| 2999 | {
|
| 3000 | unsigned long active, inactive, total;
|
| 3001 |
|
| 3002 |
|
| 3003 | active = global_node_page_state(NR_LRU_BASE + LRU_ACTIVE_ANON) +
|
| 3004 | global_node_page_state(NR_LRU_BASE + LRU_ACTIVE_FILE);
|
| 3005 | inactive = global_node_page_state(NR_LRU_BASE + LRU_INACTIVE_ANON) +
|
| 3006 | global_node_page_state(NR_LRU_BASE + LRU_INACTIVE_FILE);
|
| 3007 | total = active + inactive;
|
| 3008 | if (active_ratio)
|
| 3009 | return mult_frac(active, 10000, total);
|
| 3010 | return mult_frac(inactive, 10000, total);
|
| 3011 | }
|
| 3012 |
|
| 3013 | static void damos_set_quota_goal_current_value(struct damon_ctx *c,
|
| 3014 | struct damos *s, struct damos_quota_goal *goal)
|
| 3015 | {
|
| 3016 | u64 now_psi_total;
|
| 3017 |
|
| 3018 | switch (goal->metric) {
|
| 3019 | case DAMOS_QUOTA_USER_INPUT:
|
| 3020 |
|
| 3021 | break;
|
| 3022 | case DAMOS_QUOTA_SOME_MEM_PSI_US:
|
| 3023 | now_psi_total = damos_get_some_mem_psi_total();
|
| 3024 | goal->current_value = now_psi_total - goal->last_psi_total;
|
| 3025 | goal->last_psi_total = now_psi_total;
|
| 3026 | break;
|
| 3027 | case DAMOS_QUOTA_NODE_MEM_USED_BP:
|
| 3028 | case DAMOS_QUOTA_NODE_MEM_FREE_BP:
|
| 3029 | goal->current_value = damos_get_node_mem_bp(goal);
|
| 3030 | break;
|
| 3031 | case DAMOS_QUOTA_NODE_MEMCG_USED_BP:
|
| 3032 | case DAMOS_QUOTA_NODE_MEMCG_FREE_BP:
|
| 3033 | goal->current_value = damos_get_node_memcg_used_bp(goal);
|
| 3034 | break;
|
| 3035 | case DAMOS_QUOTA_ACTIVE_MEM_BP:
|
| 3036 | case DAMOS_QUOTA_INACTIVE_MEM_BP:
|
| 3037 | goal->current_value = damos_get_in_active_mem_bp(
|
| 3038 | goal->metric == DAMOS_QUOTA_ACTIVE_MEM_BP);
|
| 3039 | break;
|
| 3040 | case DAMOS_QUOTA_NODE_ELIGIBLE_MEM_BP:
|
| 3041 | goal->current_value = damos_get_node_eligible_mem_bp(c, s,
|
| 3042 | goal->nid);
|
| 3043 | break;
|
| 3044 | default:
|
| 3045 | break;
|
| 3046 | }
|
| 3047 | }
|
| 3048 |
|
| 3049 |
|
| 3050 | static unsigned long damos_quota_score(struct damon_ctx *c, struct damos *s)
|
| 3051 | {
|
| 3052 | struct damos_quota_goal *goal;
|
| 3053 | struct damos_quota *quota = &s->quota;
|
| 3054 | unsigned long highest_score = 0;
|
| 3055 |
|
| 3056 | damos_for_each_quota_goal(goal, quota) {
|
| 3057 | damos_set_quota_goal_current_value(c, s, goal);
|
| 3058 | highest_score = max(highest_score,
|
| 3059 | mult_frac(goal->current_value, 10000,
|
| 3060 | goal->target_value));
|
| 3061 | }
|
| 3062 |
|
| 3063 | return highest_score;
|
| 3064 | }
|
| 3065 |
|
| 3066 | static void damos_goal_tune_esz_bp_consist(struct damon_ctx *c, struct damos *s)
|
| 3067 | {
|
| 3068 | struct damos_quota *quota = &s->quota;
|
| 3069 | unsigned long score = damos_quota_score(c, s);
|
| 3070 |
|
| 3071 | quota->esz_bp = damon_feed_loop_next_input(
|
| 3072 | max(quota->esz_bp, 10000UL), score);
|
| 3073 | }
|
| 3074 |
|
| 3075 | static void damos_goal_tune_esz_bp_temporal(struct damon_ctx *c,
|
| 3076 | struct damos *s)
|
| 3077 | {
|
| 3078 | struct damos_quota *quota = &s->quota;
|
| 3079 | unsigned long score = damos_quota_score(c, s);
|
| 3080 |
|
| 3081 | if (score >= 10000)
|
| 3082 | quota->esz_bp = 0;
|
| 3083 | else if (quota->sz)
|
| 3084 | quota->esz_bp = quota->sz * 10000;
|
| 3085 | else
|
| 3086 | quota->esz_bp = ULONG_MAX;
|
| 3087 | }
|
| 3088 |
|
| 3089 | |
| 3090 | * Called only if quota->ms, or quota->sz are set, or quota->goals is not empty
|
| 3091 |
|
| 3092 | static void damos_set_effective_quota(struct damon_ctx *ctx, struct damos *s)
|
| 3093 | {
|
| 3094 | struct damos_quota *quota = &s->quota;
|
| 3095 | unsigned long throughput;
|
| 3096 | unsigned long esz = ULONG_MAX;
|
| 3097 | unsigned long esz_time;
|
| 3098 |
|
| 3099 | if (!quota->ms && list_empty("a->goals)) {
|
| 3100 | quota->esz = quota->sz;
|
| 3101 | return;
|
| 3102 | }
|
| 3103 |
|
| 3104 | if (!list_empty("a->goals)) {
|
| 3105 | if (quota->goal_tuner == DAMOS_QUOTA_GOAL_TUNER_CONSIST)
|
| 3106 | damos_goal_tune_esz_bp_consist(ctx, s);
|
| 3107 | else if (quota->goal_tuner == DAMOS_QUOTA_GOAL_TUNER_TEMPORAL)
|
| 3108 | damos_goal_tune_esz_bp_temporal(ctx, s);
|
| 3109 | esz = quota->esz_bp / 10000;
|
| 3110 | }
|
| 3111 |
|
| 3112 | if (quota->ms) {
|
| 3113 | if (quota->total_charged_ns)
|
| 3114 | throughput = mult_frac(quota->total_charged_sz,
|
| 3115 | 1000000, quota->total_charged_ns);
|
| 3116 | else
|
| 3117 | throughput = PAGE_SIZE * 1024;
|
| 3118 | esz_time = max(throughput * quota->ms, ctx->min_region_sz);
|
| 3119 | esz = min(esz_time, esz);
|
| 3120 | }
|
| 3121 |
|
| 3122 | if (quota->sz && quota->sz < esz)
|
| 3123 | esz = quota->sz;
|
| 3124 |
|
| 3125 | quota->esz = esz;
|
| 3126 | }
|
| 3127 |
|
| 3128 | static void damos_trace_esz(struct damon_ctx *c, struct damos *s,
|
| 3129 | struct damos_quota *quota)
|
| 3130 | {
|
| 3131 | unsigned int cidx = 0, sidx = 0;
|
| 3132 | struct damos *siter;
|
| 3133 |
|
| 3134 | damon_for_each_scheme(siter, c) {
|
| 3135 | if (siter == s)
|
| 3136 | break;
|
| 3137 | sidx++;
|
| 3138 | }
|
| 3139 | trace_damos_esz(cidx, sidx, quota->esz);
|
| 3140 | }
|
| 3141 |
|
| 3142 | static void damos_adjust_quota(struct damon_ctx *c, struct damos *s)
|
| 3143 | {
|
| 3144 | struct damos_quota *quota = &s->quota;
|
| 3145 | struct damon_target *t;
|
| 3146 | struct damon_region *r;
|
| 3147 | unsigned long cumulated_sz, cached_esz;
|
| 3148 | unsigned int score, max_score = 0;
|
| 3149 |
|
| 3150 | if (!damos_quota_is_set(quota))
|
| 3151 | return;
|
| 3152 |
|
| 3153 |
|
| 3154 | if (!quota->total_charged_sz && !quota->charged_from) {
|
| 3155 | quota->charged_from = jiffies;
|
| 3156 | damos_set_effective_quota(c, s);
|
| 3157 | if (trace_damos_esz_enabled())
|
| 3158 | damos_trace_esz(c, s, quota);
|
| 3159 | }
|
| 3160 |
|
| 3161 |
|
| 3162 | if (!time_in_range_open(jiffies, quota->charged_from,
|
| 3163 | quota->charged_from +
|
| 3164 | msecs_to_jiffies(quota->reset_interval))) {
|
| 3165 | if (damos_quota_is_full(quota, c->min_region_sz))
|
| 3166 | s->stat.qt_exceeds++;
|
| 3167 | quota->total_charged_sz += quota->charged_sz;
|
| 3168 | quota->charged_from = jiffies;
|
| 3169 | quota->charged_sz = 0;
|
| 3170 | if (trace_damos_esz_enabled())
|
| 3171 | cached_esz = quota->esz;
|
| 3172 | damos_set_effective_quota(c, s);
|
| 3173 | if (trace_damos_esz_enabled() && quota->esz != cached_esz)
|
| 3174 | damos_trace_esz(c, s, quota);
|
| 3175 | }
|
| 3176 |
|
| 3177 | if (!c->ops.get_scheme_score)
|
| 3178 | return;
|
| 3179 |
|
| 3180 |
|
| 3181 | memset(c->regions_score_histogram, 0,
|
| 3182 | sizeof(*c->regions_score_histogram) *
|
| 3183 | (DAMOS_MAX_SCORE + 1));
|
| 3184 | damon_for_each_target(t, c) {
|
| 3185 | damon_for_each_region(r, t) {
|
| 3186 | if (!__damos_valid_target(r, s, c))
|
| 3187 | continue;
|
| 3188 | if (damos_core_filter_out(c, t, r, s))
|
| 3189 | continue;
|
| 3190 | score = c->ops.get_scheme_score(c, r, s);
|
| 3191 | c->regions_score_histogram[score] +=
|
| 3192 | damon_sz_region(r);
|
| 3193 | if (score > max_score)
|
| 3194 | max_score = score;
|
| 3195 | }
|
| 3196 | }
|
| 3197 |
|
| 3198 |
|
| 3199 | for (cumulated_sz = 0, score = max_score; ; score--) {
|
| 3200 | cumulated_sz += c->regions_score_histogram[score];
|
| 3201 | if (cumulated_sz >= quota->esz || !score)
|
| 3202 | break;
|
| 3203 | }
|
| 3204 | quota->min_score = score;
|
| 3205 | }
|
| 3206 |
|
| 3207 | static void damos_trace_stat(struct damon_ctx *c, struct damos *s)
|
| 3208 | {
|
| 3209 | unsigned int cidx = 0, sidx = 0;
|
| 3210 | struct damos *siter;
|
| 3211 |
|
| 3212 | if (!trace_damos_stat_after_apply_interval_enabled())
|
| 3213 | return;
|
| 3214 |
|
| 3215 | damon_for_each_scheme(siter, c) {
|
| 3216 | if (siter == s)
|
| 3217 | break;
|
| 3218 | sidx++;
|
| 3219 | }
|
| 3220 | trace_call__damos_stat_after_apply_interval(cidx, sidx, &s->stat);
|
| 3221 | }
|
| 3222 |
|
| 3223 | static void kdamond_apply_schemes(struct damon_ctx *c)
|
| 3224 | {
|
| 3225 | struct damon_target *t;
|
| 3226 | struct damos *s;
|
| 3227 | bool has_schemes_to_apply = false;
|
| 3228 | unsigned long max_region_sz;
|
| 3229 |
|
| 3230 | damon_for_each_scheme(s, c) {
|
| 3231 | if (time_before(c->passed_sample_intervals, s->next_apply_sis))
|
| 3232 | continue;
|
| 3233 |
|
| 3234 | if (!s->wmarks.activated)
|
| 3235 | continue;
|
| 3236 |
|
| 3237 | has_schemes_to_apply = true;
|
| 3238 |
|
| 3239 | damos_adjust_quota(c, s);
|
| 3240 | }
|
| 3241 |
|
| 3242 | if (!has_schemes_to_apply)
|
| 3243 | return;
|
| 3244 |
|
| 3245 | max_region_sz = damon_region_sz_limit(c);
|
| 3246 | mutex_lock(&c->walk_control_lock);
|
| 3247 | damon_for_each_target(t, c) {
|
| 3248 | if (c->ops.target_valid && c->ops.target_valid(t) == false) {
|
| 3249 | damon_for_each_scheme(s, c) {
|
| 3250 | if (s->quota.charge_target_from != t)
|
| 3251 | continue;
|
| 3252 | s->quota.charge_target_from = NULL;
|
| 3253 | s->quota.charge_addr_from = 0;
|
| 3254 | }
|
| 3255 | continue;
|
| 3256 | }
|
| 3257 | damos_apply_target(c, t, max_region_sz);
|
| 3258 | }
|
| 3259 |
|
| 3260 | damon_for_each_scheme(s, c) {
|
| 3261 | if (time_before(c->passed_sample_intervals, s->next_apply_sis))
|
| 3262 | continue;
|
| 3263 | damos_walk_complete(c, s);
|
| 3264 | damos_set_next_apply_sis(s, c);
|
| 3265 | s->last_applied = NULL;
|
| 3266 | damos_trace_stat(c, s);
|
| 3267 | }
|
| 3268 | mutex_unlock(&c->walk_control_lock);
|
| 3269 | }
|
| 3270 |
|
| 3271 | #ifdef CONFIG_DAMON_DEBUG_SANITY
|
| 3272 | static void damon_verify_merge_two_regions(
|
| 3273 | struct damon_region *l, struct damon_region *r)
|
| 3274 | {
|
| 3275 |
|
| 3276 | WARN_ONCE(l->ar.start >= l->ar.end, "l: %lu-%lu, r: %lu-%lu\n",
|
| 3277 | l->ar.start, l->ar.end, r->ar.start, r->ar.end);
|
| 3278 | }
|
| 3279 | #else
|
| 3280 | static void damon_verify_merge_two_regions(
|
| 3281 | struct damon_region *l, struct damon_region *r)
|
| 3282 | {
|
| 3283 | }
|
| 3284 | #endif
|
| 3285 |
|
| 3286 | |
| 3287 | * Merge two adjacent regions into one region
|
| 3288 |
|
| 3289 | static void damon_merge_two_regions(struct damon_target *t,
|
| 3290 | struct damon_region *l, struct damon_region *r)
|
| 3291 | {
|
| 3292 | unsigned long sz_l = damon_sz_region(l), sz_r = damon_sz_region(r);
|
| 3293 | int i;
|
| 3294 |
|
| 3295 | l->nr_accesses = (l->nr_accesses * sz_l + r->nr_accesses * sz_r) /
|
| 3296 | (sz_l + sz_r);
|
| 3297 | l->age = (l->age * sz_l + r->age * sz_r) / (sz_l + sz_r);
|
| 3298 | l->ar.end = r->ar.end;
|
| 3299 |
|
| 3300 | for (i = 0; i < DAMON_MAX_PROBES; i++)
|
| 3301 | l->probe_hits[i] = (l->probe_hits[i] * sz_l + r->probe_hits[i]
|
| 3302 | * sz_r) / (sz_l + sz_r);
|
| 3303 | damon_verify_merge_two_regions(l, r);
|
| 3304 | damon_destroy_region(r, t);
|
| 3305 | }
|
| 3306 |
|
| 3307 | static unsigned int damon_merge_score(struct damon_region *r, bool last,
|
| 3308 | struct damon_ctx *ctx, bool use_probe_hits)
|
| 3309 | {
|
| 3310 | if (use_probe_hits)
|
| 3311 | return damon_probe_hits_wsum(r, last, ctx);
|
| 3312 | if (last)
|
| 3313 | return r->last_nr_accesses;
|
| 3314 | return r->nr_accesses;
|
| 3315 | }
|
| 3316 |
|
| 3317 | |
| 3318 | * Merge adjacent regions having similar access frequencies
|
| 3319 | *
|
| 3320 | * t target affected by this merge operation
|
| 3321 | * thres '->nr_accesses' diff threshold for the merge
|
| 3322 | * sz_limit size upper limit of each region
|
| 3323 |
|
| 3324 | static void damon_merge_regions_of(struct damon_target *t, unsigned int thres,
|
| 3325 | unsigned long sz_limit, struct damon_ctx *ctx, bool count_age)
|
| 3326 | {
|
| 3327 | struct damon_region *r, *prev = NULL, *next;
|
| 3328 | bool use_probe_hits = damon_has_probe_weights(ctx);
|
| 3329 |
|
| 3330 | damon_for_each_region_safe(r, next, t) {
|
| 3331 | unsigned int score, last_score, diff;
|
| 3332 |
|
| 3333 | score = damon_merge_score(r, false, ctx, use_probe_hits);
|
| 3334 | last_score = damon_merge_score(r, true, ctx, use_probe_hits);
|
| 3335 |
|
| 3336 | if (count_age) {
|
| 3337 | if (abs_diff(score, last_score) > thres)
|
| 3338 | r->age = 0;
|
| 3339 | else if ((score == 0) != (last_score == 0))
|
| 3340 | r->age = 0;
|
| 3341 | else
|
| 3342 | r->age++;
|
| 3343 | }
|
| 3344 |
|
| 3345 | if (!prev)
|
| 3346 | goto set_prev_continue;
|
| 3347 | if (prev->ar.end != r->ar.start)
|
| 3348 | goto set_prev_continue;
|
| 3349 | diff = abs_diff(score, damon_merge_score(prev, false, ctx,
|
| 3350 | use_probe_hits));
|
| 3351 | if (diff > thres)
|
| 3352 | goto set_prev_continue;
|
| 3353 | if (damon_sz_region(prev) + damon_sz_region(r) > sz_limit)
|
| 3354 | goto set_prev_continue;
|
| 3355 | damon_merge_two_regions(t, prev, r);
|
| 3356 | continue;
|
| 3357 | set_prev_continue:
|
| 3358 | prev = r;
|
| 3359 | }
|
| 3360 | }
|
| 3361 |
|
| 3362 | |
| 3363 | * Merge adjacent regions having similar access frequencies
|
| 3364 | *
|
| 3365 | * threshold '->nr_accesses' diff threshold for the merge
|
| 3366 | * sz_limit size upper limit of each region
|
| 3367 | *
|
| 3368 | * This function merges monitoring target regions which are adjacent and their
|
| 3369 | * access frequencies are similar. This is for minimizing the monitoring
|
| 3370 | * overhead under the dynamically changeable access pattern. If a merge was
|
| 3371 | * unnecessarily made, later 'kdamond_split_regions()' will revert it.
|
| 3372 | *
|
| 3373 | * The total number of regions could be higher than the user-defined limit,
|
| 3374 | * max_nr_regions for some cases. For example, the user can update
|
| 3375 | * max_nr_regions to a number that lower than the current number of regions
|
| 3376 | * while DAMON is running. For such a case, repeat merging until the limit is
|
| 3377 | * met while increasing @threshold up to possible maximum level.
|
| 3378 |
|
| 3379 | static void kdamond_merge_regions(struct damon_ctx *c, unsigned int threshold,
|
| 3380 | unsigned long sz_limit)
|
| 3381 | {
|
| 3382 | struct damon_target *t;
|
| 3383 | unsigned int nr_regions;
|
| 3384 | unsigned int max_thres;
|
| 3385 | bool count_age = true;
|
| 3386 |
|
| 3387 | max_thres = c->attrs.aggr_interval /
|
| 3388 | (c->attrs.sample_interval ? c->attrs.sample_interval : 1);
|
| 3389 | while (true) {
|
| 3390 | nr_regions = 0;
|
| 3391 | damon_for_each_target(t, c) {
|
| 3392 | damon_merge_regions_of(t, threshold, sz_limit, c,
|
| 3393 | count_age);
|
| 3394 | nr_regions += damon_nr_regions(t);
|
| 3395 | }
|
| 3396 | count_age = false;
|
| 3397 | if (nr_regions <= c->attrs.max_nr_regions ||
|
| 3398 | max_thres <= threshold)
|
| 3399 | break;
|
| 3400 | if (threshold < max_thres / 2)
|
| 3401 | threshold = max(1, threshold * 2);
|
| 3402 | else
|
| 3403 | threshold = max_thres;
|
| 3404 | }
|
| 3405 | }
|
| 3406 |
|
| 3407 | #ifdef CONFIG_DAMON_DEBUG_SANITY
|
| 3408 | static void damon_verify_split_region_at(struct damon_region *r,
|
| 3409 | unsigned long sz_r)
|
| 3410 | {
|
| 3411 | WARN_ONCE(sz_r == 0 || sz_r >= damon_sz_region(r),
|
| 3412 | "sz_r: %lu r: %lu-%lu (%lu)\n",
|
| 3413 | sz_r, r->ar.start, r->ar.end, damon_sz_region(r));
|
| 3414 | }
|
| 3415 | #else
|
| 3416 | static void damon_verify_split_region_at(struct damon_region *r,
|
| 3417 | unsigned long sz_r)
|
| 3418 | {
|
| 3419 | }
|
| 3420 | #endif
|
| 3421 |
|
| 3422 | |
| 3423 | * Split a region in two
|
| 3424 | *
|
| 3425 | * r the region to be split
|
| 3426 | * sz_r size of the first sub-region that will be made
|
| 3427 | *
|
| 3428 | * Return: 0 on success, negative error code otherwise.
|
| 3429 |
|
| 3430 | static int damon_split_region_at(struct damon_target *t,
|
| 3431 | struct damon_region *r, unsigned long sz_r)
|
| 3432 | {
|
| 3433 | struct damon_region *new;
|
| 3434 |
|
| 3435 | damon_verify_split_region_at(r, sz_r);
|
| 3436 | new = damon_new_region(r->ar.start + sz_r, r->ar.end);
|
| 3437 | if (!new)
|
| 3438 | return -ENOMEM;
|
| 3439 |
|
| 3440 | r->ar.end = new->ar.start;
|
| 3441 |
|
| 3442 | new->age = r->age;
|
| 3443 | new->last_nr_accesses = r->last_nr_accesses;
|
| 3444 | new->nr_accesses = r->nr_accesses;
|
| 3445 |
|
| 3446 | memcpy(new->probe_hits, r->probe_hits, sizeof(r->probe_hits));
|
| 3447 | memcpy(new->last_probe_hits, r->last_probe_hits,
|
| 3448 | sizeof(r->last_probe_hits));
|
| 3449 |
|
| 3450 | damon_insert_region(new, r, damon_next_region(r), t);
|
| 3451 | return 0;
|
| 3452 | }
|
| 3453 |
|
| 3454 |
|
| 3455 | static void damon_split_regions_of(struct damon_ctx *ctx,
|
| 3456 | struct damon_target *t, int nr_subs,
|
| 3457 | unsigned long min_region_sz)
|
| 3458 | {
|
| 3459 | struct damon_region *r, *next;
|
| 3460 | unsigned long sz_region, sz_sub = 0;
|
| 3461 | int i;
|
| 3462 |
|
| 3463 | damon_for_each_region_safe(r, next, t) {
|
| 3464 | sz_region = damon_sz_region(r);
|
| 3465 |
|
| 3466 | for (i = 0; i < nr_subs - 1 &&
|
| 3467 | sz_region > 2 * min_region_sz; i++) {
|
| 3468 | |
| 3469 | * Randomly select size of left sub-region to be at
|
| 3470 | * least 10 percent and at most 90% of original region
|
| 3471 |
|
| 3472 | sz_sub = ALIGN_DOWN(damon_rand(ctx, 1, 10) *
|
| 3473 | sz_region / 10, min_region_sz);
|
| 3474 |
|
| 3475 | if (sz_sub == 0 || sz_sub >= sz_region)
|
| 3476 | continue;
|
| 3477 |
|
| 3478 | damon_split_region_at(t, r, sz_sub);
|
| 3479 | sz_region = sz_sub;
|
| 3480 | }
|
| 3481 | }
|
| 3482 | }
|
| 3483 |
|
| 3484 |
|
| 3485 | static void damon_split_some_regions(struct damon_ctx *ctx,
|
| 3486 | unsigned long split_step)
|
| 3487 | {
|
| 3488 | static unsigned long rotation;
|
| 3489 | struct damon_target *t;
|
| 3490 | struct damon_region *r, *next;
|
| 3491 | unsigned long offset = rotation++ % split_step;
|
| 3492 | unsigned long idx = 0;
|
| 3493 |
|
| 3494 | damon_for_each_target(t, ctx) {
|
| 3495 | damon_for_each_region_safe(r, next, t) {
|
| 3496 | unsigned long sz_region, sz_sub;
|
| 3497 |
|
| 3498 | if (idx++ % split_step != offset)
|
| 3499 | continue;
|
| 3500 | sz_region = damon_sz_region(r);
|
| 3501 | if (sz_region < 2 * ctx->min_region_sz)
|
| 3502 | continue;
|
| 3503 |
|
| 3504 | sz_sub = ALIGN_DOWN(damon_rand(ctx, 1, 10) *
|
| 3505 | sz_region / 10, ctx->min_region_sz);
|
| 3506 |
|
| 3507 | if (sz_sub == 0 || sz_sub >= sz_region)
|
| 3508 | continue;
|
| 3509 |
|
| 3510 | damon_split_region_at(t, r, sz_sub);
|
| 3511 | }
|
| 3512 | }
|
| 3513 | }
|
| 3514 |
|
| 3515 | |
| 3516 | * Split every target region into randomly-sized small regions
|
| 3517 | *
|
| 3518 | * This function splits every target region into random-sized small regions if
|
| 3519 | * current total number of the regions is equal or smaller than half of the
|
| 3520 | * user-specified maximum number of regions. This is for maximizing the
|
| 3521 | * monitoring accuracy under the dynamically changeable access patterns. If a
|
| 3522 | * split was unnecessarily made, later 'kdamond_merge_regions()' will revert
|
| 3523 | * it.
|
| 3524 |
|
| 3525 | static void kdamond_split_regions(struct damon_ctx *ctx)
|
| 3526 | {
|
| 3527 | struct damon_target *t;
|
| 3528 | unsigned long nr_regions = 0;
|
| 3529 | unsigned long max_nr_regions = ctx->attrs.max_nr_regions;
|
| 3530 | static unsigned long last_nr_regions;
|
| 3531 | int nr_subregions = 2;
|
| 3532 |
|
| 3533 | damon_for_each_target(t, ctx)
|
| 3534 | nr_regions += damon_nr_regions(t);
|
| 3535 |
|
| 3536 | if (nr_regions >= max_nr_regions)
|
| 3537 | goto done;
|
| 3538 |
|
| 3539 | if (nr_regions > max_nr_regions / 2) {
|
| 3540 | damon_split_some_regions(ctx,
|
| 3541 | max_nr_regions / (max_nr_regions - nr_regions));
|
| 3542 | goto done;
|
| 3543 | }
|
| 3544 |
|
| 3545 |
|
| 3546 | if (last_nr_regions == nr_regions &&
|
| 3547 | nr_regions < max_nr_regions / 3)
|
| 3548 | nr_subregions = 3;
|
| 3549 |
|
| 3550 | damon_for_each_target(t, ctx)
|
| 3551 | damon_split_regions_of(ctx, t, nr_subregions,
|
| 3552 | ctx->min_region_sz);
|
| 3553 |
|
| 3554 | done:
|
| 3555 | last_nr_regions = nr_regions;
|
| 3556 | }
|
| 3557 |
|
| 3558 | |
| 3559 | * Check whether current monitoring should be stopped
|
| 3560 | *
|
| 3561 | * The monitoring is stopped when either the user requested to stop, or all
|
| 3562 | * monitoring targets are invalid.
|
| 3563 | *
|
| 3564 | * Returns true if need to stop current monitoring.
|
| 3565 |
|
| 3566 | static bool kdamond_need_stop(struct damon_ctx *ctx)
|
| 3567 | {
|
| 3568 | struct damon_target *t;
|
| 3569 |
|
| 3570 | if (kthread_should_stop())
|
| 3571 | return true;
|
| 3572 |
|
| 3573 | if (!ctx->ops.target_valid)
|
| 3574 | return false;
|
| 3575 |
|
| 3576 | damon_for_each_target(t, ctx) {
|
| 3577 | if (ctx->ops.target_valid(t))
|
| 3578 | return false;
|
| 3579 | }
|
| 3580 |
|
| 3581 | return true;
|
| 3582 | }
|
| 3583 |
|
| 3584 | static int damos_get_wmark_metric_value(enum damos_wmark_metric metric,
|
| 3585 | unsigned long *metric_value)
|
| 3586 | {
|
| 3587 | switch (metric) {
|
| 3588 | case DAMOS_WMARK_FREE_MEM_RATE:
|
| 3589 | *metric_value = global_zone_page_state(NR_FREE_PAGES) * 1000 /
|
| 3590 | totalram_pages();
|
| 3591 | return 0;
|
| 3592 | default:
|
| 3593 | break;
|
| 3594 | }
|
| 3595 | return -EINVAL;
|
| 3596 | }
|
| 3597 |
|
| 3598 | |
| 3599 | * Returns zero if the scheme is active. Else, returns time to wait for next
|
| 3600 | * watermark check in micro-seconds.
|
| 3601 |
|
| 3602 | static unsigned long damos_wmark_wait_us(struct damos *scheme)
|
| 3603 | {
|
| 3604 | unsigned long metric;
|
| 3605 |
|
| 3606 | if (damos_get_wmark_metric_value(scheme->wmarks.metric, &metric))
|
| 3607 | return 0;
|
| 3608 |
|
| 3609 |
|
| 3610 | if (metric > scheme->wmarks.high || scheme->wmarks.low > metric) {
|
| 3611 | if (scheme->wmarks.activated)
|
| 3612 | pr_debug("deactivate a scheme (%d) for %s wmark\n",
|
| 3613 | scheme->action,
|
| 3614 | str_high_low(metric > scheme->wmarks.high));
|
| 3615 | scheme->wmarks.activated = false;
|
| 3616 | return scheme->wmarks.interval;
|
| 3617 | }
|
| 3618 |
|
| 3619 |
|
| 3620 | if ((scheme->wmarks.high >= metric && metric >= scheme->wmarks.mid) &&
|
| 3621 | !scheme->wmarks.activated)
|
| 3622 | return scheme->wmarks.interval;
|
| 3623 |
|
| 3624 | if (!scheme->wmarks.activated)
|
| 3625 | pr_debug("activate a scheme (%d)\n", scheme->action);
|
| 3626 | scheme->wmarks.activated = true;
|
| 3627 | return 0;
|
| 3628 | }
|
| 3629 |
|
| 3630 | static void kdamond_usleep(unsigned long usecs)
|
| 3631 | {
|
| 3632 | if (usecs >= USLEEP_RANGE_UPPER_BOUND)
|
| 3633 | schedule_timeout_idle(usecs_to_jiffies(usecs));
|
| 3634 | else
|
| 3635 | usleep_range_idle(usecs, usecs + 1);
|
| 3636 | }
|
| 3637 |
|
| 3638 | #ifdef CONFIG_DAMON_DEBUG_SANITY
|
| 3639 | static void damon_verify_ctx(struct damon_ctx *c)
|
| 3640 | {
|
| 3641 | struct damon_target *t;
|
| 3642 | struct damon_region *r;
|
| 3643 |
|
| 3644 | damon_for_each_target(t, c) {
|
| 3645 | struct damon_region *prev_r = NULL;
|
| 3646 | unsigned int nr_regions = 0;
|
| 3647 |
|
| 3648 | damon_for_each_region(r, t) {
|
| 3649 | WARN_ONCE(r->ar.start >= r->ar.end,
|
| 3650 | "region start (%lu) >= end (%lu)\n",
|
| 3651 | r->ar.start, r->ar.end);
|
| 3652 | WARN_ONCE(prev_r && prev_r->ar.end > r->ar.start,
|
| 3653 | "region overlap (%lu > %lu)\n",
|
| 3654 | prev_r->ar.end, r->ar.start);
|
| 3655 | prev_r = r;
|
| 3656 | nr_regions++;
|
| 3657 | }
|
| 3658 | WARN_ONCE(damon_nr_regions(t) != nr_regions,
|
| 3659 | "nr_regions mismatch: %u != %u\n",
|
| 3660 | damon_nr_regions(t), nr_regions);
|
| 3661 | }
|
| 3662 | }
|
| 3663 | #else
|
| 3664 | static void damon_verify_ctx(struct damon_ctx *c)
|
| 3665 | {
|
| 3666 | }
|
| 3667 | #endif
|
| 3668 |
|
| 3669 | |
| 3670 | * kdamond_call() - handle damon_call_control objects.
|
| 3671 | * @ctx: The &struct damon_ctx of the kdamond.
|
| 3672 | * @cancel: Whether to cancel the invocation of the function.
|
| 3673 | *
|
| 3674 | * If there are &struct damon_call_control requests that registered via
|
| 3675 | * &damon_call() on @ctx, do or cancel the invocation of the function depending
|
| 3676 | * on @cancel. @cancel is set when the kdamond is already out of the main loop
|
| 3677 | * and therefore will be terminated.
|
| 3678 |
|
| 3679 | static void kdamond_call(struct damon_ctx *ctx, bool cancel)
|
| 3680 | {
|
| 3681 | struct damon_call_control *control, *next;
|
| 3682 | LIST_HEAD(controls);
|
| 3683 |
|
| 3684 | damon_verify_ctx(ctx);
|
| 3685 |
|
| 3686 | mutex_lock(&ctx->call_controls_lock);
|
| 3687 | list_splice_tail_init(&ctx->call_controls, &controls);
|
| 3688 | mutex_unlock(&ctx->call_controls_lock);
|
| 3689 |
|
| 3690 | list_for_each_entry_safe(control, next, &controls, list) {
|
| 3691 | if (!control->repeat || cancel)
|
| 3692 | list_del(&control->list);
|
| 3693 |
|
| 3694 | if (cancel)
|
| 3695 | control->canceled = true;
|
| 3696 | else
|
| 3697 | control->return_code = control->fn(control->data);
|
| 3698 |
|
| 3699 | if (!control->repeat)
|
| 3700 | complete(&control->completion);
|
| 3701 | else if (control->canceled && control->dealloc_on_cancel)
|
| 3702 | kfree(control);
|
| 3703 | if (!cancel && ctx->maybe_corrupted)
|
| 3704 | break;
|
| 3705 | }
|
| 3706 |
|
| 3707 | mutex_lock(&ctx->call_controls_lock);
|
| 3708 | list_splice_tail(&controls, &ctx->call_controls);
|
| 3709 | mutex_unlock(&ctx->call_controls_lock);
|
| 3710 | }
|
| 3711 |
|
| 3712 |
|
| 3713 | static int kdamond_wait_activation(struct damon_ctx *ctx)
|
| 3714 | {
|
| 3715 | struct damos *s;
|
| 3716 | unsigned long wait_time;
|
| 3717 | unsigned long min_wait_time = 0;
|
| 3718 | bool init_wait_time = false;
|
| 3719 |
|
| 3720 | while (!kdamond_need_stop(ctx)) {
|
| 3721 | damon_for_each_scheme(s, ctx) {
|
| 3722 | wait_time = damos_wmark_wait_us(s);
|
| 3723 | if (!init_wait_time || wait_time < min_wait_time) {
|
| 3724 | init_wait_time = true;
|
| 3725 | min_wait_time = wait_time;
|
| 3726 | }
|
| 3727 | }
|
| 3728 | if (!min_wait_time)
|
| 3729 | return 0;
|
| 3730 |
|
| 3731 | kdamond_usleep(min_wait_time);
|
| 3732 |
|
| 3733 | kdamond_call(ctx, false);
|
| 3734 | if (ctx->maybe_corrupted)
|
| 3735 | return -EINVAL;
|
| 3736 | damos_walk_cancel(ctx);
|
| 3737 | }
|
| 3738 | return -EBUSY;
|
| 3739 | }
|
| 3740 |
|
| 3741 | static void kdamond_init_ctx(struct damon_ctx *ctx)
|
| 3742 | {
|
| 3743 | unsigned long sample_interval = ctx->attrs.sample_interval ?
|
| 3744 | ctx->attrs.sample_interval : 1;
|
| 3745 | struct damos *scheme;
|
| 3746 |
|
| 3747 | ctx->passed_sample_intervals = 0;
|
| 3748 | ctx->next_aggregation_sis = ctx->attrs.aggr_interval / sample_interval;
|
| 3749 | ctx->next_ops_update_sis = ctx->attrs.ops_update_interval /
|
| 3750 | sample_interval;
|
| 3751 | ctx->next_intervals_tune_sis = ctx->next_aggregation_sis *
|
| 3752 | ctx->attrs.intervals_goal.aggrs;
|
| 3753 |
|
| 3754 | damon_for_each_scheme(scheme, ctx) {
|
| 3755 | damos_set_next_apply_sis(scheme, ctx);
|
| 3756 | damos_set_filters_default_reject(scheme);
|
| 3757 | }
|
| 3758 | }
|
| 3759 |
|
| 3760 | |
| 3761 | * The monitoring daemon that runs as a kernel thread
|
| 3762 |
|
| 3763 | static int kdamond_fn(void *data)
|
| 3764 | {
|
| 3765 | struct damon_ctx *ctx = data;
|
| 3766 | unsigned long sz_limit = 0;
|
| 3767 |
|
| 3768 | pr_debug("kdamond (%d) starts\n", current->pid);
|
| 3769 |
|
| 3770 | mutex_lock(&ctx->call_controls_lock);
|
| 3771 | ctx->call_controls_obsolete = false;
|
| 3772 | mutex_unlock(&ctx->call_controls_lock);
|
| 3773 | mutex_lock(&ctx->walk_control_lock);
|
| 3774 | ctx->walk_control_obsolete = false;
|
| 3775 | mutex_unlock(&ctx->walk_control_lock);
|
| 3776 | complete(&ctx->kdamond_started);
|
| 3777 | kdamond_init_ctx(ctx);
|
| 3778 |
|
| 3779 | if (ctx->ops.init)
|
| 3780 | ctx->ops.init(ctx);
|
| 3781 | ctx->regions_score_histogram = kmalloc_array(DAMOS_MAX_SCORE + 1,
|
| 3782 | sizeof(*ctx->regions_score_histogram), GFP_KERNEL);
|
| 3783 | if (!ctx->regions_score_histogram)
|
| 3784 | goto done;
|
| 3785 |
|
| 3786 | sz_limit = damon_apply_min_nr_regions(ctx);
|
| 3787 |
|
| 3788 | while (!kdamond_need_stop(ctx)) {
|
| 3789 | |
| 3790 | * ctx->attrs and ctx->next_{aggregation,ops_update}_sis could
|
| 3791 | * be changed from kdamond_call(). Read the values here, and
|
| 3792 | * use those for this iteration. That is, damon_set_attrs()
|
| 3793 | * updated new values are respected from next iteration.
|
| 3794 |
|
| 3795 | unsigned long next_aggregation_sis = ctx->next_aggregation_sis;
|
| 3796 | unsigned long next_ops_update_sis = ctx->next_ops_update_sis;
|
| 3797 | unsigned long sample_interval = ctx->attrs.sample_interval;
|
| 3798 | bool access_check_disabled = damon_has_probe_weights(ctx);
|
| 3799 | unsigned int max_merge_score = 0, max_wsum;
|
| 3800 | bool get_max_wsum;
|
| 3801 |
|
| 3802 | if (kdamond_wait_activation(ctx))
|
| 3803 | break;
|
| 3804 |
|
| 3805 | if (!access_check_disabled && ctx->ops.prepare_access_checks)
|
| 3806 | ctx->ops.prepare_access_checks(ctx);
|
| 3807 |
|
| 3808 | kdamond_usleep(sample_interval);
|
| 3809 | ctx->passed_sample_intervals++;
|
| 3810 |
|
| 3811 | if (!access_check_disabled && ctx->ops.check_accesses)
|
| 3812 | max_merge_score = ctx->ops.check_accesses(ctx);
|
| 3813 | if (ctx->ops.apply_probes) {
|
| 3814 | if (time_after_eq(ctx->passed_sample_intervals,
|
| 3815 | next_aggregation_sis) &&
|
| 3816 | access_check_disabled)
|
| 3817 | get_max_wsum = true;
|
| 3818 | else
|
| 3819 | get_max_wsum = false;
|
| 3820 | max_wsum = ctx->ops.apply_probes(ctx,
|
| 3821 | access_check_disabled, get_max_wsum);
|
| 3822 | if (get_max_wsum)
|
| 3823 | max_merge_score = max_wsum;
|
| 3824 | }
|
| 3825 |
|
| 3826 | if (time_after_eq(ctx->passed_sample_intervals,
|
| 3827 | next_aggregation_sis)) {
|
| 3828 | kdamond_merge_regions(ctx,
|
| 3829 | max_merge_score / 10,
|
| 3830 | sz_limit);
|
| 3831 |
|
| 3832 | sz_limit = damon_apply_min_nr_regions(ctx);
|
| 3833 | }
|
| 3834 |
|
| 3835 | |
| 3836 | * do kdamond_call() and kdamond_apply_schemes() after
|
| 3837 | * kdamond_merge_regions() if possible, to reduce overhead
|
| 3838 |
|
| 3839 | kdamond_call(ctx, false);
|
| 3840 | if (ctx->maybe_corrupted)
|
| 3841 | break;
|
| 3842 | while (ctx->pause) {
|
| 3843 | damos_walk_cancel(ctx);
|
| 3844 | kdamond_usleep(ctx->attrs.sample_interval);
|
| 3845 |
|
| 3846 | kdamond_call(ctx, false);
|
| 3847 | if (kdamond_need_stop(ctx) || ctx->maybe_corrupted)
|
| 3848 | goto done;
|
| 3849 | }
|
| 3850 | if (!list_empty(&ctx->schemes))
|
| 3851 | kdamond_apply_schemes(ctx);
|
| 3852 | else
|
| 3853 | damos_walk_cancel(ctx);
|
| 3854 |
|
| 3855 | sample_interval = ctx->attrs.sample_interval ?
|
| 3856 | ctx->attrs.sample_interval : 1;
|
| 3857 | if (time_after_eq(ctx->passed_sample_intervals,
|
| 3858 | next_aggregation_sis)) {
|
| 3859 | if (ctx->attrs.intervals_goal.aggrs &&
|
| 3860 | time_after_eq(
|
| 3861 | ctx->passed_sample_intervals,
|
| 3862 | ctx->next_intervals_tune_sis)) {
|
| 3863 | |
| 3864 | * ctx->next_aggregation_sis might be updated
|
| 3865 | * from kdamond_call(). In the case,
|
| 3866 | * damon_set_attrs() which will be called from
|
| 3867 | * kdamond_tune_interval() may wrongly think
|
| 3868 | * this is in the middle of the current
|
| 3869 | * aggregation, and make aggregation
|
| 3870 | * information reset for all regions. Then,
|
| 3871 | * following kdamond_reset_aggregated() call
|
| 3872 | * will make the region information invalid.
|
| 3873 | *
|
| 3874 | * Reset ->next_aggregation_sis to avoid that.
|
| 3875 | * It will anyway correctly updated after this
|
| 3876 | * if clause.
|
| 3877 |
|
| 3878 | ctx->next_aggregation_sis =
|
| 3879 | next_aggregation_sis;
|
| 3880 | ctx->next_intervals_tune_sis +=
|
| 3881 | ctx->attrs.aggr_samples *
|
| 3882 | ctx->attrs.intervals_goal.aggrs;
|
| 3883 | kdamond_tune_intervals(ctx);
|
| 3884 | sample_interval = ctx->attrs.sample_interval ?
|
| 3885 | ctx->attrs.sample_interval : 1;
|
| 3886 |
|
| 3887 | }
|
| 3888 | ctx->next_aggregation_sis = next_aggregation_sis +
|
| 3889 | ctx->attrs.aggr_interval / sample_interval;
|
| 3890 |
|
| 3891 | kdamond_reset_aggregated(ctx);
|
| 3892 | kdamond_split_regions(ctx);
|
| 3893 | }
|
| 3894 |
|
| 3895 | if (time_after_eq(ctx->passed_sample_intervals,
|
| 3896 | next_ops_update_sis)) {
|
| 3897 | ctx->next_ops_update_sis = next_ops_update_sis +
|
| 3898 | ctx->attrs.ops_update_interval /
|
| 3899 | sample_interval;
|
| 3900 | if (ctx->ops.update)
|
| 3901 | ctx->ops.update(ctx);
|
| 3902 | }
|
| 3903 | }
|
| 3904 | done:
|
| 3905 | damon_destroy_targets(ctx);
|
| 3906 |
|
| 3907 | kfree(ctx->regions_score_histogram);
|
| 3908 | mutex_lock(&ctx->call_controls_lock);
|
| 3909 | ctx->call_controls_obsolete = true;
|
| 3910 | mutex_unlock(&ctx->call_controls_lock);
|
| 3911 | kdamond_call(ctx, true);
|
| 3912 | mutex_lock(&ctx->walk_control_lock);
|
| 3913 | ctx->walk_control_obsolete = true;
|
| 3914 | mutex_unlock(&ctx->walk_control_lock);
|
| 3915 | damos_walk_cancel(ctx);
|
| 3916 |
|
| 3917 | pr_debug("kdamond (%d) finishes\n", current->pid);
|
| 3918 | mutex_lock(&ctx->kdamond_lock);
|
| 3919 | ctx->kdamond = NULL;
|
| 3920 | mutex_unlock(&ctx->kdamond_lock);
|
| 3921 |
|
| 3922 | mutex_lock(&damon_lock);
|
| 3923 | nr_running_ctxs--;
|
| 3924 | if (!nr_running_ctxs && running_exclusive_ctxs)
|
| 3925 | running_exclusive_ctxs = false;
|
| 3926 | mutex_unlock(&damon_lock);
|
| 3927 |
|
| 3928 | return 0;
|
| 3929 | }
|
| 3930 |
|
| 3931 | struct damon_system_ram_range_walk_arg {
|
| 3932 | bool walked;
|
| 3933 | struct resource res;
|
| 3934 | };
|
| 3935 |
|
| 3936 | static int damon_system_ram_walk_fn(struct resource *res, void *arg)
|
| 3937 | {
|
| 3938 | struct damon_system_ram_range_walk_arg *a = arg;
|
| 3939 |
|
| 3940 | if (!a->walked) {
|
| 3941 | a->walked = true;
|
| 3942 | a->res.start = res->start;
|
| 3943 | }
|
| 3944 | a->res.end = res->end;
|
| 3945 | return 0;
|
| 3946 | }
|
| 3947 |
|
| 3948 | static unsigned long damon_res_to_core_addr(resource_size_t ra,
|
| 3949 | unsigned long addr_unit)
|
| 3950 | {
|
| 3951 | |
| 3952 | * Use div_u64() for avoiding linking errors related with __udivdi3,
|
| 3953 | * __aeabi_uldivmod, or similar problems. This should also improve the
|
| 3954 | * performance optimization (read div_u64() comment for the detail).
|
| 3955 |
|
| 3956 | if (sizeof(ra) == 8 && sizeof(addr_unit) == 4)
|
| 3957 | return div_u64(ra, addr_unit);
|
| 3958 | return ra / addr_unit;
|
| 3959 | }
|
| 3960 |
|
| 3961 | static bool damon_find_system_rams_range(unsigned long *start,
|
| 3962 | unsigned long *end, unsigned long addr_unit)
|
| 3963 | {
|
| 3964 | struct damon_system_ram_range_walk_arg arg = {};
|
| 3965 |
|
| 3966 | walk_system_ram_res(0, -1, &arg, damon_system_ram_walk_fn);
|
| 3967 | if (!arg.walked)
|
| 3968 | return false;
|
| 3969 | *start = damon_res_to_core_addr(arg.res.start, addr_unit);
|
| 3970 | *end = damon_res_to_core_addr(arg.res.end + 1, addr_unit);
|
| 3971 | if (*end <= *start)
|
| 3972 | return false;
|
| 3973 | return true;
|
| 3974 | }
|
| 3975 |
|
| 3976 | |
| 3977 | * damon_set_region_system_rams_default() - Set the region of the given
|
| 3978 | * monitoring target as requested, or to cover all 'System RAM' resources.
|
| 3979 | * @t: The monitoring target to set the region.
|
| 3980 | * @start: The pointer to the start address of the region.
|
| 3981 | * @end: The pointer to the end address of the region.
|
| 3982 | * @addr_unit: The address unit for the damon_ctx of @t.
|
| 3983 | * @min_region_sz: Minimum region size.
|
| 3984 | *
|
| 3985 | * This function sets the region of @t as requested by @start and @end. If the
|
| 3986 | * values of @start and @end are zero, however, this function finds 'System
|
| 3987 | * RAM' resources and sets the region to cover all the resource. In the latter
|
| 3988 | * case, this function saves the start and the end addresseses of the first and
|
| 3989 | * the last resources in @start and @end, respectively.
|
| 3990 | *
|
| 3991 | * Return: 0 on success, negative error code otherwise.
|
| 3992 |
|
| 3993 | int damon_set_region_system_rams_default(struct damon_target *t,
|
| 3994 | unsigned long *start, unsigned long *end,
|
| 3995 | unsigned long addr_unit, unsigned long min_region_sz)
|
| 3996 | {
|
| 3997 | struct damon_addr_range addr_range;
|
| 3998 |
|
| 3999 | if (!*start && !*end &&
|
| 4000 | !damon_find_system_rams_range(start, end, addr_unit))
|
| 4001 | return -EINVAL;
|
| 4002 |
|
| 4003 | addr_range.start = *start;
|
| 4004 | addr_range.end = *end;
|
| 4005 | return damon_set_regions(t, &addr_range, 1, min_region_sz);
|
| 4006 | }
|
| 4007 |
|
| 4008 | |
| 4009 | * damon_update_region_access_rate() - Update the access rate of a region.
|
| 4010 | * @r: The DAMON region to update for its access check result.
|
| 4011 | * @accessed: Whether the region has accessed during last sampling interval.
|
| 4012 | *
|
| 4013 | * Update the access rate of a region with the region's last sampling interval
|
| 4014 | * access check result.
|
| 4015 | *
|
| 4016 | * Usually this will be called by &damon_operations->check_accesses callback.
|
| 4017 |
|
| 4018 | void damon_update_region_access_rate(struct damon_region *r, bool accessed)
|
| 4019 | {
|
| 4020 | if (accessed)
|
| 4021 | r->nr_accesses++;
|
| 4022 | }
|
| 4023 |
|
| 4024 | |
| 4025 | * damon_initialized() - Return if DAMON is ready to be used.
|
| 4026 | *
|
| 4027 | * Return: true if DAMON is ready to be used, false otherwise.
|
| 4028 |
|
| 4029 | bool damon_initialized(void)
|
| 4030 | {
|
| 4031 | return damon_region_cache != NULL;
|
| 4032 | }
|
| 4033 |
|
| 4034 | static int __init damon_init(void)
|
| 4035 | {
|
| 4036 | damon_region_cache = KMEM_CACHE(damon_region, 0);
|
| 4037 | if (unlikely(!damon_region_cache)) {
|
| 4038 | pr_err("creating damon_region_cache fails\n");
|
| 4039 | return -ENOMEM;
|
| 4040 | }
|
| 4041 |
|
| 4042 | return 0;
|
| 4043 | }
|
| 4044 |
|
| 4045 | subsys_initcall(damon_init);
|
| 4046 |
|
| 4047 | #include "tests/core-kunit.h"
|