| 1 | |
| 2 | |
| 3 | * Shadow Call Stack support.
|
| 4 | *
|
| 5 | * Copyright (C) 2019 Google LLC
|
| 6 |
|
| 7 |
|
| 8 | #include <linux/kasan.h>
|
| 9 | #include <linux/mm.h>
|
| 10 | #include <linux/scs.h>
|
| 11 | #include <linux/slab.h>
|
| 12 | #include <linux/vmstat.h>
|
| 13 |
|
| 14 | static struct kmem_cache *scs_cache;
|
| 15 |
|
| 16 | static void __scs_account(void *s, int account)
|
| 17 | {
|
| 18 | struct page *scs_page = virt_to_page(s);
|
| 19 |
|
| 20 | mod_zone_page_state(page_zone(scs_page), NR_KERNEL_SCS_KB,
|
| 21 | account * (SCS_SIZE / SZ_1K));
|
| 22 | }
|
| 23 |
|
| 24 | static void *scs_alloc(int node)
|
| 25 | {
|
| 26 | void *s = kmem_cache_alloc_node(scs_cache, GFP_SCS, node);
|
| 27 |
|
| 28 | if (!s)
|
| 29 | return NULL;
|
| 30 |
|
| 31 | *__scs_magic(s) = SCS_END_MAGIC;
|
| 32 |
|
| 33 | |
| 34 | * Poison the allocation to catch unintentional accesses to
|
| 35 | * the shadow stack when KASAN is enabled.
|
| 36 |
|
| 37 | kasan_poison_object_data(scs_cache, s);
|
| 38 | __scs_account(s, 1);
|
| 39 | return s;
|
| 40 | }
|
| 41 |
|
| 42 | static void scs_free(void *s)
|
| 43 | {
|
| 44 | __scs_account(s, -1);
|
| 45 | kasan_unpoison_object_data(scs_cache, s);
|
| 46 | kmem_cache_free(scs_cache, s);
|
| 47 | }
|
| 48 |
|
| 49 | void __init scs_init(void)
|
| 50 | {
|
| 51 | scs_cache = kmem_cache_create("scs_cache", SCS_SIZE, 0, 0, NULL);
|
| 52 | }
|
| 53 |
|
| 54 | int scs_prepare(struct task_struct *tsk, int node)
|
| 55 | {
|
| 56 | void *s = scs_alloc(node);
|
| 57 |
|
| 58 | if (!s)
|
| 59 | return -ENOMEM;
|
| 60 |
|
| 61 | task_scs(tsk) = task_scs_sp(tsk) = s;
|
| 62 | return 0;
|
| 63 | }
|
| 64 |
|
| 65 | static void scs_check_usage(struct task_struct *tsk)
|
| 66 | {
|
| 67 | static unsigned long highest;
|
| 68 |
|
| 69 | unsigned long *p, prev, curr = highest, used = 0;
|
| 70 |
|
| 71 | if (!IS_ENABLED(CONFIG_DEBUG_STACK_USAGE))
|
| 72 | return;
|
| 73 |
|
| 74 | for (p = task_scs(tsk); p < __scs_magic(tsk); ++p) {
|
| 75 | if (!READ_ONCE_NOCHECK(*p))
|
| 76 | break;
|
| 77 | used += sizeof(*p);
|
| 78 | }
|
| 79 |
|
| 80 | while (used > curr) {
|
| 81 | prev = cmpxchg_relaxed(&highest, curr, used);
|
| 82 |
|
| 83 | if (prev == curr) {
|
| 84 | pr_info("%s (%d): highest shadow stack usage: %lu bytes\n",
|
| 85 | tsk->comm, task_pid_nr(tsk), used);
|
| 86 | break;
|
| 87 | }
|
| 88 |
|
| 89 | curr = prev;
|
| 90 | }
|
| 91 | }
|
| 92 |
|
| 93 | void scs_release(struct task_struct *tsk)
|
| 94 | {
|
| 95 | void *s = task_scs(tsk);
|
| 96 |
|
| 97 | if (!s)
|
| 98 | return;
|
| 99 |
|
| 100 | WARN(task_scs_end_corrupted(tsk),
|
| 101 | "corrupted shadow stack detected when freeing task\n");
|
| 102 | scs_check_usage(tsk);
|
| 103 | scs_free(s);
|
| 104 | }
|