| 1 | |
| 2 | |
| 3 | * KCSAN reporting.
|
| 4 | *
|
| 5 | * Copyright (C) 2019, Google LLC.
|
| 6 |
|
| 7 |
|
| 8 | #include <linux/debug_locks.h>
|
| 9 | #include <linux/delay.h>
|
| 10 | #include <linux/jiffies.h>
|
| 11 | #include <linux/kallsyms.h>
|
| 12 | #include <linux/kernel.h>
|
| 13 | #include <linux/lockdep.h>
|
| 14 | #include <linux/preempt.h>
|
| 15 | #include <linux/printk.h>
|
| 16 | #include <linux/sched.h>
|
| 17 | #include <linux/spinlock.h>
|
| 18 | #include <linux/stacktrace.h>
|
| 19 |
|
| 20 | #include "kcsan.h"
|
| 21 | #include "encoding.h"
|
| 22 |
|
| 23 | |
| 24 | * Max. number of stack entries to show in the report.
|
| 25 |
|
| 26 | #define NUM_STACK_ENTRIES 64
|
| 27 |
|
| 28 |
|
| 29 | struct access_info {
|
| 30 | const volatile void *ptr;
|
| 31 | size_t size;
|
| 32 | int access_type;
|
| 33 | int task_pid;
|
| 34 | int cpu_id;
|
| 35 | unsigned long ip;
|
| 36 | };
|
| 37 |
|
| 38 | |
| 39 | * Other thread info: communicated from other racing thread to thread that set
|
| 40 | * up the watchpoint, which then prints the complete report atomically.
|
| 41 |
|
| 42 | struct other_info {
|
| 43 | struct access_info ai;
|
| 44 | unsigned long stack_entries[NUM_STACK_ENTRIES];
|
| 45 | int num_stack_entries;
|
| 46 |
|
| 47 | |
| 48 | * Optionally pass @current. Typically we do not need to pass @current
|
| 49 | * via @other_info since just @task_pid is sufficient. Passing @current
|
| 50 | * has additional overhead.
|
| 51 | *
|
| 52 | * To safely pass @current, we must either use get_task_struct/
|
| 53 | * put_task_struct, or stall the thread that populated @other_info.
|
| 54 | *
|
| 55 | * We cannot rely on get_task_struct/put_task_struct in case
|
| 56 | * release_report() races with a task being released, and would have to
|
| 57 | * free it in release_report(). This may result in deadlock if we want
|
| 58 | * to use KCSAN on the allocators.
|
| 59 | *
|
| 60 | * Since we also want to reliably print held locks for
|
| 61 | * CONFIG_KCSAN_VERBOSE, the current implementation stalls the thread
|
| 62 | * that populated @other_info until it has been consumed.
|
| 63 |
|
| 64 | struct task_struct *task;
|
| 65 | };
|
| 66 |
|
| 67 | |
| 68 | * To never block any producers of struct other_info, we need as many elements
|
| 69 | * as we have watchpoints (upper bound on concurrent races to report).
|
| 70 |
|
| 71 | static struct other_info other_infos[CONFIG_KCSAN_NUM_WATCHPOINTS + NUM_SLOTS-1];
|
| 72 |
|
| 73 | |
| 74 | * Information about reported races; used to rate limit reporting.
|
| 75 |
|
| 76 | struct report_time {
|
| 77 | |
| 78 | * The last time the race was reported.
|
| 79 |
|
| 80 | unsigned long time;
|
| 81 |
|
| 82 | |
| 83 | * The frames of the 2 threads; if only 1 thread is known, one frame
|
| 84 | * will be 0.
|
| 85 |
|
| 86 | unsigned long frame1;
|
| 87 | unsigned long frame2;
|
| 88 | };
|
| 89 |
|
| 90 | |
| 91 | * Since we also want to be able to debug allocators with KCSAN, to avoid
|
| 92 | * deadlock, report_times cannot be dynamically resized with krealloc in
|
| 93 | * rate_limit_report.
|
| 94 | *
|
| 95 | * Therefore, we use a fixed-size array, which at most will occupy a page. This
|
| 96 | * still adequately rate limits reports, assuming that a) number of unique data
|
| 97 | * races is not excessive, and b) occurrence of unique races within the
|
| 98 | * same time window is limited.
|
| 99 |
|
| 100 | #define REPORT_TIMES_MAX (PAGE_SIZE / sizeof(struct report_time))
|
| 101 | #define REPORT_TIMES_SIZE \
|
| 102 | (CONFIG_KCSAN_REPORT_ONCE_IN_MS > REPORT_TIMES_MAX ? \
|
| 103 | REPORT_TIMES_MAX : \
|
| 104 | CONFIG_KCSAN_REPORT_ONCE_IN_MS)
|
| 105 | static struct report_time report_times[REPORT_TIMES_SIZE];
|
| 106 |
|
| 107 | |
| 108 | * Spinlock serializing report generation, and access to @other_infos. Although
|
| 109 | * it could make sense to have a finer-grained locking story for @other_infos,
|
| 110 | * report generation needs to be serialized either way, so not much is gained.
|
| 111 |
|
| 112 | static DEFINE_RAW_SPINLOCK(report_lock);
|
| 113 |
|
| 114 | |
| 115 | * Checks if the race identified by thread frames frame1 and frame2 has
|
| 116 | * been reported since (now - KCSAN_REPORT_ONCE_IN_MS).
|
| 117 |
|
| 118 | static bool rate_limit_report(unsigned long frame1, unsigned long frame2)
|
| 119 | __must_hold(&report_lock)
|
| 120 | {
|
| 121 | struct report_time *use_entry = &report_times[0];
|
| 122 | unsigned long invalid_before;
|
| 123 | int i;
|
| 124 |
|
| 125 | BUILD_BUG_ON(CONFIG_KCSAN_REPORT_ONCE_IN_MS != 0 && REPORT_TIMES_SIZE == 0);
|
| 126 |
|
| 127 | if (CONFIG_KCSAN_REPORT_ONCE_IN_MS == 0)
|
| 128 | return false;
|
| 129 |
|
| 130 | invalid_before = jiffies - msecs_to_jiffies(CONFIG_KCSAN_REPORT_ONCE_IN_MS);
|
| 131 |
|
| 132 |
|
| 133 | for (i = 0; i < REPORT_TIMES_SIZE; ++i) {
|
| 134 | struct report_time *rt = &report_times[i];
|
| 135 |
|
| 136 | |
| 137 | * Must always select an entry for use to store info as we
|
| 138 | * cannot resize report_times; at the end of the scan, use_entry
|
| 139 | * will be the oldest entry, which ideally also happened before
|
| 140 | * KCSAN_REPORT_ONCE_IN_MS ago.
|
| 141 |
|
| 142 | if (time_before(rt->time, use_entry->time))
|
| 143 | use_entry = rt;
|
| 144 |
|
| 145 | |
| 146 | * Initially, no need to check any further as this entry as well
|
| 147 | * as following entries have never been used.
|
| 148 |
|
| 149 | if (rt->time == 0)
|
| 150 | break;
|
| 151 |
|
| 152 |
|
| 153 | if (time_before(rt->time, invalid_before))
|
| 154 | continue;
|
| 155 |
|
| 156 |
|
| 157 | if ((rt->frame1 == frame1 && rt->frame2 == frame2) ||
|
| 158 | (rt->frame1 == frame2 && rt->frame2 == frame1))
|
| 159 | return true;
|
| 160 | }
|
| 161 |
|
| 162 | use_entry->time = jiffies;
|
| 163 | use_entry->frame1 = frame1;
|
| 164 | use_entry->frame2 = frame2;
|
| 165 | return false;
|
| 166 | }
|
| 167 |
|
| 168 | |
| 169 | * Special rules to skip reporting.
|
| 170 |
|
| 171 | static bool
|
| 172 | skip_report(enum kcsan_value_change value_change, unsigned long top_frame)
|
| 173 | {
|
| 174 |
|
| 175 | WARN_ON_ONCE(value_change == KCSAN_VALUE_CHANGE_FALSE);
|
| 176 |
|
| 177 | |
| 178 | * The first call to skip_report always has value_change==TRUE, since we
|
| 179 | * cannot know the value written of an instrumented access. For the 2nd
|
| 180 | * call there are 6 cases with CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY:
|
| 181 | *
|
| 182 | * 1. read watchpoint, conflicting write (value_change==TRUE): report;
|
| 183 | * 2. read watchpoint, conflicting write (value_change==MAYBE): skip;
|
| 184 | * 3. write watchpoint, conflicting write (value_change==TRUE): report;
|
| 185 | * 4. write watchpoint, conflicting write (value_change==MAYBE): skip;
|
| 186 | * 5. write watchpoint, conflicting read (value_change==MAYBE): skip;
|
| 187 | * 6. write watchpoint, conflicting read (value_change==TRUE): report;
|
| 188 | *
|
| 189 | * Cases 1-4 are intuitive and expected; case 5 ensures we do not report
|
| 190 | * data races where the write may have rewritten the same value; case 6
|
| 191 | * is possible either if the size is larger than what we check value
|
| 192 | * changes for or the access type is KCSAN_ACCESS_ASSERT.
|
| 193 |
|
| 194 | if (IS_ENABLED(CONFIG_KCSAN_REPORT_VALUE_CHANGE_ONLY) &&
|
| 195 | value_change == KCSAN_VALUE_CHANGE_MAYBE) {
|
| 196 | |
| 197 | * The access is a write, but the data value did not change.
|
| 198 | *
|
| 199 | * We opt-out of this filter for certain functions at request of
|
| 200 | * maintainers.
|
| 201 |
|
| 202 | char buf[64];
|
| 203 | int len = scnprintf(buf, sizeof(buf), "%ps", (void *)top_frame);
|
| 204 |
|
| 205 | if (!strnstr(buf, "rcu_", len) &&
|
| 206 | !strnstr(buf, "_rcu", len) &&
|
| 207 | !strnstr(buf, "_srcu", len))
|
| 208 | return true;
|
| 209 | }
|
| 210 |
|
| 211 | return kcsan_skip_report_debugfs(top_frame);
|
| 212 | }
|
| 213 |
|
| 214 | static const char *get_access_type(int type)
|
| 215 | {
|
| 216 | if (type & KCSAN_ACCESS_ASSERT) {
|
| 217 | if (type & KCSAN_ACCESS_SCOPED) {
|
| 218 | if (type & KCSAN_ACCESS_WRITE)
|
| 219 | return "assert no accesses (reordered)";
|
| 220 | else
|
| 221 | return "assert no writes (reordered)";
|
| 222 | } else {
|
| 223 | if (type & KCSAN_ACCESS_WRITE)
|
| 224 | return "assert no accesses";
|
| 225 | else
|
| 226 | return "assert no writes";
|
| 227 | }
|
| 228 | }
|
| 229 |
|
| 230 | switch (type) {
|
| 231 | case 0:
|
| 232 | return "read";
|
| 233 | case KCSAN_ACCESS_ATOMIC:
|
| 234 | return "read (marked)";
|
| 235 | case KCSAN_ACCESS_WRITE:
|
| 236 | return "write";
|
| 237 | case KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC:
|
| 238 | return "write (marked)";
|
| 239 | case KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE:
|
| 240 | return "read-write";
|
| 241 | case KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC:
|
| 242 | return "read-write (marked)";
|
| 243 | case KCSAN_ACCESS_SCOPED:
|
| 244 | return "read (reordered)";
|
| 245 | case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_ATOMIC:
|
| 246 | return "read (marked, reordered)";
|
| 247 | case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_WRITE:
|
| 248 | return "write (reordered)";
|
| 249 | case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC:
|
| 250 | return "write (marked, reordered)";
|
| 251 | case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE:
|
| 252 | return "read-write (reordered)";
|
| 253 | case KCSAN_ACCESS_SCOPED | KCSAN_ACCESS_COMPOUND | KCSAN_ACCESS_WRITE | KCSAN_ACCESS_ATOMIC:
|
| 254 | return "read-write (marked, reordered)";
|
| 255 | default:
|
| 256 | BUG();
|
| 257 | }
|
| 258 | }
|
| 259 |
|
| 260 | static const char *get_bug_type(int type)
|
| 261 | {
|
| 262 | return (type & KCSAN_ACCESS_ASSERT) != 0 ? "assert: race" : "data-race";
|
| 263 | }
|
| 264 |
|
| 265 |
|
| 266 | static const char *get_thread_desc(int task_id)
|
| 267 | {
|
| 268 | if (task_id != -1) {
|
| 269 | static char buf[32];
|
| 270 |
|
| 271 | snprintf(buf, sizeof(buf), "task %i", task_id);
|
| 272 | return buf;
|
| 273 | }
|
| 274 | return "interrupt";
|
| 275 | }
|
| 276 |
|
| 277 |
|
| 278 | static int get_stack_skipnr(const unsigned long stack_entries[], int num_entries)
|
| 279 | {
|
| 280 | char buf[64];
|
| 281 | char *cur;
|
| 282 | int len, skip;
|
| 283 |
|
| 284 | for (skip = 0; skip < num_entries; ++skip) {
|
| 285 | len = scnprintf(buf, sizeof(buf), "%ps", (void *)stack_entries[skip]);
|
| 286 |
|
| 287 |
|
| 288 | if (strnstr(buf, "tsan_", len) ||
|
| 289 | strnstr(buf, "_once_size", len))
|
| 290 | continue;
|
| 291 |
|
| 292 | cur = strnstr(buf, "kcsan_", len);
|
| 293 | if (cur) {
|
| 294 | cur += strlen("kcsan_");
|
| 295 | if (!str_has_prefix(cur, "test"))
|
| 296 | continue;
|
| 297 |
|
| 298 | }
|
| 299 |
|
| 300 | |
| 301 | * No match for runtime functions -- @skip entries to skip to
|
| 302 | * get to first frame of interest.
|
| 303 |
|
| 304 | break;
|
| 305 | }
|
| 306 |
|
| 307 | return skip;
|
| 308 | }
|
| 309 |
|
| 310 | |
| 311 | * Skips to the first entry that matches the function of @ip, and then replaces
|
| 312 | * that entry with @ip, returning the entries to skip with @replaced containing
|
| 313 | * the replaced entry.
|
| 314 |
|
| 315 | static int
|
| 316 | replace_stack_entry(unsigned long stack_entries[], int num_entries, unsigned long ip,
|
| 317 | unsigned long *replaced)
|
| 318 | {
|
| 319 | unsigned long symbolsize, offset;
|
| 320 | unsigned long target_func;
|
| 321 | int skip;
|
| 322 |
|
| 323 | if (kallsyms_lookup_size_offset(ip, &symbolsize, &offset))
|
| 324 | target_func = ip - offset;
|
| 325 | else
|
| 326 | goto fallback;
|
| 327 |
|
| 328 | for (skip = 0; skip < num_entries; ++skip) {
|
| 329 | unsigned long func = stack_entries[skip];
|
| 330 |
|
| 331 | if (!kallsyms_lookup_size_offset(func, &symbolsize, &offset))
|
| 332 | goto fallback;
|
| 333 | func -= offset;
|
| 334 |
|
| 335 | if (func == target_func) {
|
| 336 | *replaced = stack_entries[skip];
|
| 337 | stack_entries[skip] = ip;
|
| 338 | return skip;
|
| 339 | }
|
| 340 | }
|
| 341 |
|
| 342 | fallback:
|
| 343 |
|
| 344 | WARN_ONCE(1, "Cannot find frame for %pS in stack trace", (void *)ip);
|
| 345 | return get_stack_skipnr(stack_entries, num_entries);
|
| 346 | }
|
| 347 |
|
| 348 | static int
|
| 349 | sanitize_stack_entries(unsigned long stack_entries[], int num_entries, unsigned long ip,
|
| 350 | unsigned long *replaced)
|
| 351 | {
|
| 352 | return ip ? replace_stack_entry(stack_entries, num_entries, ip, replaced) :
|
| 353 | get_stack_skipnr(stack_entries, num_entries);
|
| 354 | }
|
| 355 |
|
| 356 |
|
| 357 | static int sym_strcmp(void *addr1, void *addr2)
|
| 358 | {
|
| 359 | char buf1[64];
|
| 360 | char buf2[64];
|
| 361 |
|
| 362 | snprintf(buf1, sizeof(buf1), "%pS", addr1);
|
| 363 | snprintf(buf2, sizeof(buf2), "%pS", addr2);
|
| 364 |
|
| 365 | return strncmp(buf1, buf2, sizeof(buf1));
|
| 366 | }
|
| 367 |
|
| 368 | static void
|
| 369 | print_stack_trace(unsigned long stack_entries[], int num_entries, unsigned long reordered_to)
|
| 370 | __must_hold(&report_lock)
|
| 371 | {
|
| 372 | stack_trace_print(stack_entries, num_entries, 0);
|
| 373 | if (reordered_to)
|
| 374 | pr_err(" |\n +-> reordered to: %pS\n", (void *)reordered_to);
|
| 375 | }
|
| 376 |
|
| 377 | static void print_verbose_info(struct task_struct *task)
|
| 378 | __must_hold(&report_lock)
|
| 379 | {
|
| 380 | if (!task)
|
| 381 | return;
|
| 382 |
|
| 383 |
|
| 384 | kcsan_restore_irqtrace(task);
|
| 385 |
|
| 386 | pr_err("\n");
|
| 387 | debug_show_held_locks(task);
|
| 388 | print_irqtrace_events(task);
|
| 389 | }
|
| 390 |
|
| 391 | static void print_report(enum kcsan_value_change value_change,
|
| 392 | const struct access_info *ai,
|
| 393 | struct other_info *other_info,
|
| 394 | u64 old, u64 new, u64 mask)
|
| 395 | __must_hold(&report_lock)
|
| 396 | {
|
| 397 | unsigned long reordered_to = 0;
|
| 398 | unsigned long stack_entries[NUM_STACK_ENTRIES] = { 0 };
|
| 399 | int num_stack_entries = stack_trace_save(stack_entries, NUM_STACK_ENTRIES, 1);
|
| 400 | int skipnr = sanitize_stack_entries(stack_entries, num_stack_entries, ai->ip, &reordered_to);
|
| 401 | unsigned long this_frame = stack_entries[skipnr];
|
| 402 | unsigned long other_reordered_to = 0;
|
| 403 | unsigned long other_frame = 0;
|
| 404 | int other_skipnr = 0;
|
| 405 |
|
| 406 | |
| 407 | * Must check report filter rules before starting to print.
|
| 408 |
|
| 409 | if (skip_report(KCSAN_VALUE_CHANGE_TRUE, stack_entries[skipnr]))
|
| 410 | return;
|
| 411 |
|
| 412 | if (other_info) {
|
| 413 | other_skipnr = sanitize_stack_entries(other_info->stack_entries,
|
| 414 | other_info->num_stack_entries,
|
| 415 | other_info->ai.ip, &other_reordered_to);
|
| 416 | other_frame = other_info->stack_entries[other_skipnr];
|
| 417 |
|
| 418 |
|
| 419 | if (skip_report(value_change, other_frame))
|
| 420 | return;
|
| 421 | }
|
| 422 |
|
| 423 | if (rate_limit_report(this_frame, other_frame))
|
| 424 | return;
|
| 425 |
|
| 426 |
|
| 427 | pr_err("==================================================================\n");
|
| 428 | if (other_info) {
|
| 429 | int cmp;
|
| 430 |
|
| 431 | |
| 432 | * Order functions lexographically for consistent bug titles.
|
| 433 | * Do not print offset of functions to keep title short.
|
| 434 |
|
| 435 | cmp = sym_strcmp((void *)other_frame, (void *)this_frame);
|
| 436 | pr_err("BUG: KCSAN: %s in %ps / %ps\n",
|
| 437 | get_bug_type(ai->access_type | other_info->ai.access_type),
|
| 438 | (void *)(cmp < 0 ? other_frame : this_frame),
|
| 439 | (void *)(cmp < 0 ? this_frame : other_frame));
|
| 440 | } else {
|
| 441 | pr_err("BUG: KCSAN: %s in %pS\n", get_bug_type(ai->access_type),
|
| 442 | (void *)this_frame);
|
| 443 | }
|
| 444 |
|
| 445 | pr_err("\n");
|
| 446 |
|
| 447 |
|
| 448 | if (other_info) {
|
| 449 | pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n",
|
| 450 | get_access_type(other_info->ai.access_type), other_info->ai.ptr,
|
| 451 | other_info->ai.size, get_thread_desc(other_info->ai.task_pid),
|
| 452 | other_info->ai.cpu_id);
|
| 453 |
|
| 454 |
|
| 455 | print_stack_trace(other_info->stack_entries + other_skipnr,
|
| 456 | other_info->num_stack_entries - other_skipnr,
|
| 457 | other_reordered_to);
|
| 458 | if (IS_ENABLED(CONFIG_KCSAN_VERBOSE))
|
| 459 | print_verbose_info(other_info->task);
|
| 460 |
|
| 461 | pr_err("\n");
|
| 462 | pr_err("%s to 0x%px of %zu bytes by %s on cpu %i:\n",
|
| 463 | get_access_type(ai->access_type), ai->ptr, ai->size,
|
| 464 | get_thread_desc(ai->task_pid), ai->cpu_id);
|
| 465 | } else {
|
| 466 | pr_err("race at unknown origin, with %s to 0x%px of %zu bytes by %s on cpu %i:\n",
|
| 467 | get_access_type(ai->access_type), ai->ptr, ai->size,
|
| 468 | get_thread_desc(ai->task_pid), ai->cpu_id);
|
| 469 | }
|
| 470 |
|
| 471 | print_stack_trace(stack_entries + skipnr, num_stack_entries - skipnr, reordered_to);
|
| 472 | if (IS_ENABLED(CONFIG_KCSAN_VERBOSE))
|
| 473 | print_verbose_info(current);
|
| 474 |
|
| 475 |
|
| 476 | if (ai->size <= 8) {
|
| 477 | int hex_len = ai->size * 2;
|
| 478 | u64 diff = old ^ new;
|
| 479 |
|
| 480 | if (mask)
|
| 481 | diff &= mask;
|
| 482 | if (diff) {
|
| 483 | pr_err("\n");
|
| 484 | pr_err("value changed: 0x%0*llx -> 0x%0*llx\n",
|
| 485 | hex_len, old, hex_len, new);
|
| 486 | if (mask) {
|
| 487 | pr_err(" bits changed: 0x%0*llx with mask 0x%0*llx\n",
|
| 488 | hex_len, diff, hex_len, mask);
|
| 489 | }
|
| 490 | }
|
| 491 | }
|
| 492 |
|
| 493 |
|
| 494 | pr_err("\n");
|
| 495 | pr_err("Reported by Kernel Concurrency Sanitizer on:\n");
|
| 496 | dump_stack_print_info(KERN_DEFAULT);
|
| 497 | pr_err("==================================================================\n");
|
| 498 |
|
| 499 | check_panic_on_warn("KCSAN");
|
| 500 | }
|
| 501 |
|
| 502 | static void release_report(unsigned long *flags, struct other_info *other_info)
|
| 503 | __releases(&report_lock)
|
| 504 | {
|
| 505 | |
| 506 | * Use size to denote valid/invalid, since KCSAN entirely ignores
|
| 507 | * 0-sized accesses.
|
| 508 |
|
| 509 | other_info->ai.size = 0;
|
| 510 | raw_spin_unlock_irqrestore(&report_lock, *flags);
|
| 511 | }
|
| 512 |
|
| 513 | |
| 514 | * Sets @other_info->task and awaits consumption of @other_info.
|
| 515 |
|
| 516 | static void set_other_info_task_blocking(unsigned long *flags,
|
| 517 | const struct access_info *ai,
|
| 518 | struct other_info *other_info)
|
| 519 | __must_hold(&report_lock)
|
| 520 | {
|
| 521 | |
| 522 | * We may be instrumenting a code-path where current->state is already
|
| 523 | * something other than TASK_RUNNING.
|
| 524 |
|
| 525 | const bool is_running = task_is_running(current);
|
| 526 | |
| 527 | * To avoid deadlock in case we are in an interrupt here and this is a
|
| 528 | * race with a task on the same CPU (KCSAN_INTERRUPT_WATCHER), provide a
|
| 529 | * timeout to ensure this works in all contexts.
|
| 530 | *
|
| 531 | * Await approximately the worst case delay of the reporting thread (if
|
| 532 | * we are not interrupted).
|
| 533 |
|
| 534 | int timeout = max(kcsan_udelay_task, kcsan_udelay_interrupt);
|
| 535 |
|
| 536 | other_info->task = current;
|
| 537 | do {
|
| 538 | if (is_running) {
|
| 539 | |
| 540 | * Let lockdep know the real task is sleeping, to print
|
| 541 | * the held locks (recall we turned lockdep off, so
|
| 542 | * locking/unlocking @report_lock won't be recorded).
|
| 543 |
|
| 544 | set_current_state(TASK_UNINTERRUPTIBLE);
|
| 545 | }
|
| 546 | raw_spin_unlock_irqrestore(&report_lock, *flags);
|
| 547 | |
| 548 | * We cannot call schedule() since we also cannot reliably
|
| 549 | * determine if sleeping here is permitted -- see in_atomic().
|
| 550 |
|
| 551 |
|
| 552 | udelay(1);
|
| 553 | raw_spin_lock_irqsave(&report_lock, *flags);
|
| 554 | if (timeout-- < 0) {
|
| 555 | |
| 556 | * Abort. Reset @other_info->task to NULL, since it
|
| 557 | * appears the other thread is still going to consume
|
| 558 | * it. It will result in no verbose info printed for
|
| 559 | * this task.
|
| 560 |
|
| 561 | other_info->task = NULL;
|
| 562 | break;
|
| 563 | }
|
| 564 | |
| 565 | * If invalid, or @ptr nor @current matches, then @other_info
|
| 566 | * has been consumed and we may continue. If not, retry.
|
| 567 |
|
| 568 | } while (other_info->ai.size && other_info->ai.ptr == ai->ptr &&
|
| 569 | other_info->task == current);
|
| 570 | if (is_running)
|
| 571 | set_current_state(TASK_RUNNING);
|
| 572 | }
|
| 573 |
|
| 574 |
|
| 575 | static void prepare_report_producer(unsigned long *flags,
|
| 576 | const struct access_info *ai,
|
| 577 | struct other_info *other_info)
|
| 578 | __must_not_hold(&report_lock)
|
| 579 | {
|
| 580 | raw_spin_lock_irqsave(&report_lock, *flags);
|
| 581 |
|
| 582 | |
| 583 | * The same @other_infos entry cannot be used concurrently, because
|
| 584 | * there is a one-to-one mapping to watchpoint slots (@watchpoints in
|
| 585 | * core.c), and a watchpoint is only released for reuse after reporting
|
| 586 | * is done by the consumer of @other_info. Therefore, it is impossible
|
| 587 | * for another concurrent prepare_report_producer() to set the same
|
| 588 | * @other_info, and are guaranteed exclusivity for the @other_infos
|
| 589 | * entry pointed to by @other_info.
|
| 590 | *
|
| 591 | * To check this property holds, size should never be non-zero here,
|
| 592 | * because every consumer of struct other_info resets size to 0 in
|
| 593 | * release_report().
|
| 594 |
|
| 595 | WARN_ON(other_info->ai.size);
|
| 596 |
|
| 597 | other_info->ai = *ai;
|
| 598 | other_info->num_stack_entries = stack_trace_save(other_info->stack_entries, NUM_STACK_ENTRIES, 2);
|
| 599 |
|
| 600 | if (IS_ENABLED(CONFIG_KCSAN_VERBOSE))
|
| 601 | set_other_info_task_blocking(flags, ai, other_info);
|
| 602 |
|
| 603 | raw_spin_unlock_irqrestore(&report_lock, *flags);
|
| 604 | }
|
| 605 |
|
| 606 |
|
| 607 | static bool prepare_report_consumer(unsigned long *flags,
|
| 608 | const struct access_info *ai,
|
| 609 | struct other_info *other_info)
|
| 610 | __cond_acquires(true, &report_lock)
|
| 611 | {
|
| 612 |
|
| 613 | raw_spin_lock_irqsave(&report_lock, *flags);
|
| 614 | while (!other_info->ai.size) {
|
| 615 | raw_spin_unlock_irqrestore(&report_lock, *flags);
|
| 616 | cpu_relax();
|
| 617 | raw_spin_lock_irqsave(&report_lock, *flags);
|
| 618 | }
|
| 619 |
|
| 620 |
|
| 621 | if (WARN_ON(!matching_access((unsigned long)other_info->ai.ptr & WATCHPOINT_ADDR_MASK, other_info->ai.size,
|
| 622 | (unsigned long)ai->ptr & WATCHPOINT_ADDR_MASK, ai->size)))
|
| 623 | goto discard;
|
| 624 |
|
| 625 | if (!matching_access((unsigned long)other_info->ai.ptr, other_info->ai.size,
|
| 626 | (unsigned long)ai->ptr, ai->size)) {
|
| 627 | |
| 628 | * If the actual accesses to not match, this was a false
|
| 629 | * positive due to watchpoint encoding.
|
| 630 |
|
| 631 | atomic_long_inc(&kcsan_counters[KCSAN_COUNTER_ENCODING_FALSE_POSITIVES]);
|
| 632 | goto discard;
|
| 633 | }
|
| 634 |
|
| 635 | return true;
|
| 636 |
|
| 637 | discard:
|
| 638 | release_report(flags, other_info);
|
| 639 | return false;
|
| 640 | }
|
| 641 |
|
| 642 | static struct access_info prepare_access_info(const volatile void *ptr, size_t size,
|
| 643 | int access_type, unsigned long ip)
|
| 644 | {
|
| 645 | return (struct access_info) {
|
| 646 | .ptr = ptr,
|
| 647 | .size = size,
|
| 648 | .access_type = access_type,
|
| 649 | .task_pid = in_task() ? task_pid_nr(current) : -1,
|
| 650 | .cpu_id = raw_smp_processor_id(),
|
| 651 |
|
| 652 | .ip = (access_type & KCSAN_ACCESS_SCOPED) ? ip : 0,
|
| 653 | };
|
| 654 | }
|
| 655 |
|
| 656 | void kcsan_report_set_info(const volatile void *ptr, size_t size, int access_type,
|
| 657 | unsigned long ip, int watchpoint_idx)
|
| 658 | {
|
| 659 | const struct access_info ai = prepare_access_info(ptr, size, access_type, ip);
|
| 660 | unsigned long flags;
|
| 661 |
|
| 662 | kcsan_disable_current();
|
| 663 | lockdep_off();
|
| 664 |
|
| 665 | prepare_report_producer(&flags, &ai, &other_infos[watchpoint_idx]);
|
| 666 |
|
| 667 | lockdep_on();
|
| 668 | kcsan_enable_current();
|
| 669 | }
|
| 670 |
|
| 671 | void kcsan_report_known_origin(const volatile void *ptr, size_t size, int access_type,
|
| 672 | unsigned long ip, enum kcsan_value_change value_change,
|
| 673 | int watchpoint_idx, u64 old, u64 new, u64 mask)
|
| 674 | {
|
| 675 | const struct access_info ai = prepare_access_info(ptr, size, access_type, ip);
|
| 676 | struct other_info *other_info = &other_infos[watchpoint_idx];
|
| 677 | unsigned long flags = 0;
|
| 678 |
|
| 679 | kcsan_disable_current();
|
| 680 | |
| 681 | * Because we may generate reports when we're in scheduler code, the use
|
| 682 | * of printk() could deadlock. Until such time that all printing code
|
| 683 | * called in print_report() is scheduler-safe, accept the risk, and just
|
| 684 | * get our message out. As such, also disable lockdep to hide the
|
| 685 | * warning, and avoid disabling lockdep for the rest of the kernel.
|
| 686 |
|
| 687 | lockdep_off();
|
| 688 |
|
| 689 | if (!prepare_report_consumer(&flags, &ai, other_info))
|
| 690 | goto out;
|
| 691 | |
| 692 | * Never report if value_change is FALSE, only when it is
|
| 693 | * either TRUE or MAYBE. In case of MAYBE, further filtering may
|
| 694 | * be done once we know the full stack trace in print_report().
|
| 695 |
|
| 696 | if (value_change != KCSAN_VALUE_CHANGE_FALSE)
|
| 697 | print_report(value_change, &ai, other_info, old, new, mask);
|
| 698 |
|
| 699 | release_report(&flags, other_info);
|
| 700 | out:
|
| 701 | lockdep_on();
|
| 702 | kcsan_enable_current();
|
| 703 | }
|
| 704 |
|
| 705 | void kcsan_report_unknown_origin(const volatile void *ptr, size_t size, int access_type,
|
| 706 | unsigned long ip, u64 old, u64 new, u64 mask)
|
| 707 | {
|
| 708 | const struct access_info ai = prepare_access_info(ptr, size, access_type, ip);
|
| 709 | unsigned long flags;
|
| 710 |
|
| 711 | kcsan_disable_current();
|
| 712 | lockdep_off();
|
| 713 |
|
| 714 | raw_spin_lock_irqsave(&report_lock, flags);
|
| 715 | print_report(KCSAN_VALUE_CHANGE_TRUE, &ai, NULL, old, new, mask);
|
| 716 | raw_spin_unlock_irqrestore(&report_lock, flags);
|
| 717 |
|
| 718 | lockdep_on();
|
| 719 | kcsan_enable_current();
|
| 720 | }
|