| 1 | |
| 2 | |
| 3 | * Copyright (c) 2016 Facebook
|
| 4 | * Copyright (c) 2018 Covalent IO, Inc. http://covalent.io
|
| 5 |
|
| 6 | #include <uapi/linux/btf.h>
|
| 7 | #include <linux/kernel.h>
|
| 8 | #include <linux/types.h>
|
| 9 | #include <linux/bpf.h>
|
| 10 | #include <linux/bpf_verifier.h>
|
| 11 | #include <linux/math64.h>
|
| 12 | #include <linux/string.h>
|
| 13 |
|
| 14 | #define verbose(env, fmt, args...) bpf_verifier_log_write(env, fmt, ##args)
|
| 15 |
|
| 16 | static bool bpf_verifier_log_attr_valid(u32 log_level, char __user *log_buf, u32 log_size)
|
| 17 | {
|
| 18 |
|
| 19 | if (!!log_buf != !!log_size)
|
| 20 | return false;
|
| 21 |
|
| 22 | if (log_buf && log_level == 0)
|
| 23 | return false;
|
| 24 | if (log_level & ~BPF_LOG_MASK)
|
| 25 | return false;
|
| 26 | if (log_size > UINT_MAX >> 2)
|
| 27 | return false;
|
| 28 | return true;
|
| 29 | }
|
| 30 |
|
| 31 | int bpf_vlog_init(struct bpf_verifier_log *log, u32 log_level,
|
| 32 | char __user *log_buf, u32 log_size)
|
| 33 | {
|
| 34 | log->level = log_level;
|
| 35 | log->ubuf = log_buf;
|
| 36 | log->len_total = log_size;
|
| 37 |
|
| 38 |
|
| 39 | if (!bpf_verifier_log_attr_valid(log_level, log_buf, log_size))
|
| 40 | return -EINVAL;
|
| 41 |
|
| 42 | return 0;
|
| 43 | }
|
| 44 |
|
| 45 | static void bpf_vlog_update_len_max(struct bpf_verifier_log *log, u32 add_len)
|
| 46 | {
|
| 47 |
|
| 48 | u64 len = log->end_pos + add_len;
|
| 49 |
|
| 50 | |
| 51 | * bpf_vlog_reset() calls, so we maintain the max of any length at any
|
| 52 | * previous point
|
| 53 |
|
| 54 | if (len > UINT_MAX)
|
| 55 | log->len_max = UINT_MAX;
|
| 56 | else if (len > log->len_max)
|
| 57 | log->len_max = len;
|
| 58 | }
|
| 59 |
|
| 60 | void bpf_verifier_vlog(struct bpf_verifier_log *log, const char *fmt,
|
| 61 | va_list args)
|
| 62 | {
|
| 63 | u64 cur_pos;
|
| 64 | u32 new_n, n;
|
| 65 |
|
| 66 | n = vscnprintf(log->kbuf, BPF_VERIFIER_TMP_LOG_SIZE, fmt, args);
|
| 67 |
|
| 68 | if (log->level == BPF_LOG_KERNEL) {
|
| 69 | bool newline = n > 0 && log->kbuf[n - 1] == '\n';
|
| 70 |
|
| 71 | pr_err("BPF: %s%s", log->kbuf, newline ? "" : "\n");
|
| 72 | return;
|
| 73 | }
|
| 74 |
|
| 75 | n += 1;
|
| 76 | bpf_vlog_update_len_max(log, n);
|
| 77 |
|
| 78 | if (log->level & BPF_LOG_FIXED) {
|
| 79 |
|
| 80 | new_n = 0;
|
| 81 | if (log->end_pos < log->len_total) {
|
| 82 | new_n = min_t(u32, log->len_total - log->end_pos, n);
|
| 83 | log->kbuf[new_n - 1] = '\0';
|
| 84 | }
|
| 85 |
|
| 86 | cur_pos = log->end_pos;
|
| 87 | log->end_pos += n - 1;
|
| 88 |
|
| 89 | if (log->ubuf && new_n &&
|
| 90 | copy_to_user(log->ubuf + cur_pos, log->kbuf, new_n))
|
| 91 | goto fail;
|
| 92 | } else {
|
| 93 | u64 new_end, new_start;
|
| 94 | u32 buf_start, buf_end;
|
| 95 |
|
| 96 | new_end = log->end_pos + n;
|
| 97 | if (new_end - log->start_pos >= log->len_total)
|
| 98 | new_start = new_end - log->len_total;
|
| 99 | else
|
| 100 | new_start = log->start_pos;
|
| 101 |
|
| 102 | log->start_pos = new_start;
|
| 103 | log->end_pos = new_end - 1;
|
| 104 |
|
| 105 | if (!log->ubuf)
|
| 106 | return;
|
| 107 |
|
| 108 | new_n = min(n, log->len_total);
|
| 109 | cur_pos = new_end - new_n;
|
| 110 | div_u64_rem(cur_pos, log->len_total, &buf_start);
|
| 111 | div_u64_rem(new_end, log->len_total, &buf_end);
|
| 112 | |
| 113 | * exactly zero, then it actually points right to the end of
|
| 114 | * ubuf and there is no wrap around
|
| 115 |
|
| 116 | if (buf_end == 0)
|
| 117 | buf_end = log->len_total;
|
| 118 |
|
| 119 | |
| 120 | * if buf_start == buf_end, then we fill ubuf completely; we
|
| 121 | * can't have buf_start == buf_end to mean that there is
|
| 122 | * nothing to write, because we always write at least
|
| 123 | * something, even if terminal '\0'
|
| 124 |
|
| 125 | if (buf_start < buf_end) {
|
| 126 |
|
| 127 | if (copy_to_user(log->ubuf + buf_start,
|
| 128 | log->kbuf + n - new_n,
|
| 129 | buf_end - buf_start))
|
| 130 | goto fail;
|
| 131 | } else {
|
| 132 |
|
| 133 | if (copy_to_user(log->ubuf + buf_start,
|
| 134 | log->kbuf + n - new_n,
|
| 135 | log->len_total - buf_start))
|
| 136 | goto fail;
|
| 137 | if (copy_to_user(log->ubuf,
|
| 138 | log->kbuf + n - buf_end,
|
| 139 | buf_end))
|
| 140 | goto fail;
|
| 141 | }
|
| 142 | }
|
| 143 |
|
| 144 | return;
|
| 145 | fail:
|
| 146 | log->ubuf = NULL;
|
| 147 | }
|
| 148 |
|
| 149 | void bpf_vlog_reset(struct bpf_verifier_log *log, u64 new_pos)
|
| 150 | {
|
| 151 | char zero = 0;
|
| 152 | u32 pos;
|
| 153 |
|
| 154 | if (WARN_ON_ONCE(new_pos > log->end_pos))
|
| 155 | return;
|
| 156 |
|
| 157 | if (!bpf_verifier_log_needed(log) || log->level == BPF_LOG_KERNEL)
|
| 158 | return;
|
| 159 |
|
| 160 | |
| 161 | * then we didn't preserve any useful content and should adjust
|
| 162 | * start_pos to end up with an empty log (start_pos == end_pos)
|
| 163 |
|
| 164 | log->end_pos = new_pos;
|
| 165 | if (log->end_pos < log->start_pos)
|
| 166 | log->start_pos = log->end_pos;
|
| 167 |
|
| 168 | if (!log->ubuf)
|
| 169 | return;
|
| 170 |
|
| 171 | if (log->level & BPF_LOG_FIXED)
|
| 172 | pos = log->end_pos + 1;
|
| 173 | else
|
| 174 | div_u64_rem(new_pos, log->len_total, &pos);
|
| 175 |
|
| 176 | if (pos < log->len_total && put_user(zero, log->ubuf + pos))
|
| 177 | log->ubuf = NULL;
|
| 178 | }
|
| 179 |
|
| 180 | static void bpf_vlog_reverse_kbuf(char *buf, int len)
|
| 181 | {
|
| 182 | int i, j;
|
| 183 |
|
| 184 | for (i = 0, j = len - 1; i < j; i++, j--)
|
| 185 | swap(buf[i], buf[j]);
|
| 186 | }
|
| 187 |
|
| 188 | static int bpf_vlog_reverse_ubuf(struct bpf_verifier_log *log, int start, int end)
|
| 189 | {
|
| 190 |
|
| 191 | int n = sizeof(log->kbuf) / 2, nn;
|
| 192 | char *lbuf = log->kbuf, *rbuf = log->kbuf + n;
|
| 193 |
|
| 194 | |
| 195 | * and right side; within each chunk, swap all the bytes; after that
|
| 196 | * reverse the order of lbuf and rbuf and write result back to ubuf.
|
| 197 | * This way we'll end up with swapped contents of specified
|
| 198 | * [start, end) ubuf segment.
|
| 199 |
|
| 200 | while (end - start > 1) {
|
| 201 | nn = min(n, (end - start ) / 2);
|
| 202 |
|
| 203 | if (copy_from_user(lbuf, log->ubuf + start, nn))
|
| 204 | return -EFAULT;
|
| 205 | if (copy_from_user(rbuf, log->ubuf + end - nn, nn))
|
| 206 | return -EFAULT;
|
| 207 |
|
| 208 | bpf_vlog_reverse_kbuf(lbuf, nn);
|
| 209 | bpf_vlog_reverse_kbuf(rbuf, nn);
|
| 210 |
|
| 211 | |
| 212 | * left one to end up with properly reversed overall ubuf
|
| 213 |
|
| 214 | if (copy_to_user(log->ubuf + start, rbuf, nn))
|
| 215 | return -EFAULT;
|
| 216 | if (copy_to_user(log->ubuf + end - nn, lbuf, nn))
|
| 217 | return -EFAULT;
|
| 218 |
|
| 219 | start += nn;
|
| 220 | end -= nn;
|
| 221 | }
|
| 222 |
|
| 223 | return 0;
|
| 224 | }
|
| 225 |
|
| 226 | int bpf_vlog_finalize(struct bpf_verifier_log *log, u32 *log_size_actual)
|
| 227 | {
|
| 228 | u32 sublen;
|
| 229 | int err;
|
| 230 |
|
| 231 | *log_size_actual = 0;
|
| 232 | if (!log || log->level == 0 || log->level == BPF_LOG_KERNEL)
|
| 233 | return 0;
|
| 234 |
|
| 235 | if (!log->ubuf)
|
| 236 | goto skip_log_rotate;
|
| 237 |
|
| 238 | if (log->start_pos == 0)
|
| 239 | goto skip_log_rotate;
|
| 240 |
|
| 241 | |
| 242 | * buffer beginning and be a continuous zero-terminated string. Note
|
| 243 | * that if log->start_pos != 0 then we definitely filled up entire log
|
| 244 | * buffer with no gaps, and we just need to shift buffer contents to
|
| 245 | * the left by (log->start_pos % log->len_total) bytes.
|
| 246 | *
|
| 247 | * Unfortunately, user buffer could be huge and we don't want to
|
| 248 | * allocate temporary kernel memory of the same size just to shift
|
| 249 | * contents in a straightforward fashion. Instead, we'll be clever and
|
| 250 | * do in-place array rotation. This is a leetcode-style problem, which
|
| 251 | * could be solved by three rotations.
|
| 252 | *
|
| 253 | * Let's say we have log buffer that has to be shifted left by 7 bytes
|
| 254 | * (spaces and vertical bar is just for demonstrative purposes):
|
| 255 | * E F G H I J K | A B C D
|
| 256 | *
|
| 257 | * First, we reverse entire array:
|
| 258 | * D C B A | K J I H G F E
|
| 259 | *
|
| 260 | * Then we rotate first 4 bytes (DCBA) and separately last 7 bytes
|
| 261 | * (KJIHGFE), resulting in a properly rotated array:
|
| 262 | * A B C D | E F G H I J K
|
| 263 | *
|
| 264 | * We'll utilize log->kbuf to read user memory chunk by chunk, swap
|
| 265 | * bytes, and write them back. Doing it byte-by-byte would be
|
| 266 | * unnecessarily inefficient. Altogether we are going to read and
|
| 267 | * write each byte twice, for total 4 memory copies between kernel and
|
| 268 | * user space.
|
| 269 |
|
| 270 |
|
| 271 | |
| 272 | * len(ABCD) in the example above
|
| 273 |
|
| 274 | div_u64_rem(log->start_pos, log->len_total, &sublen);
|
| 275 | sublen = log->len_total - sublen;
|
| 276 |
|
| 277 | err = bpf_vlog_reverse_ubuf(log, 0, log->len_total);
|
| 278 | err = err ?: bpf_vlog_reverse_ubuf(log, 0, sublen);
|
| 279 | err = err ?: bpf_vlog_reverse_ubuf(log, sublen, log->len_total);
|
| 280 | if (err)
|
| 281 | log->ubuf = NULL;
|
| 282 |
|
| 283 | skip_log_rotate:
|
| 284 | *log_size_actual = log->len_max;
|
| 285 |
|
| 286 | |
| 287 | * or ubuf==NULL and len_total==0, so if this condition doesn't hold,
|
| 288 | * we got a fault somewhere along the way, so report it back
|
| 289 |
|
| 290 | if (!!log->ubuf != !!log->len_total)
|
| 291 | return -EFAULT;
|
| 292 |
|
| 293 |
|
| 294 | if (log->ubuf && log->len_max > log->len_total)
|
| 295 | return -ENOSPC;
|
| 296 |
|
| 297 | return 0;
|
| 298 | }
|
| 299 |
|
| 300 | |
| 301 | * bpf_verifier_log_write() is used to dump the verification trace to the log,
|
| 302 | * so the user can figure out what's wrong with the program
|
| 303 |
|
| 304 | __printf(2, 3) void bpf_verifier_log_write(struct bpf_verifier_env *env,
|
| 305 | const char *fmt, ...)
|
| 306 | {
|
| 307 | va_list args;
|
| 308 |
|
| 309 | if (!bpf_verifier_log_needed(&env->log))
|
| 310 | return;
|
| 311 |
|
| 312 | va_start(args, fmt);
|
| 313 | bpf_verifier_vlog(&env->log, fmt, args);
|
| 314 | va_end(args);
|
| 315 | }
|
| 316 | EXPORT_SYMBOL_GPL(bpf_verifier_log_write);
|
| 317 |
|
| 318 | __printf(2, 3) void bpf_log(struct bpf_verifier_log *log,
|
| 319 | const char *fmt, ...)
|
| 320 | {
|
| 321 | va_list args;
|
| 322 |
|
| 323 | if (!bpf_verifier_log_needed(log))
|
| 324 | return;
|
| 325 |
|
| 326 | va_start(args, fmt);
|
| 327 | bpf_verifier_vlog(log, fmt, args);
|
| 328 | va_end(args);
|
| 329 | }
|
| 330 | EXPORT_SYMBOL_GPL(bpf_log);
|
| 331 |
|
| 332 | static const char *ltrim(const char *s)
|
| 333 | {
|
| 334 | while (isspace(*s))
|
| 335 | s++;
|
| 336 |
|
| 337 | return s;
|
| 338 | }
|
| 339 |
|
| 340 | __printf(3, 4) void verbose_linfo(struct bpf_verifier_env *env,
|
| 341 | u32 insn_off,
|
| 342 | const char *prefix_fmt, ...)
|
| 343 | {
|
| 344 | const struct bpf_line_info *linfo, *prev_linfo;
|
| 345 | const struct btf *btf;
|
| 346 | const char *s, *fname;
|
| 347 |
|
| 348 | if (!bpf_verifier_log_needed(&env->log))
|
| 349 | return;
|
| 350 |
|
| 351 | prev_linfo = env->prev_linfo;
|
| 352 | linfo = bpf_find_linfo(env->prog, insn_off);
|
| 353 | if (!linfo || linfo == prev_linfo)
|
| 354 | return;
|
| 355 |
|
| 356 | |
| 357 | * source code line, but have differing column numbers. Given verifier
|
| 358 | * log doesn't emit column information, from user perspective we just
|
| 359 | * end up emitting the same source code line twice unnecessarily.
|
| 360 | * So instead check that previous and current linfo record point to
|
| 361 | * the same file (file_name_offs match) and the same line number, and
|
| 362 | * avoid emitting duplicated source code line in such case.
|
| 363 |
|
| 364 | if (prev_linfo && linfo->file_name_off == prev_linfo->file_name_off &&
|
| 365 | BPF_LINE_INFO_LINE_NUM(linfo->line_col) == BPF_LINE_INFO_LINE_NUM(prev_linfo->line_col))
|
| 366 | return;
|
| 367 |
|
| 368 | if (prefix_fmt) {
|
| 369 | va_list args;
|
| 370 |
|
| 371 | va_start(args, prefix_fmt);
|
| 372 | bpf_verifier_vlog(&env->log, prefix_fmt, args);
|
| 373 | va_end(args);
|
| 374 | }
|
| 375 |
|
| 376 | btf = env->prog->aux->btf;
|
| 377 | s = ltrim(btf_name_by_offset(btf, linfo->line_off));
|
| 378 | verbose(env, "%s", s);
|
| 379 |
|
| 380 | s = btf_name_by_offset(btf, linfo->file_name_off);
|
| 381 |
|
| 382 | fname = strrchr(s, '/');
|
| 383 | fname = fname ? fname + 1 : s;
|
| 384 | verbose(env, " @ %s:%u\n", fname, BPF_LINE_INFO_LINE_NUM(linfo->line_col));
|
| 385 |
|
| 386 | env->prev_linfo = linfo;
|
| 387 | }
|
| 388 |
|
| 389 | static const char *btf_type_name(const struct btf *btf, u32 id)
|
| 390 | {
|
| 391 | return btf_name_by_offset(btf, btf_type_by_id(btf, id)->name_off);
|
| 392 | }
|
| 393 |
|
| 394 | |
| 395 | *
|
| 396 | * Note that reg_type_str() can not appear more than once in a single verbose()
|
| 397 | * statement.
|
| 398 |
|
| 399 | const char *reg_type_str(struct bpf_verifier_env *env, enum bpf_reg_type type)
|
| 400 | {
|
| 401 | char postfix[16] = {0}, prefix[64] = {0};
|
| 402 | static const char * const str[] = {
|
| 403 | [NOT_INIT] = "?",
|
| 404 | [SCALAR_VALUE] = "scalar",
|
| 405 | [PTR_TO_CTX] = "ctx",
|
| 406 | [CONST_PTR_TO_MAP] = "map_ptr",
|
| 407 | [PTR_TO_MAP_VALUE] = "map_value",
|
| 408 | [PTR_TO_STACK] = "fp",
|
| 409 | [PTR_TO_PACKET] = "pkt",
|
| 410 | [PTR_TO_PACKET_META] = "pkt_meta",
|
| 411 | [PTR_TO_PACKET_END] = "pkt_end",
|
| 412 | [PTR_TO_FLOW_KEYS] = "flow_keys",
|
| 413 | [PTR_TO_SOCKET] = "sock",
|
| 414 | [PTR_TO_SOCK_COMMON] = "sock_common",
|
| 415 | [PTR_TO_TCP_SOCK] = "tcp_sock",
|
| 416 | [PTR_TO_TP_BUFFER] = "tp_buffer",
|
| 417 | [PTR_TO_XDP_SOCK] = "xdp_sock",
|
| 418 | [PTR_TO_BTF_ID] = "ptr_",
|
| 419 | [PTR_TO_MEM] = "mem",
|
| 420 | [PTR_TO_ARENA] = "arena",
|
| 421 | [PTR_TO_BUF] = "buf",
|
| 422 | [PTR_TO_FUNC] = "func",
|
| 423 | [PTR_TO_INSN] = "insn",
|
| 424 | [PTR_TO_MAP_KEY] = "map_key",
|
| 425 | [CONST_PTR_TO_DYNPTR] = "dynptr_ptr",
|
| 426 | };
|
| 427 |
|
| 428 | if (type & PTR_MAYBE_NULL) {
|
| 429 | if (base_type(type) == PTR_TO_BTF_ID)
|
| 430 | strscpy(postfix, "or_null_");
|
| 431 | else
|
| 432 | strscpy(postfix, "_or_null");
|
| 433 | }
|
| 434 |
|
| 435 | snprintf(prefix, sizeof(prefix), "%s%s%s%s%s%s%s",
|
| 436 | type & MEM_RDONLY ? "rdonly_" : "",
|
| 437 | type & MEM_RINGBUF ? "ringbuf_" : "",
|
| 438 | type & MEM_USER ? "user_" : "",
|
| 439 | type & MEM_PERCPU ? "percpu_" : "",
|
| 440 | type & MEM_RCU ? "rcu_" : "",
|
| 441 | type & PTR_UNTRUSTED ? "untrusted_" : "",
|
| 442 | type & PTR_TRUSTED ? "trusted_" : ""
|
| 443 | );
|
| 444 |
|
| 445 | snprintf(env->tmp_str_buf, TMP_STR_BUF_LEN, "%s%s%s",
|
| 446 | prefix, str[base_type(type)], postfix);
|
| 447 | return env->tmp_str_buf;
|
| 448 | }
|
| 449 |
|
| 450 | const char *dynptr_type_str(enum bpf_dynptr_type type)
|
| 451 | {
|
| 452 | switch (type) {
|
| 453 | case BPF_DYNPTR_TYPE_LOCAL:
|
| 454 | return "local";
|
| 455 | case BPF_DYNPTR_TYPE_RINGBUF:
|
| 456 | return "ringbuf";
|
| 457 | case BPF_DYNPTR_TYPE_SKB:
|
| 458 | return "skb";
|
| 459 | case BPF_DYNPTR_TYPE_XDP:
|
| 460 | return "xdp";
|
| 461 | case BPF_DYNPTR_TYPE_SKB_META:
|
| 462 | return "skb_meta";
|
| 463 | case BPF_DYNPTR_TYPE_FILE:
|
| 464 | return "file";
|
| 465 | case BPF_DYNPTR_TYPE_INVALID:
|
| 466 | return "<invalid>";
|
| 467 | default:
|
| 468 | WARN_ONCE(1, "unknown dynptr type %d\n", type);
|
| 469 | return "<unknown>";
|
| 470 | }
|
| 471 | }
|
| 472 |
|
| 473 | const char *iter_type_str(const struct btf *btf, u32 btf_id)
|
| 474 | {
|
| 475 | if (!btf || btf_id == 0)
|
| 476 | return "<invalid>";
|
| 477 |
|
| 478 |
|
| 479 | return btf_type_name(btf, btf_id) + sizeof(ITER_PREFIX) - 1;
|
| 480 | }
|
| 481 |
|
| 482 | const char *iter_state_str(enum bpf_iter_state state)
|
| 483 | {
|
| 484 | switch (state) {
|
| 485 | case BPF_ITER_STATE_ACTIVE:
|
| 486 | return "active";
|
| 487 | case BPF_ITER_STATE_DRAINED:
|
| 488 | return "drained";
|
| 489 | case BPF_ITER_STATE_INVALID:
|
| 490 | return "<invalid>";
|
| 491 | default:
|
| 492 | WARN_ONCE(1, "unknown iter state %d\n", state);
|
| 493 | return "<unknown>";
|
| 494 | }
|
| 495 | }
|
| 496 |
|
| 497 | static char slot_type_char[] = {
|
| 498 | [STACK_INVALID] = '?',
|
| 499 | [STACK_SPILL] = 'r',
|
| 500 | [STACK_MISC] = 'm',
|
| 501 | [STACK_ZERO] = '0',
|
| 502 | [STACK_DYNPTR] = 'd',
|
| 503 | [STACK_ITER] = 'i',
|
| 504 | [STACK_IRQ_FLAG] = 'f',
|
| 505 | [STACK_POISON] = 'p',
|
| 506 | };
|
| 507 |
|
| 508 | #define UNUM_MAX_DECIMAL U16_MAX
|
| 509 | #define SNUM_MAX_DECIMAL S16_MAX
|
| 510 | #define SNUM_MIN_DECIMAL S16_MIN
|
| 511 |
|
| 512 | static bool is_unum_decimal(u64 num)
|
| 513 | {
|
| 514 | return num <= UNUM_MAX_DECIMAL;
|
| 515 | }
|
| 516 |
|
| 517 | static bool is_snum_decimal(s64 num)
|
| 518 | {
|
| 519 | return num >= SNUM_MIN_DECIMAL && num <= SNUM_MAX_DECIMAL;
|
| 520 | }
|
| 521 |
|
| 522 | static void verbose_unum(struct bpf_verifier_env *env, u64 num)
|
| 523 | {
|
| 524 | if (is_unum_decimal(num))
|
| 525 | verbose(env, "%llu", num);
|
| 526 | else
|
| 527 | verbose(env, "%#llx", num);
|
| 528 | }
|
| 529 |
|
| 530 | static void verbose_snum(struct bpf_verifier_env *env, s64 num)
|
| 531 | {
|
| 532 | if (is_snum_decimal(num))
|
| 533 | verbose(env, "%lld", num);
|
| 534 | else
|
| 535 | verbose(env, "%#llx", num);
|
| 536 | }
|
| 537 |
|
| 538 | int tnum_strn(char *str, size_t size, struct tnum a)
|
| 539 | {
|
| 540 |
|
| 541 | if (a.mask == 0) {
|
| 542 | if (is_unum_decimal(a.value))
|
| 543 | return snprintf(str, size, "%llu", a.value);
|
| 544 | if (is_snum_decimal(a.value))
|
| 545 | return snprintf(str, size, "%lld", a.value);
|
| 546 | else
|
| 547 | return snprintf(str, size, "%#llx", a.value);
|
| 548 | }
|
| 549 | return snprintf(str, size, "(%#llx; %#llx)", a.value, a.mask);
|
| 550 | }
|
| 551 | EXPORT_SYMBOL_GPL(tnum_strn);
|
| 552 |
|
| 553 | static void print_scalar_ranges(struct bpf_verifier_env *env,
|
| 554 | const struct bpf_reg_state *reg,
|
| 555 | const char **sep)
|
| 556 | {
|
| 557 | |
| 558 | * output as much as possible, but there is a bit of a complication.
|
| 559 | * If we choose to print values as decimals, this is natural to do,
|
| 560 | * because negative 64-bit and 32-bit values >= -S32_MIN have the same
|
| 561 | * representation due to sign extension. But if we choose to print
|
| 562 | * them in hex format (see is_snum_decimal()), then sign extension is
|
| 563 | * misleading.
|
| 564 | * E.g., smin=-2 and smin32=-2 are exactly the same in decimal, but in
|
| 565 | * hex they will be smin=0xfffffffffffffffe and smin32=0xfffffffe, two
|
| 566 | * very different numbers.
|
| 567 | * So we avoid sign extension if we choose to print values in hex.
|
| 568 |
|
| 569 | struct {
|
| 570 | const char *name;
|
| 571 | u64 val;
|
| 572 | bool omit;
|
| 573 | } minmaxs[] = {
|
| 574 | {"smin", reg_smin(reg), reg_smin(reg) == S64_MIN},
|
| 575 | {"smax", reg_smax(reg), reg_smax(reg) == S64_MAX},
|
| 576 | {"umin", reg_umin(reg), reg_umin(reg) == 0},
|
| 577 | {"umax", reg_umax(reg), reg_umax(reg) == U64_MAX},
|
| 578 | {"smin32",
|
| 579 | is_snum_decimal((s64)reg_s32_min(reg))
|
| 580 | ? (s64)reg_s32_min(reg)
|
| 581 | : (u32)reg_s32_min(reg), reg_s32_min(reg) == S32_MIN},
|
| 582 | {"smax32",
|
| 583 | is_snum_decimal((s64)reg_s32_max(reg))
|
| 584 | ? (s64)reg_s32_max(reg)
|
| 585 | : (u32)reg_s32_max(reg), reg_s32_max(reg) == S32_MAX},
|
| 586 | {"umin32", reg_u32_min(reg), reg_u32_min(reg) == 0},
|
| 587 | {"umax32", reg_u32_max(reg), reg_u32_max(reg) == U32_MAX},
|
| 588 | }, *m1, *m2, *mend = &minmaxs[ARRAY_SIZE(minmaxs)];
|
| 589 | bool neg1, neg2;
|
| 590 |
|
| 591 | for (m1 = &minmaxs[0]; m1 < mend; m1++) {
|
| 592 | if (m1->omit)
|
| 593 | continue;
|
| 594 |
|
| 595 | neg1 = m1->name[0] == 's' && (s64)m1->val < 0;
|
| 596 |
|
| 597 | verbose(env, "%s%s=", *sep, m1->name);
|
| 598 | *sep = ",";
|
| 599 |
|
| 600 | for (m2 = m1 + 2; m2 < mend; m2 += 2) {
|
| 601 | if (m2->omit || m2->val != m1->val)
|
| 602 | continue;
|
| 603 |
|
| 604 | neg2 = m2->name[0] == 's' && (s64)m2->val < 0;
|
| 605 | if (neg2 != neg1)
|
| 606 | continue;
|
| 607 | m2->omit = true;
|
| 608 | verbose(env, "%s=", m2->name);
|
| 609 | }
|
| 610 |
|
| 611 | if (m1->name[0] == 's')
|
| 612 | verbose_snum(env, m1->val);
|
| 613 | else
|
| 614 | verbose_unum(env, m1->val);
|
| 615 | }
|
| 616 | }
|
| 617 |
|
| 618 | |
| 619 | * _a stands for append, was shortened to avoid multiline statements below.
|
| 620 | * This macro is used to output a comma separated list of attributes.
|
| 621 |
|
| 622 | #define verbose_a(fmt, ...) ({ verbose(env, "%s" fmt, sep, ##__VA_ARGS__); sep = ","; })
|
| 623 |
|
| 624 | static void print_reg_state(struct bpf_verifier_env *env,
|
| 625 | const struct bpf_func_state *state,
|
| 626 | const struct bpf_reg_state *reg)
|
| 627 | {
|
| 628 | enum bpf_reg_type t;
|
| 629 | const char *sep = "";
|
| 630 |
|
| 631 | t = reg->type;
|
| 632 | if (t == SCALAR_VALUE && reg->precise)
|
| 633 | verbose(env, "P");
|
| 634 | if (t == SCALAR_VALUE && tnum_is_const(reg->var_off)) {
|
| 635 | verbose_snum(env, reg->var_off.value);
|
| 636 | return;
|
| 637 | }
|
| 638 |
|
| 639 | verbose(env, "%s", reg_type_str(env, t));
|
| 640 | if (t == PTR_TO_ARENA)
|
| 641 | return;
|
| 642 | if (t == PTR_TO_STACK) {
|
| 643 | if (state->frameno != reg->frameno)
|
| 644 | verbose(env, "[%d]", reg->frameno);
|
| 645 | if (tnum_is_const(reg->var_off)) {
|
| 646 | verbose_snum(env, reg->var_off.value + reg->delta);
|
| 647 | return;
|
| 648 | }
|
| 649 | }
|
| 650 | if (base_type(t) == PTR_TO_BTF_ID)
|
| 651 | verbose(env, "%s", btf_type_name(reg->btf, reg->btf_id));
|
| 652 | verbose(env, "(");
|
| 653 | if (reg->id)
|
| 654 | verbose_a("id=%d", reg->id & ~BPF_ADD_CONST);
|
| 655 | if (reg->id & BPF_ADD_CONST)
|
| 656 | verbose(env, "%+d", reg->delta);
|
| 657 | if (reg->parent_id)
|
| 658 | verbose_a("parent_id=%d", reg->parent_id);
|
| 659 | if (type_is_non_owning_ref(reg->type))
|
| 660 | verbose_a("%s", "non_own_ref");
|
| 661 | if (type_is_map_ptr(t)) {
|
| 662 | if (reg->map_ptr->name[0])
|
| 663 | verbose_a("map=%s", reg->map_ptr->name);
|
| 664 | verbose_a("ks=%d,vs=%d",
|
| 665 | reg->map_ptr->key_size,
|
| 666 | reg->map_ptr->value_size);
|
| 667 | }
|
| 668 | if (t != SCALAR_VALUE && reg->delta) {
|
| 669 | verbose_a("off=");
|
| 670 | verbose_snum(env, reg->delta);
|
| 671 | }
|
| 672 | if (type_is_pkt_pointer(t)) {
|
| 673 | verbose_a("r=");
|
| 674 | verbose_unum(env, reg->range);
|
| 675 | }
|
| 676 | if (base_type(t) == PTR_TO_MEM) {
|
| 677 | verbose_a("sz=");
|
| 678 | verbose_unum(env, reg->mem_size);
|
| 679 | }
|
| 680 | if (t == CONST_PTR_TO_DYNPTR)
|
| 681 | verbose_a("type=%s", dynptr_type_str(reg->dynptr.type));
|
| 682 | if (tnum_is_const(reg->var_off)) {
|
| 683 |
|
| 684 | if (reg->var_off.value) {
|
| 685 | verbose_a("imm=");
|
| 686 | verbose_snum(env, reg->var_off.value);
|
| 687 | }
|
| 688 | } else {
|
| 689 | print_scalar_ranges(env, reg, &sep);
|
| 690 | if (!tnum_is_unknown(reg->var_off)) {
|
| 691 | char tn_buf[48];
|
| 692 |
|
| 693 | tnum_strn(tn_buf, sizeof(tn_buf), reg->var_off);
|
| 694 | verbose_a("var_off=%s", tn_buf);
|
| 695 | }
|
| 696 | }
|
| 697 | verbose(env, ")");
|
| 698 | }
|
| 699 |
|
| 700 | void print_verifier_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,
|
| 701 | u32 frameno, bool print_all)
|
| 702 | {
|
| 703 | const struct bpf_func_state *state = vstate->frame[frameno];
|
| 704 | const struct bpf_reg_state *reg;
|
| 705 | int i;
|
| 706 |
|
| 707 | if (state->frameno)
|
| 708 | verbose(env, " frame%d:", state->frameno);
|
| 709 | for (i = 0; i < MAX_BPF_REG; i++) {
|
| 710 | reg = &state->regs[i];
|
| 711 | if (reg->type == NOT_INIT)
|
| 712 | continue;
|
| 713 | if (!print_all && !reg_scratched(env, i))
|
| 714 | continue;
|
| 715 | verbose(env, " R%d", i);
|
| 716 | verbose(env, "=");
|
| 717 | print_reg_state(env, state, reg);
|
| 718 | }
|
| 719 | for (i = 0; i < state->allocated_stack / BPF_REG_SIZE; i++) {
|
| 720 | char types_buf[BPF_REG_SIZE + 1];
|
| 721 | const char *sep = "";
|
| 722 | bool valid = false;
|
| 723 | u8 slot_type;
|
| 724 | int j;
|
| 725 |
|
| 726 | if (!print_all && !stack_slot_scratched(env, i))
|
| 727 | continue;
|
| 728 |
|
| 729 | for (j = 0; j < BPF_REG_SIZE; j++) {
|
| 730 | slot_type = state->stack[i].slot_type[j];
|
| 731 | if (slot_type != STACK_INVALID && slot_type != STACK_POISON)
|
| 732 | valid = true;
|
| 733 | types_buf[j] = slot_type_char[slot_type];
|
| 734 | }
|
| 735 | types_buf[BPF_REG_SIZE] = 0;
|
| 736 | if (!valid)
|
| 737 | continue;
|
| 738 |
|
| 739 | reg = &state->stack[i].spilled_ptr;
|
| 740 | switch (state->stack[i].slot_type[BPF_REG_SIZE - 1]) {
|
| 741 | case STACK_SPILL:
|
| 742 |
|
| 743 | for (j = 0; j < BPF_REG_SIZE; j++)
|
| 744 | if (state->stack[i].slot_type[j] == STACK_SPILL)
|
| 745 | break;
|
| 746 | types_buf[j] = '\0';
|
| 747 |
|
| 748 | verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf);
|
| 749 | print_reg_state(env, state, reg);
|
| 750 | break;
|
| 751 | case STACK_DYNPTR:
|
| 752 |
|
| 753 | i += BPF_DYNPTR_NR_SLOTS - 1;
|
| 754 | reg = &state->stack[i].spilled_ptr;
|
| 755 |
|
| 756 | verbose(env, " fp%d", (-i - 1) * BPF_REG_SIZE);
|
| 757 | verbose(env, "=dynptr_%s(", dynptr_type_str(reg->dynptr.type));
|
| 758 | if (reg->id)
|
| 759 | verbose_a("id=%d", reg->id);
|
| 760 | if (reg->parent_id)
|
| 761 | verbose_a("parent_id=%d", reg->parent_id);
|
| 762 | verbose(env, ")");
|
| 763 | break;
|
| 764 | case STACK_ITER:
|
| 765 |
|
| 766 | if (!reg->id)
|
| 767 | continue;
|
| 768 |
|
| 769 | verbose(env, " fp%d=iter_%s(id=%d,state=%s,depth=%u)",
|
| 770 | (-i - 1) * BPF_REG_SIZE,
|
| 771 | iter_type_str(reg->iter.btf, reg->iter.btf_id),
|
| 772 | reg->id, iter_state_str(reg->iter.state),
|
| 773 | reg->iter.depth);
|
| 774 | break;
|
| 775 | case STACK_MISC:
|
| 776 | case STACK_ZERO:
|
| 777 | default:
|
| 778 | verbose(env, " fp%d=%s", (-i - 1) * BPF_REG_SIZE, types_buf);
|
| 779 | break;
|
| 780 | }
|
| 781 | }
|
| 782 | if (vstate->acquired_refs && vstate->refs[0].id) {
|
| 783 | verbose(env, " refs=%d", vstate->refs[0].id);
|
| 784 | for (i = 1; i < vstate->acquired_refs; i++)
|
| 785 | if (vstate->refs[i].id)
|
| 786 | verbose(env, ",%d", vstate->refs[i].id);
|
| 787 | }
|
| 788 | if (state->in_callback_fn)
|
| 789 | verbose(env, " cb");
|
| 790 | if (state->in_async_callback_fn)
|
| 791 | verbose(env, " async_cb");
|
| 792 | verbose(env, "\n");
|
| 793 | if (!print_all)
|
| 794 | mark_verifier_state_clean(env);
|
| 795 | }
|
| 796 |
|
| 797 | u32 bpf_vlog_alignment(u32 pos)
|
| 798 | {
|
| 799 | return round_up(max(pos + BPF_LOG_MIN_ALIGNMENT / 2, BPF_LOG_ALIGNMENT),
|
| 800 | BPF_LOG_MIN_ALIGNMENT) - pos - 1;
|
| 801 | }
|
| 802 |
|
| 803 | void print_insn_state(struct bpf_verifier_env *env, const struct bpf_verifier_state *vstate,
|
| 804 | u32 frameno)
|
| 805 | {
|
| 806 | if (env->prev_log_pos && env->prev_log_pos == env->log.end_pos) {
|
| 807 |
|
| 808 | bpf_vlog_reset(&env->log, env->prev_log_pos - 1);
|
| 809 | verbose(env, "%*c;", bpf_vlog_alignment(env->prev_insn_print_pos), ' ');
|
| 810 | } else {
|
| 811 | verbose(env, "%d:", env->insn_idx);
|
| 812 | }
|
| 813 | print_verifier_state(env, vstate, frameno, false);
|
| 814 | }
|
| 815 |
|
| 816 | int bpf_log_attr_init(struct bpf_log_attr *log, u64 log_buf, u32 log_size, u32 log_level,
|
| 817 | u32 offsetof_log_true_size, bpfptr_t uattr, struct bpf_common_attr *common,
|
| 818 | bpfptr_t uattr_common, u32 size_common)
|
| 819 | {
|
| 820 | char __user *ubuf_common = u64_to_user_ptr(common->log_buf);
|
| 821 | char __user *ubuf = u64_to_user_ptr(log_buf);
|
| 822 |
|
| 823 | if (!bpf_verifier_log_attr_valid(common->log_level, ubuf_common, common->log_size) ||
|
| 824 | !bpf_verifier_log_attr_valid(log_level, ubuf, log_size))
|
| 825 | return -EINVAL;
|
| 826 |
|
| 827 | if (ubuf && ubuf_common && (ubuf != ubuf_common || log_size != common->log_size ||
|
| 828 | log_level != common->log_level))
|
| 829 | return -EINVAL;
|
| 830 |
|
| 831 | memset(log, 0, sizeof(*log));
|
| 832 | log->ubuf = ubuf;
|
| 833 | log->size = log_size;
|
| 834 | log->level = log_level;
|
| 835 | log->offsetof_true_size = offsetof_log_true_size;
|
| 836 | log->uattr = uattr;
|
| 837 |
|
| 838 | if (!ubuf && ubuf_common) {
|
| 839 | log->ubuf = ubuf_common;
|
| 840 | log->size = common->log_size;
|
| 841 | log->level = common->log_level;
|
| 842 | log->uattr = uattr_common;
|
| 843 | log->offsetof_true_size = 0;
|
| 844 | if (size_common >= offsetofend(struct bpf_common_attr, log_true_size))
|
| 845 | log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size);
|
| 846 | }
|
| 847 | return 0;
|
| 848 | }
|
| 849 |
|
| 850 | struct bpf_verifier_log *bpf_log_attr_create_vlog(struct bpf_log_attr *attr_log,
|
| 851 | struct bpf_common_attr *common, bpfptr_t uattr,
|
| 852 | u32 size)
|
| 853 | {
|
| 854 | struct bpf_verifier_log *log;
|
| 855 | int err;
|
| 856 |
|
| 857 | memset(attr_log, 0, sizeof(*attr_log));
|
| 858 | attr_log->uattr = uattr;
|
| 859 | if (size >= offsetofend(struct bpf_common_attr, log_true_size))
|
| 860 | attr_log->offsetof_true_size = offsetof(struct bpf_common_attr, log_true_size);
|
| 861 |
|
| 862 | if (!size)
|
| 863 | return NULL;
|
| 864 |
|
| 865 | log = kzalloc_obj(*log);
|
| 866 | if (!log)
|
| 867 | return ERR_PTR(-ENOMEM);
|
| 868 |
|
| 869 | err = bpf_vlog_init(log, common->log_level, u64_to_user_ptr(common->log_buf),
|
| 870 | common->log_size);
|
| 871 | if (err) {
|
| 872 | kfree(log);
|
| 873 | return ERR_PTR(err);
|
| 874 | }
|
| 875 |
|
| 876 | return log;
|
| 877 | }
|
| 878 |
|
| 879 | int bpf_log_attr_finalize(struct bpf_log_attr *attr, struct bpf_verifier_log *log)
|
| 880 | {
|
| 881 | u32 log_true_size;
|
| 882 | int err;
|
| 883 |
|
| 884 | err = bpf_vlog_finalize(log, &log_true_size);
|
| 885 |
|
| 886 | if (attr->offsetof_true_size &&
|
| 887 | copy_to_bpfptr_offset(attr->uattr, attr->offsetof_true_size, &log_true_size,
|
| 888 | sizeof(log_true_size)))
|
| 889 | return -EFAULT;
|
| 890 |
|
| 891 | return err;
|
| 892 | }
|