| 1 | |
| 2 |
|
| 3 |
|
| 4 | #include <ctype.h>
|
| 5 | #include <stdlib.h>
|
| 6 | #include <string.h>
|
| 7 | #include <errno.h>
|
| 8 | #include <sys/mman.h>
|
| 9 | #include <alloca.h>
|
| 10 | #include <bpf/bpf.h>
|
| 11 | #include <bpf/libbpf.h>
|
| 12 | #include "disasm.h"
|
| 13 | #include "test_progs.h"
|
| 14 | #include "testing_helpers.h"
|
| 15 | #include <linux/membarrier.h>
|
| 16 | #include <linux/userfaultfd.h>
|
| 17 |
|
| 18 | int parse_num_list(const char *s, bool **num_set, int *num_set_len)
|
| 19 | {
|
| 20 | int i, set_len = 0, new_len, num, start = 0, end = -1;
|
| 21 | bool *set = NULL, *tmp, parsing_end = false;
|
| 22 | char *next;
|
| 23 |
|
| 24 | while (s[0]) {
|
| 25 | errno = 0;
|
| 26 | num = strtol(s, &next, 10);
|
| 27 | if (errno)
|
| 28 | return -errno;
|
| 29 |
|
| 30 | if (parsing_end)
|
| 31 | end = num;
|
| 32 | else
|
| 33 | start = num;
|
| 34 |
|
| 35 | if (!parsing_end && *next == '-') {
|
| 36 | s = next + 1;
|
| 37 | parsing_end = true;
|
| 38 | continue;
|
| 39 | } else if (*next == ',') {
|
| 40 | parsing_end = false;
|
| 41 | s = next + 1;
|
| 42 | end = num;
|
| 43 | } else if (*next == '\0') {
|
| 44 | parsing_end = false;
|
| 45 | s = next;
|
| 46 | end = num;
|
| 47 | } else {
|
| 48 | return -EINVAL;
|
| 49 | }
|
| 50 |
|
| 51 | if (start > end)
|
| 52 | return -EINVAL;
|
| 53 |
|
| 54 | if (end + 1 > set_len) {
|
| 55 | new_len = end + 1;
|
| 56 | tmp = realloc(set, new_len);
|
| 57 | if (!tmp) {
|
| 58 | free(set);
|
| 59 | return -ENOMEM;
|
| 60 | }
|
| 61 | for (i = set_len; i < start; i++)
|
| 62 | tmp[i] = false;
|
| 63 | set = tmp;
|
| 64 | set_len = new_len;
|
| 65 | }
|
| 66 | for (i = start; i <= end; i++)
|
| 67 | set[i] = true;
|
| 68 | }
|
| 69 |
|
| 70 | if (!set || parsing_end)
|
| 71 | return -EINVAL;
|
| 72 |
|
| 73 | *num_set = set;
|
| 74 | *num_set_len = set_len;
|
| 75 |
|
| 76 | return 0;
|
| 77 | }
|
| 78 |
|
| 79 | static int do_insert_test(struct test_filter_set *set,
|
| 80 | char *test_str,
|
| 81 | char *subtest_str)
|
| 82 | {
|
| 83 | struct test_filter *tmp, *test;
|
| 84 | char **ctmp;
|
| 85 | int i;
|
| 86 |
|
| 87 | for (i = 0; i < set->cnt; i++) {
|
| 88 | test = &set->tests[i];
|
| 89 |
|
| 90 | if (strcmp(test_str, test->name) == 0) {
|
| 91 | free(test_str);
|
| 92 | goto subtest;
|
| 93 | }
|
| 94 | }
|
| 95 |
|
| 96 | tmp = realloc(set->tests, sizeof(*test) * (set->cnt + 1));
|
| 97 | if (!tmp)
|
| 98 | return -ENOMEM;
|
| 99 |
|
| 100 | set->tests = tmp;
|
| 101 | test = &set->tests[set->cnt];
|
| 102 |
|
| 103 | test->name = test_str;
|
| 104 | test->subtests = NULL;
|
| 105 | test->subtest_cnt = 0;
|
| 106 |
|
| 107 | set->cnt++;
|
| 108 |
|
| 109 | subtest:
|
| 110 | if (!subtest_str)
|
| 111 | return 0;
|
| 112 |
|
| 113 | for (i = 0; i < test->subtest_cnt; i++) {
|
| 114 | if (strcmp(subtest_str, test->subtests[i]) == 0) {
|
| 115 | free(subtest_str);
|
| 116 | return 0;
|
| 117 | }
|
| 118 | }
|
| 119 |
|
| 120 | ctmp = realloc(test->subtests,
|
| 121 | sizeof(*test->subtests) * (test->subtest_cnt + 1));
|
| 122 | if (!ctmp)
|
| 123 | return -ENOMEM;
|
| 124 |
|
| 125 | test->subtests = ctmp;
|
| 126 | test->subtests[test->subtest_cnt] = subtest_str;
|
| 127 |
|
| 128 | test->subtest_cnt++;
|
| 129 |
|
| 130 | return 0;
|
| 131 | }
|
| 132 |
|
| 133 | static int insert_test(struct test_filter_set *set,
|
| 134 | char *test_spec,
|
| 135 | bool is_glob_pattern)
|
| 136 | {
|
| 137 | char *pattern, *subtest_str, *ext_test_str, *ext_subtest_str = NULL;
|
| 138 | int glob_chars = 0;
|
| 139 |
|
| 140 | if (is_glob_pattern) {
|
| 141 | pattern = "%s";
|
| 142 | } else {
|
| 143 | pattern = "*%s*";
|
| 144 | glob_chars = 2;
|
| 145 | }
|
| 146 |
|
| 147 | subtest_str = strchr(test_spec, '/');
|
| 148 | if (subtest_str) {
|
| 149 | *subtest_str = '\0';
|
| 150 | subtest_str += 1;
|
| 151 | }
|
| 152 |
|
| 153 | ext_test_str = malloc(strlen(test_spec) + glob_chars + 1);
|
| 154 | if (!ext_test_str)
|
| 155 | goto err;
|
| 156 |
|
| 157 | sprintf(ext_test_str, pattern, test_spec);
|
| 158 |
|
| 159 | if (subtest_str) {
|
| 160 | ext_subtest_str = malloc(strlen(subtest_str) + glob_chars + 1);
|
| 161 | if (!ext_subtest_str)
|
| 162 | goto err;
|
| 163 |
|
| 164 | sprintf(ext_subtest_str, pattern, subtest_str);
|
| 165 | }
|
| 166 |
|
| 167 | return do_insert_test(set, ext_test_str, ext_subtest_str);
|
| 168 |
|
| 169 | err:
|
| 170 | free(ext_test_str);
|
| 171 | free(ext_subtest_str);
|
| 172 |
|
| 173 | return -ENOMEM;
|
| 174 | }
|
| 175 |
|
| 176 | int parse_test_list_file(const char *path,
|
| 177 | struct test_filter_set *set,
|
| 178 | bool is_glob_pattern)
|
| 179 | {
|
| 180 | char *buf = NULL, *capture_start, *capture_end, *scan_end;
|
| 181 | size_t buflen = 0;
|
| 182 | int err = 0;
|
| 183 | FILE *f;
|
| 184 |
|
| 185 | f = fopen(path, "r");
|
| 186 | if (!f) {
|
| 187 | err = -errno;
|
| 188 | fprintf(stderr, "Failed to open '%s': %d\n", path, err);
|
| 189 | return err;
|
| 190 | }
|
| 191 |
|
| 192 | while (getline(&buf, &buflen, f) != -1) {
|
| 193 | capture_start = buf;
|
| 194 |
|
| 195 | while (isspace(*capture_start))
|
| 196 | ++capture_start;
|
| 197 |
|
| 198 | capture_end = capture_start;
|
| 199 | scan_end = capture_start;
|
| 200 |
|
| 201 | while (*scan_end && *scan_end != '#') {
|
| 202 | if (!isspace(*scan_end))
|
| 203 | capture_end = scan_end;
|
| 204 |
|
| 205 | ++scan_end;
|
| 206 | }
|
| 207 |
|
| 208 | if (capture_end == capture_start)
|
| 209 | continue;
|
| 210 |
|
| 211 | *(++capture_end) = '\0';
|
| 212 |
|
| 213 | err = insert_test(set, capture_start, is_glob_pattern);
|
| 214 | if (err)
|
| 215 | break;
|
| 216 | }
|
| 217 |
|
| 218 | free(buf);
|
| 219 | fclose(f);
|
| 220 | return err;
|
| 221 | }
|
| 222 |
|
| 223 | int parse_test_list(const char *s,
|
| 224 | struct test_filter_set *set,
|
| 225 | bool is_glob_pattern)
|
| 226 | {
|
| 227 | char *input, *state = NULL, *test_spec;
|
| 228 | int err = 0, cnt = 0;
|
| 229 |
|
| 230 | input = strdup(s);
|
| 231 | if (!input)
|
| 232 | return -ENOMEM;
|
| 233 |
|
| 234 | while ((test_spec = strtok_r(cnt++ ? NULL : input, ",", &state))) {
|
| 235 | err = insert_test(set, test_spec, is_glob_pattern);
|
| 236 | if (err)
|
| 237 | break;
|
| 238 | }
|
| 239 |
|
| 240 | free(input);
|
| 241 | return err;
|
| 242 | }
|
| 243 |
|
| 244 | __u32 link_info_prog_id(const struct bpf_link *link, struct bpf_link_info *info)
|
| 245 | {
|
| 246 | __u32 info_len = sizeof(*info);
|
| 247 | int err;
|
| 248 |
|
| 249 | memset(info, 0, sizeof(*info));
|
| 250 | err = bpf_link_get_info_by_fd(bpf_link__fd(link), info, &info_len);
|
| 251 | if (err) {
|
| 252 | printf("failed to get link info: %d\n", -errno);
|
| 253 | return 0;
|
| 254 | }
|
| 255 | return info->prog_id;
|
| 256 | }
|
| 257 |
|
| 258 | int extra_prog_load_log_flags = 0;
|
| 259 |
|
| 260 | int testing_prog_flags(void)
|
| 261 | {
|
| 262 | static int cached_flags = -1;
|
| 263 | static int prog_flags[] = { BPF_F_TEST_RND_HI32, BPF_F_TEST_REG_INVARIANTS };
|
| 264 | static struct bpf_insn insns[] = {
|
| 265 | BPF_MOV64_IMM(BPF_REG_0, 0),
|
| 266 | BPF_EXIT_INSN(),
|
| 267 | };
|
| 268 | int insn_cnt = ARRAY_SIZE(insns), i, fd, flags = 0;
|
| 269 | LIBBPF_OPTS(bpf_prog_load_opts, opts);
|
| 270 |
|
| 271 | if (cached_flags >= 0)
|
| 272 | return cached_flags;
|
| 273 |
|
| 274 | for (i = 0; i < ARRAY_SIZE(prog_flags); i++) {
|
| 275 | opts.prog_flags = prog_flags[i];
|
| 276 | fd = bpf_prog_load(BPF_PROG_TYPE_SOCKET_FILTER, "flag-test", "GPL",
|
| 277 | insns, insn_cnt, &opts);
|
| 278 | if (fd >= 0) {
|
| 279 | flags |= prog_flags[i];
|
| 280 | close(fd);
|
| 281 | }
|
| 282 | }
|
| 283 |
|
| 284 | cached_flags = flags;
|
| 285 | return cached_flags;
|
| 286 | }
|
| 287 |
|
| 288 | int bpf_prog_test_load(const char *file, enum bpf_prog_type type,
|
| 289 | struct bpf_object **pobj, int *prog_fd)
|
| 290 | {
|
| 291 | LIBBPF_OPTS(bpf_object_open_opts, opts,
|
| 292 | .kernel_log_level = extra_prog_load_log_flags,
|
| 293 | );
|
| 294 | struct bpf_object *obj;
|
| 295 | struct bpf_program *prog;
|
| 296 | __u32 flags;
|
| 297 | int err;
|
| 298 |
|
| 299 | obj = bpf_object__open_file(file, &opts);
|
| 300 | if (!obj)
|
| 301 | return -errno;
|
| 302 |
|
| 303 | prog = bpf_object__next_program(obj, NULL);
|
| 304 | if (!prog) {
|
| 305 | err = -ENOENT;
|
| 306 | goto err_out;
|
| 307 | }
|
| 308 |
|
| 309 | if (type != BPF_PROG_TYPE_UNSPEC && bpf_program__type(prog) != type)
|
| 310 | bpf_program__set_type(prog, type);
|
| 311 |
|
| 312 | flags = bpf_program__flags(prog) | testing_prog_flags();
|
| 313 | bpf_program__set_flags(prog, flags);
|
| 314 |
|
| 315 | err = bpf_object__load(obj);
|
| 316 | if (err)
|
| 317 | goto err_out;
|
| 318 |
|
| 319 | *pobj = obj;
|
| 320 | *prog_fd = bpf_program__fd(prog);
|
| 321 |
|
| 322 | return 0;
|
| 323 | err_out:
|
| 324 | bpf_object__close(obj);
|
| 325 | return err;
|
| 326 | }
|
| 327 |
|
| 328 | int bpf_test_load_program(enum bpf_prog_type type, const struct bpf_insn *insns,
|
| 329 | size_t insns_cnt, const char *license,
|
| 330 | __u32 kern_version, char *log_buf,
|
| 331 | size_t log_buf_sz)
|
| 332 | {
|
| 333 | LIBBPF_OPTS(bpf_prog_load_opts, opts,
|
| 334 | .kern_version = kern_version,
|
| 335 | .prog_flags = testing_prog_flags(),
|
| 336 | .log_level = extra_prog_load_log_flags,
|
| 337 | .log_buf = log_buf,
|
| 338 | .log_size = log_buf_sz,
|
| 339 | );
|
| 340 |
|
| 341 | return bpf_prog_load(type, NULL, license, insns, insns_cnt, &opts);
|
| 342 | }
|
| 343 |
|
| 344 | __u64 read_perf_max_sample_freq(void)
|
| 345 | {
|
| 346 | __u64 sample_freq = 5000;
|
| 347 | FILE *f;
|
| 348 |
|
| 349 | f = fopen("/proc/sys/kernel/perf_event_max_sample_rate", "r");
|
| 350 | if (f == NULL) {
|
| 351 | printf("Failed to open /proc/sys/kernel/perf_event_max_sample_rate: err %d\n"
|
| 352 | "return default value: 5000\n", -errno);
|
| 353 | return sample_freq;
|
| 354 | }
|
| 355 | if (fscanf(f, "%llu", &sample_freq) != 1) {
|
| 356 | printf("Failed to parse /proc/sys/kernel/perf_event_max_sample_rate: err %d\n"
|
| 357 | "return default value: 5000\n", -errno);
|
| 358 | }
|
| 359 |
|
| 360 | fclose(f);
|
| 361 | return sample_freq;
|
| 362 | }
|
| 363 |
|
| 364 | int finit_module(int fd, const char *param_values, int flags)
|
| 365 | {
|
| 366 | return syscall(__NR_finit_module, fd, param_values, flags);
|
| 367 | }
|
| 368 |
|
| 369 | int delete_module(const char *name, int flags)
|
| 370 | {
|
| 371 | return syscall(__NR_delete_module, name, flags);
|
| 372 | }
|
| 373 |
|
| 374 | int try_unload_module(const char *name, int retries, bool verbose)
|
| 375 | {
|
| 376 | int ret, cnt = 0;
|
| 377 |
|
| 378 | if (kern_sync_rcu())
|
| 379 | fprintf(stdout, "Failed to trigger kernel-side RCU sync!\n");
|
| 380 |
|
| 381 | for (;;) {
|
| 382 | ret = delete_module(name, 0);
|
| 383 | if (!ret || errno != EAGAIN)
|
| 384 | break;
|
| 385 | if (++cnt > retries) {
|
| 386 | fprintf(stdout, "Unload of %s timed out\n", name);
|
| 387 | break;
|
| 388 | }
|
| 389 | usleep(100);
|
| 390 | }
|
| 391 |
|
| 392 | if (ret) {
|
| 393 | if (errno == ENOENT) {
|
| 394 | if (verbose)
|
| 395 | fprintf(stdout, "%s.ko is already unloaded.\n", name);
|
| 396 | return -1;
|
| 397 | }
|
| 398 | fprintf(stdout, "Failed to unload %s.ko from kernel: %d\n", name, -errno);
|
| 399 | return -1;
|
| 400 | }
|
| 401 | if (verbose)
|
| 402 | fprintf(stdout, "Successfully unloaded %s.ko.\n", name);
|
| 403 | return 0;
|
| 404 | }
|
| 405 |
|
| 406 | int unload_module(const char *name, bool verbose)
|
| 407 | {
|
| 408 | return try_unload_module(name, 10000, verbose);
|
| 409 | }
|
| 410 |
|
| 411 | static int __load_module(const char *path, const char *param_values, bool verbose)
|
| 412 | {
|
| 413 | int fd;
|
| 414 |
|
| 415 | if (verbose)
|
| 416 | fprintf(stdout, "Loading %s...\n", path);
|
| 417 |
|
| 418 | fd = open(path, O_RDONLY);
|
| 419 | if (fd < 0) {
|
| 420 | fprintf(stdout, "Can't find %s kernel module: %d\n", path, -errno);
|
| 421 | return -ENOENT;
|
| 422 | }
|
| 423 | if (finit_module(fd, param_values, 0)) {
|
| 424 | fprintf(stdout, "Failed to load %s into the kernel: %d\n", path, -errno);
|
| 425 | close(fd);
|
| 426 | return -EINVAL;
|
| 427 | }
|
| 428 | close(fd);
|
| 429 |
|
| 430 | if (verbose)
|
| 431 | fprintf(stdout, "Successfully loaded %s.\n", path);
|
| 432 | return 0;
|
| 433 | }
|
| 434 |
|
| 435 | int load_module_params(const char *path, const char *param_values, bool verbose)
|
| 436 | {
|
| 437 | return __load_module(path, param_values, verbose);
|
| 438 | }
|
| 439 |
|
| 440 | int load_module(const char *path, bool verbose)
|
| 441 | {
|
| 442 | return __load_module(path, "", verbose);
|
| 443 | }
|
| 444 |
|
| 445 | int unload_bpf_testmod(bool verbose)
|
| 446 | {
|
| 447 | return unload_module("bpf_testmod", verbose);
|
| 448 | }
|
| 449 |
|
| 450 | int load_bpf_testmod(bool verbose)
|
| 451 | {
|
| 452 | return load_module("bpf_testmod.ko", verbose);
|
| 453 | }
|
| 454 |
|
| 455 | |
| 456 | * Trigger synchronize_rcu() in kernel.
|
| 457 |
|
| 458 | int kern_sync_rcu(void)
|
| 459 | {
|
| 460 | return syscall(__NR_membarrier, MEMBARRIER_CMD_SHARED, 0, 0);
|
| 461 | }
|
| 462 |
|
| 463 | int uffd_block_page(void *fault_addr)
|
| 464 | {
|
| 465 | struct uffdio_register uffd_register = {};
|
| 466 | struct uffdio_api uffd_api = {};
|
| 467 | int uffd;
|
| 468 |
|
| 469 | uffd = syscall(__NR_userfaultfd, O_CLOEXEC);
|
| 470 | if (uffd < 0)
|
| 471 | return -errno;
|
| 472 |
|
| 473 | uffd_api.api = UFFD_API;
|
| 474 | uffd_api.features = 0;
|
| 475 | if (ioctl(uffd, UFFDIO_API, &uffd_api)) {
|
| 476 | close(uffd);
|
| 477 | return -1;
|
| 478 | }
|
| 479 |
|
| 480 | uffd_register.range.start = (unsigned long)fault_addr;
|
| 481 | uffd_register.range.len = getpagesize();
|
| 482 | uffd_register.mode = UFFDIO_REGISTER_MODE_MISSING;
|
| 483 | if (ioctl(uffd, UFFDIO_REGISTER, &uffd_register)) {
|
| 484 | close(uffd);
|
| 485 | return -1;
|
| 486 | }
|
| 487 | return uffd;
|
| 488 | }
|
| 489 |
|
| 490 | int get_xlated_program(int fd_prog, struct bpf_insn **buf, __u32 *cnt)
|
| 491 | {
|
| 492 | __u32 buf_element_size = sizeof(struct bpf_insn);
|
| 493 | struct bpf_prog_info info = {};
|
| 494 | __u32 info_len = sizeof(info);
|
| 495 | __u32 xlated_prog_len;
|
| 496 |
|
| 497 | if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) {
|
| 498 | perror("bpf_prog_get_info_by_fd failed");
|
| 499 | return -1;
|
| 500 | }
|
| 501 |
|
| 502 | xlated_prog_len = info.xlated_prog_len;
|
| 503 | if (xlated_prog_len % buf_element_size) {
|
| 504 | printf("Program length %u is not multiple of %u\n",
|
| 505 | xlated_prog_len, buf_element_size);
|
| 506 | return -1;
|
| 507 | }
|
| 508 |
|
| 509 | *cnt = xlated_prog_len / buf_element_size;
|
| 510 | *buf = calloc(*cnt, buf_element_size);
|
| 511 | if (!*buf) {
|
| 512 | perror("can't allocate xlated program buffer");
|
| 513 | return -ENOMEM;
|
| 514 | }
|
| 515 |
|
| 516 | bzero(&info, sizeof(info));
|
| 517 | info.xlated_prog_len = xlated_prog_len;
|
| 518 | info.xlated_prog_insns = (__u64)(unsigned long)*buf;
|
| 519 | if (bpf_prog_get_info_by_fd(fd_prog, &info, &info_len)) {
|
| 520 | perror("second bpf_prog_get_info_by_fd failed");
|
| 521 | goto out_free_buf;
|
| 522 | }
|
| 523 |
|
| 524 | return 0;
|
| 525 |
|
| 526 | out_free_buf:
|
| 527 | free(*buf);
|
| 528 | *buf = NULL;
|
| 529 | return -1;
|
| 530 | }
|
| 531 |
|
| 532 | bool is_jit_enabled(void)
|
| 533 | {
|
| 534 | const char *jit_sysctl = "/proc/sys/net/core/bpf_jit_enable";
|
| 535 | bool enabled = false;
|
| 536 | int sysctl_fd;
|
| 537 |
|
| 538 | sysctl_fd = open(jit_sysctl, O_RDONLY);
|
| 539 | if (sysctl_fd != -1) {
|
| 540 | char tmpc;
|
| 541 |
|
| 542 | if (read(sysctl_fd, &tmpc, sizeof(tmpc)) == 1)
|
| 543 | enabled = (tmpc != '0');
|
| 544 | close(sysctl_fd);
|
| 545 | }
|
| 546 |
|
| 547 | return enabled;
|
| 548 | }
|
| 549 |
|
| 550 | int stack_mprotect(void)
|
| 551 | {
|
| 552 | void *buf;
|
| 553 | long sz;
|
| 554 | int ret;
|
| 555 |
|
| 556 | sz = sysconf(_SC_PAGESIZE);
|
| 557 | if (sz < 0)
|
| 558 | return sz;
|
| 559 |
|
| 560 | buf = alloca(sz * 3);
|
| 561 | ret = mprotect((void *)(((unsigned long)(buf + sz)) & ~(sz - 1)), sz,
|
| 562 | PROT_READ | PROT_WRITE | PROT_EXEC);
|
| 563 | return ret;
|
| 564 | }
|