| 1 | |
| 2 |
|
| 3 | #include <ctype.h>
|
| 4 | #include <stdio.h>
|
| 5 | #include <stdlib.h>
|
| 6 | #include <string.h>
|
| 7 | #include <libelf.h>
|
| 8 | #include <gelf.h>
|
| 9 | #include <unistd.h>
|
| 10 | #include <linux/ptrace.h>
|
| 11 | #include <linux/kernel.h>
|
| 12 |
|
| 13 |
|
| 14 | #if defined(__riscv)
|
| 15 | #define rv_s8 s8
|
| 16 | #endif
|
| 17 |
|
| 18 | #include "bpf.h"
|
| 19 | #include "libbpf.h"
|
| 20 | #include "libbpf_common.h"
|
| 21 | #include "libbpf_internal.h"
|
| 22 | #include "hashmap.h"
|
| 23 |
|
| 24 | |
| 25 | * state/code working together in concert. BPF-side parts are defined in
|
| 26 | * usdt.bpf.h header library. User-space state is encapsulated by struct
|
| 27 | * usdt_manager and all the supporting code centered around usdt_manager.
|
| 28 | *
|
| 29 | * usdt.bpf.h defines two BPF maps that usdt_manager expects: USDT spec map
|
| 30 | * and IP-to-spec-ID map, which is auxiliary map necessary for kernels that
|
| 31 | * don't support BPF cookie (see below). These two maps are implicitly
|
| 32 | * embedded into user's end BPF object file when user's code included
|
| 33 | * usdt.bpf.h. This means that libbpf doesn't do anything special to create
|
| 34 | * these USDT support maps. They are created by normal libbpf logic of
|
| 35 | * instantiating BPF maps when opening and loading BPF object.
|
| 36 | *
|
| 37 | * As such, libbpf is basically unaware of the need to do anything
|
| 38 | * USDT-related until the very first call to bpf_program__attach_usdt(), which
|
| 39 | * can be called by user explicitly or happen automatically during skeleton
|
| 40 | * attach (or, equivalently, through generic bpf_program__attach() call). At
|
| 41 | * this point, libbpf will instantiate and initialize struct usdt_manager and
|
| 42 | * store it in bpf_object. USDT manager is per-BPF object construct, as each
|
| 43 | * independent BPF object might or might not have USDT programs, and thus all
|
| 44 | * the expected USDT-related state. There is no coordination between two
|
| 45 | * bpf_object in parts of USDT attachment, they are oblivious of each other's
|
| 46 | * existence and libbpf is just oblivious, dealing with bpf_object-specific
|
| 47 | * USDT state.
|
| 48 | *
|
| 49 | * Quick crash course on USDTs.
|
| 50 | *
|
| 51 | * From user-space application's point of view, USDT is essentially just
|
| 52 | * a slightly special function call that normally has zero overhead, unless it
|
| 53 | * is being traced by some external entity (e.g, BPF-based tool). Here's how
|
| 54 | * a typical application can trigger USDT probe:
|
| 55 | *
|
| 56 | * #include <sys/sdt.h> // provided by systemtap-sdt-devel package
|
| 57 | * // folly also provide similar functionality in folly/tracing/StaticTracepoint.h
|
| 58 | *
|
| 59 | * STAP_PROBE3(my_usdt_provider, my_usdt_probe_name, 123, x, &y);
|
| 60 | *
|
| 61 | * USDT is identified by its <provider-name>:<probe-name> pair of names. Each
|
| 62 | * individual USDT has a fixed number of arguments (3 in the above example)
|
| 63 | * and specifies values of each argument as if it was a function call.
|
| 64 | *
|
| 65 | * USDT call is actually not a function call, but is instead replaced by
|
| 66 | * a single NOP instruction (thus zero overhead, effectively). But in addition
|
| 67 | * to that, those USDT macros generate special SHT_NOTE ELF records in
|
| 68 | * .note.stapsdt ELF section. Here's an example USDT definition as emitted by
|
| 69 | * `readelf -n <binary>`:
|
| 70 | *
|
| 71 | * stapsdt 0x00000089 NT_STAPSDT (SystemTap probe descriptors)
|
| 72 | * Provider: test
|
| 73 | * Name: usdt12
|
| 74 | * Location: 0x0000000000549df3, Base: 0x00000000008effa4, Semaphore: 0x0000000000a4606e
|
| 75 | * Arguments: -4@-1204(%rbp) -4@%edi -8@-1216(%rbp) -8@%r8 -4@$5 -8@%r9 8@%rdx 8@%r10 -4@$-9 -2@%cx -2@%ax -1@%sil
|
| 76 | *
|
| 77 | * In this case we have USDT test:usdt12 with 12 arguments.
|
| 78 | *
|
| 79 | * Location and base are offsets used to calculate absolute IP address of that
|
| 80 | * NOP instruction that kernel can replace with an interrupt instruction to
|
| 81 | * trigger instrumentation code (BPF program for all that we care about).
|
| 82 | *
|
| 83 | * Semaphore above is an optional feature. It records an address of a 2-byte
|
| 84 | * refcount variable (normally in '.probes' ELF section) used for signaling if
|
| 85 | * there is anything that is attached to USDT. This is useful for user
|
| 86 | * applications if, for example, they need to prepare some arguments that are
|
| 87 | * passed only to USDTs and preparation is expensive. By checking if USDT is
|
| 88 | * "activated", an application can avoid paying those costs unnecessarily.
|
| 89 | * Recent enough kernel has built-in support for automatically managing this
|
| 90 | * refcount, which libbpf expects and relies on. If USDT is defined without
|
| 91 | * associated semaphore, this value will be zero. See selftests for semaphore
|
| 92 | * examples.
|
| 93 | *
|
| 94 | * Arguments is the most interesting part. This USDT specification string is
|
| 95 | * providing information about all the USDT arguments and their locations. The
|
| 96 | * part before @ sign defined byte size of the argument (1, 2, 4, or 8) and
|
| 97 | * whether the argument is signed or unsigned (negative size means signed).
|
| 98 | * The part after @ sign is assembly-like definition of argument location
|
| 99 | * (see [0] for more details). Technically, assembler can provide some pretty
|
| 100 | * advanced definitions, but libbpf is currently supporting three most common
|
| 101 | * cases:
|
| 102 | * 1) immediate constant, see 5th and 9th args above (-4@$5 and -4@-9);
|
| 103 | * 2) register value, e.g., 8@%rdx, which means "unsigned 8-byte integer
|
| 104 | * whose value is in register %rdx";
|
| 105 | * 3) memory dereference addressed by register, e.g., -4@-1204(%rbp), which
|
| 106 | * specifies signed 32-bit integer stored at offset -1204 bytes from
|
| 107 | * memory address stored in %rbp.
|
| 108 | *
|
| 109 | * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation
|
| 110 | *
|
| 111 | * During attachment, libbpf parses all the relevant USDT specifications and
|
| 112 | * prepares `struct usdt_spec` (USDT spec), which is then provided to BPF-side
|
| 113 | * code through spec map. This allows BPF applications to quickly fetch the
|
| 114 | * actual value at runtime using a simple BPF-side code.
|
| 115 | *
|
| 116 | * With basics out of the way, let's go over less immediately obvious aspects
|
| 117 | * of supporting USDTs.
|
| 118 | *
|
| 119 | * First, there is no special USDT BPF program type. It is actually just
|
| 120 | * a uprobe BPF program (which for kernel, at least currently, is just a kprobe
|
| 121 | * program, so BPF_PROG_TYPE_KPROBE program type). With the only difference
|
| 122 | * that uprobe is usually attached at the function entry, while USDT will
|
| 123 | * normally be somewhere inside the function. But it should always be
|
| 124 | * pointing to NOP instruction, which makes such uprobes the fastest uprobe
|
| 125 | * kind.
|
| 126 | *
|
| 127 | * Second, it's important to realize that such STAP_PROBEn(provider, name, ...)
|
| 128 | * macro invocations can end up being inlined many-many times, depending on
|
| 129 | * specifics of each individual user application. So single conceptual USDT
|
| 130 | * (identified by provider:name pair of identifiers) is, generally speaking,
|
| 131 | * multiple uprobe locations (USDT call sites) in different places in user
|
| 132 | * application. Further, again due to inlining, each USDT call site might end
|
| 133 | * up having the same argument #N be located in a different place. In one call
|
| 134 | * site it could be a constant, in another will end up in a register, and in
|
| 135 | * yet another could be some other register or even somewhere on the stack.
|
| 136 | *
|
| 137 | * As such, "attaching to USDT" means (in general case) attaching the same
|
| 138 | * uprobe BPF program to multiple target locations in user application, each
|
| 139 | * potentially having a completely different USDT spec associated with it.
|
| 140 | * To wire all this up together libbpf allocates a unique integer spec ID for
|
| 141 | * each unique USDT spec. Spec IDs are allocated as sequential small integers
|
| 142 | * so that they can be used as keys in array BPF map (for performance reasons).
|
| 143 | * Spec ID allocation and accounting is big part of what usdt_manager is
|
| 144 | * about. This state has to be maintained per-BPF object and coordinate
|
| 145 | * between different USDT attachments within the same BPF object.
|
| 146 | *
|
| 147 | * Spec ID is the key in spec BPF map, value is the actual USDT spec layed out
|
| 148 | * as struct usdt_spec. Each invocation of BPF program at runtime needs to
|
| 149 | * know its associated spec ID. It gets it either through BPF cookie, which
|
| 150 | * libbpf sets to spec ID during attach time, or, if kernel is too old to
|
| 151 | * support BPF cookie, through IP-to-spec-ID map that libbpf maintains in such
|
| 152 | * case. The latter means that some modes of operation can't be supported
|
| 153 | * without BPF cookie. Such a mode is attaching to shared library "generically",
|
| 154 | * without specifying target process. In such case, it's impossible to
|
| 155 | * calculate absolute IP addresses for IP-to-spec-ID map, and thus such mode
|
| 156 | * is not supported without BPF cookie support.
|
| 157 | *
|
| 158 | * Note that libbpf is using BPF cookie functionality for its own internal
|
| 159 | * needs, so user itself can't rely on BPF cookie feature. To that end, libbpf
|
| 160 | * provides conceptually equivalent USDT cookie support. It's still u64
|
| 161 | * user-provided value that can be associated with USDT attachment. Note that
|
| 162 | * this will be the same value for all USDT call sites within the same single
|
| 163 | * *logical* USDT attachment. This makes sense because to user attaching to
|
| 164 | * USDT is a single BPF program triggered for singular USDT probe. The fact
|
| 165 | * that this is done at multiple actual locations is a mostly hidden
|
| 166 | * implementation details. This USDT cookie value can be fetched with
|
| 167 | * bpf_usdt_cookie(ctx) API provided by usdt.bpf.h
|
| 168 | *
|
| 169 | * Lastly, while single USDT can have tons of USDT call sites, it doesn't
|
| 170 | * necessarily have that many different USDT specs. It very well might be
|
| 171 | * that 1000 USDT call sites only need 5 different USDT specs, because all the
|
| 172 | * arguments are typically contained in a small set of registers or stack
|
| 173 | * locations. As such, it's wasteful to allocate as many USDT spec IDs as
|
| 174 | * there are USDT call sites. So libbpf tries to be frugal and performs
|
| 175 | * on-the-fly deduplication during a single USDT attachment to only allocate
|
| 176 | * the minimal required amount of unique USDT specs (and thus spec IDs). This
|
| 177 | * is trivially achieved by using USDT spec string (Arguments string from USDT
|
| 178 | * note) as a lookup key in a hashmap. USDT spec string uniquely defines
|
| 179 | * everything about how to fetch USDT arguments, so two USDT call sites
|
| 180 | * sharing USDT spec string can safely share the same USDT spec and spec ID.
|
| 181 | * Note, this spec string deduplication is happening only during the same USDT
|
| 182 | * attachment, so each USDT spec shares the same USDT cookie value. This is
|
| 183 | * not generally true for other USDT attachments within the same BPF object,
|
| 184 | * as even if USDT spec string is the same, USDT cookie value can be
|
| 185 | * different. It was deemed excessive to try to deduplicate across independent
|
| 186 | * USDT attachments by taking into account USDT spec string *and* USDT cookie
|
| 187 | * value, which would complicate spec ID accounting significantly for little
|
| 188 | * gain.
|
| 189 |
|
| 190 |
|
| 191 | #define USDT_BASE_SEC ".stapsdt.base"
|
| 192 | #define USDT_SEMA_SEC ".probes"
|
| 193 | #define USDT_NOTE_SEC ".note.stapsdt"
|
| 194 | #define USDT_NOTE_TYPE 3
|
| 195 | #define USDT_NOTE_NAME "stapsdt"
|
| 196 |
|
| 197 |
|
| 198 | enum usdt_arg_type {
|
| 199 | USDT_ARG_CONST,
|
| 200 | USDT_ARG_REG,
|
| 201 | USDT_ARG_REG_DEREF,
|
| 202 | USDT_ARG_SIB,
|
| 203 | };
|
| 204 |
|
| 205 |
|
| 206 | struct usdt_arg_spec {
|
| 207 | __u64 val_off;
|
| 208 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
| 209 | enum usdt_arg_type arg_type: 8;
|
| 210 | __u16 idx_reg_off: 12;
|
| 211 | __u16 scale_bitshift: 4;
|
| 212 | __u8 __reserved: 8;
|
| 213 | #else
|
| 214 | __u8 __reserved: 8;
|
| 215 | __u16 idx_reg_off: 12;
|
| 216 | __u16 scale_bitshift: 4;
|
| 217 | enum usdt_arg_type arg_type: 8;
|
| 218 | #endif
|
| 219 | short reg_off;
|
| 220 | bool arg_signed;
|
| 221 | char arg_bitshift;
|
| 222 | };
|
| 223 |
|
| 224 |
|
| 225 | #define USDT_MAX_ARG_CNT 12
|
| 226 |
|
| 227 |
|
| 228 | struct usdt_spec {
|
| 229 | struct usdt_arg_spec args[USDT_MAX_ARG_CNT];
|
| 230 | __u64 usdt_cookie;
|
| 231 | short arg_cnt;
|
| 232 | };
|
| 233 |
|
| 234 | struct usdt_note {
|
| 235 | const char *provider;
|
| 236 | const char *name;
|
| 237 | |
| 238 | * "-4@%esi -4@-24(%rbp) -4@%ecx 2@%ax 8@%rdx"
|
| 239 |
|
| 240 | const char *args;
|
| 241 | long loc_addr;
|
| 242 | long base_addr;
|
| 243 | long sema_addr;
|
| 244 | };
|
| 245 |
|
| 246 | struct usdt_target {
|
| 247 | long abs_ip;
|
| 248 | long rel_ip;
|
| 249 | long sema_off;
|
| 250 | struct usdt_spec spec;
|
| 251 | const char *spec_str;
|
| 252 | };
|
| 253 |
|
| 254 | struct usdt_manager {
|
| 255 | struct bpf_map *specs_map;
|
| 256 | struct bpf_map *ip_to_spec_id_map;
|
| 257 |
|
| 258 | int *free_spec_ids;
|
| 259 | size_t free_spec_cnt;
|
| 260 | size_t next_free_spec_id;
|
| 261 |
|
| 262 | bool has_bpf_cookie;
|
| 263 | bool has_sema_refcnt;
|
| 264 | bool has_uprobe_multi;
|
| 265 | bool has_uprobe_syscall;
|
| 266 | };
|
| 267 |
|
| 268 | struct usdt_manager *usdt_manager_new(struct bpf_object *obj)
|
| 269 | {
|
| 270 | static const char *ref_ctr_sysfs_path = "/sys/bus/event_source/devices/uprobe/format/ref_ctr_offset";
|
| 271 | struct usdt_manager *man;
|
| 272 | struct bpf_map *specs_map, *ip_to_spec_id_map;
|
| 273 |
|
| 274 | specs_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_specs");
|
| 275 | ip_to_spec_id_map = bpf_object__find_map_by_name(obj, "__bpf_usdt_ip_to_spec_id");
|
| 276 | if (!specs_map || !ip_to_spec_id_map) {
|
| 277 | pr_warn("usdt: failed to find USDT support BPF maps, did you forget to include bpf/usdt.bpf.h?\n");
|
| 278 | return ERR_PTR(-ESRCH);
|
| 279 | }
|
| 280 |
|
| 281 | man = calloc(1, sizeof(*man));
|
| 282 | if (!man)
|
| 283 | return ERR_PTR(-ENOMEM);
|
| 284 |
|
| 285 | man->specs_map = specs_map;
|
| 286 | man->ip_to_spec_id_map = ip_to_spec_id_map;
|
| 287 |
|
| 288 | |
| 289 | * We don't need IP-to-ID mapping if we can use BPF cookies.
|
| 290 | * Added in: 7adfc6c9b315 ("bpf: Add bpf_get_attach_cookie() BPF helper to access bpf_cookie value")
|
| 291 |
|
| 292 | man->has_bpf_cookie = kernel_supports(obj, FEAT_BPF_COOKIE);
|
| 293 |
|
| 294 | |
| 295 | * If this is not supported, USDTs with semaphores will not be supported.
|
| 296 | * Added in: a6ca88b241d5 ("trace_uprobe: support reference counter in fd-based uprobe")
|
| 297 |
|
| 298 | man->has_sema_refcnt = faccessat(AT_FDCWD, ref_ctr_sysfs_path, F_OK, AT_EACCESS) == 0;
|
| 299 |
|
| 300 | |
| 301 | * Detect kernel support for uprobe multi link to be used for attaching
|
| 302 | * usdt probes.
|
| 303 |
|
| 304 | man->has_uprobe_multi = kernel_supports(obj, FEAT_UPROBE_MULTI_LINK);
|
| 305 |
|
| 306 | |
| 307 | * Detect kernel support for uprobe() syscall, it's presence means we can
|
| 308 | * take advantage of faster nop10 uprobe handling.
|
| 309 | * Added in: 56101b69c919 ("uprobes/x86: Add uprobe syscall to speed up uprobe")
|
| 310 |
|
| 311 | man->has_uprobe_syscall = kernel_supports(obj, FEAT_UPROBE_SYSCALL);
|
| 312 | return man;
|
| 313 | }
|
| 314 |
|
| 315 | void usdt_manager_free(struct usdt_manager *man)
|
| 316 | {
|
| 317 | if (IS_ERR_OR_NULL(man))
|
| 318 | return;
|
| 319 |
|
| 320 | free(man->free_spec_ids);
|
| 321 | free(man);
|
| 322 | }
|
| 323 |
|
| 324 | static int sanity_check_usdt_elf(Elf *elf, const char *path)
|
| 325 | {
|
| 326 | GElf_Ehdr ehdr;
|
| 327 | int endianness;
|
| 328 |
|
| 329 | if (elf_kind(elf) != ELF_K_ELF) {
|
| 330 | pr_warn("usdt: unrecognized ELF kind %u for '%s'\n", elf_kind(elf), path);
|
| 331 | return -EBADF;
|
| 332 | }
|
| 333 |
|
| 334 | switch (gelf_getclass(elf)) {
|
| 335 | case ELFCLASS64:
|
| 336 | if (sizeof(void *) != 8) {
|
| 337 | pr_warn("usdt: attaching to 64-bit ELF binary '%s' is not supported\n", path);
|
| 338 | return -EBADF;
|
| 339 | }
|
| 340 | break;
|
| 341 | case ELFCLASS32:
|
| 342 | if (sizeof(void *) != 4) {
|
| 343 | pr_warn("usdt: attaching to 32-bit ELF binary '%s' is not supported\n", path);
|
| 344 | return -EBADF;
|
| 345 | }
|
| 346 | break;
|
| 347 | default:
|
| 348 | pr_warn("usdt: unsupported ELF class for '%s'\n", path);
|
| 349 | return -EBADF;
|
| 350 | }
|
| 351 |
|
| 352 | if (!gelf_getehdr(elf, &ehdr))
|
| 353 | return -EINVAL;
|
| 354 |
|
| 355 | if (ehdr.e_type != ET_EXEC && ehdr.e_type != ET_DYN) {
|
| 356 | pr_warn("usdt: unsupported type of ELF binary '%s' (%d), only ET_EXEC and ET_DYN are supported\n",
|
| 357 | path, ehdr.e_type);
|
| 358 | return -EBADF;
|
| 359 | }
|
| 360 |
|
| 361 | #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
|
| 362 | endianness = ELFDATA2LSB;
|
| 363 | #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
|
| 364 | endianness = ELFDATA2MSB;
|
| 365 | #else
|
| 366 | # error "Unrecognized __BYTE_ORDER__"
|
| 367 | #endif
|
| 368 | if (endianness != ehdr.e_ident[EI_DATA]) {
|
| 369 | pr_warn("usdt: ELF endianness mismatch for '%s'\n", path);
|
| 370 | return -EBADF;
|
| 371 | }
|
| 372 |
|
| 373 | return 0;
|
| 374 | }
|
| 375 |
|
| 376 | static int find_elf_sec_by_name(Elf *elf, const char *sec_name, GElf_Shdr *shdr, Elf_Scn **scn)
|
| 377 | {
|
| 378 | Elf_Scn *sec = NULL;
|
| 379 | size_t shstrndx;
|
| 380 |
|
| 381 | if (elf_getshdrstrndx(elf, &shstrndx))
|
| 382 | return -EINVAL;
|
| 383 |
|
| 384 |
|
| 385 | if (!elf_rawdata(elf_getscn(elf, shstrndx), NULL))
|
| 386 | return -EINVAL;
|
| 387 |
|
| 388 | while ((sec = elf_nextscn(elf, sec)) != NULL) {
|
| 389 | char *name;
|
| 390 |
|
| 391 | if (!gelf_getshdr(sec, shdr))
|
| 392 | return -EINVAL;
|
| 393 |
|
| 394 | name = elf_strptr(elf, shstrndx, shdr->sh_name);
|
| 395 | if (name && strcmp(sec_name, name) == 0) {
|
| 396 | *scn = sec;
|
| 397 | return 0;
|
| 398 | }
|
| 399 | }
|
| 400 |
|
| 401 | return -ENOENT;
|
| 402 | }
|
| 403 |
|
| 404 | struct elf_seg {
|
| 405 | long start;
|
| 406 | long end;
|
| 407 | long offset;
|
| 408 | bool is_exec;
|
| 409 | };
|
| 410 |
|
| 411 | static int cmp_elf_segs(const void *_a, const void *_b)
|
| 412 | {
|
| 413 | const struct elf_seg *a = _a;
|
| 414 | const struct elf_seg *b = _b;
|
| 415 |
|
| 416 | return a->start < b->start ? -1 : 1;
|
| 417 | }
|
| 418 |
|
| 419 | static int parse_elf_segs(Elf *elf, const char *path, struct elf_seg **segs, size_t *seg_cnt)
|
| 420 | {
|
| 421 | GElf_Phdr phdr;
|
| 422 | size_t n;
|
| 423 | int i, err;
|
| 424 | struct elf_seg *seg;
|
| 425 | void *tmp;
|
| 426 |
|
| 427 | *seg_cnt = 0;
|
| 428 |
|
| 429 | if (elf_getphdrnum(elf, &n)) {
|
| 430 | err = -errno;
|
| 431 | return err;
|
| 432 | }
|
| 433 |
|
| 434 | for (i = 0; i < n; i++) {
|
| 435 | if (!gelf_getphdr(elf, i, &phdr)) {
|
| 436 | err = -errno;
|
| 437 | return err;
|
| 438 | }
|
| 439 |
|
| 440 | pr_debug("usdt: discovered PHDR #%d in '%s': vaddr 0x%lx memsz 0x%lx offset 0x%lx type 0x%lx flags 0x%lx\n",
|
| 441 | i, path,
|
| 442 | (unsigned long)phdr.p_vaddr, (unsigned long)phdr.p_memsz, (unsigned long)phdr.p_offset,
|
| 443 | (unsigned long)phdr.p_type, (unsigned long)phdr.p_flags);
|
| 444 | if (phdr.p_type != PT_LOAD)
|
| 445 | continue;
|
| 446 |
|
| 447 | tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs));
|
| 448 | if (!tmp)
|
| 449 | return -ENOMEM;
|
| 450 |
|
| 451 | *segs = tmp;
|
| 452 | seg = *segs + *seg_cnt;
|
| 453 | (*seg_cnt)++;
|
| 454 |
|
| 455 | seg->start = phdr.p_vaddr;
|
| 456 | seg->end = phdr.p_vaddr + phdr.p_memsz;
|
| 457 | seg->offset = phdr.p_offset;
|
| 458 | seg->is_exec = phdr.p_flags & PF_X;
|
| 459 | }
|
| 460 |
|
| 461 | if (*seg_cnt == 0) {
|
| 462 | pr_warn("usdt: failed to find PT_LOAD program headers in '%s'\n", path);
|
| 463 | return -ESRCH;
|
| 464 | }
|
| 465 |
|
| 466 | qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs);
|
| 467 | return 0;
|
| 468 | }
|
| 469 |
|
| 470 | static int parse_vma_segs(int pid, const char *lib_path, struct elf_seg **segs, size_t *seg_cnt)
|
| 471 | {
|
| 472 | char path[PATH_MAX], line[4096], mode[16];
|
| 473 | size_t seg_start, seg_end, seg_off;
|
| 474 | struct elf_seg *seg;
|
| 475 | int tmp_pid, n, i, err;
|
| 476 | FILE *f;
|
| 477 |
|
| 478 | *seg_cnt = 0;
|
| 479 |
|
| 480 | |
| 481 | * /proc/<pid>/root/<path>. They will be reported as just /<path> in
|
| 482 | * /proc/<pid>/maps.
|
| 483 |
|
| 484 |
|
| 485 | n = 0;
|
| 486 | if (sscanf(lib_path, "/proc/%d/root%n", &tmp_pid, &n) == 1 &&
|
| 487 | n > 0 && pid == tmp_pid && lib_path[n] == '/') {
|
| 488 | libbpf_strlcpy(path, lib_path + n, sizeof(path));
|
| 489 | goto proceed;
|
| 490 | }
|
| 491 |
|
| 492 | if (!realpath(lib_path, path)) {
|
| 493 | pr_warn("usdt: failed to get absolute path of '%s' (err %s), using path as is...\n",
|
| 494 | lib_path, errstr(-errno));
|
| 495 | libbpf_strlcpy(path, lib_path, sizeof(path));
|
| 496 | }
|
| 497 |
|
| 498 | proceed:
|
| 499 | sprintf(line, "/proc/%d/maps", pid);
|
| 500 | f = fopen(line, "re");
|
| 501 | if (!f) {
|
| 502 | err = -errno;
|
| 503 | pr_warn("usdt: failed to open '%s' to get base addr of '%s': %s\n",
|
| 504 | line, lib_path, errstr(err));
|
| 505 | return err;
|
| 506 | }
|
| 507 |
|
| 508 | |
| 509 | *
|
| 510 | * 7f5c6f5d1000-7f5c6f5d3000 rw-p 001c7000 08:04 21238613 /usr/lib64/libc-2.17.so
|
| 511 | * 7f5c6f5d3000-7f5c6f5d8000 rw-p 00000000 00:00 0
|
| 512 | * 7f5c6f5d8000-7f5c6f5d9000 r-xp 00000000 103:01 362990598 /data/users/andriin/linux/tools/bpf/usdt/libhello_usdt.so
|
| 513 | *
|
| 514 | * Some VMA names can be longer than the local buffer. Bound the
|
| 515 | * writes, but still consume the rest of the line.
|
| 516 |
|
| 517 | while (fscanf(f, "%zx-%zx %15s %zx %*s %*d%4095[^\n]%*[^\n]\n",
|
| 518 | &seg_start, &seg_end, mode, &seg_off, line) == 5) {
|
| 519 | void *tmp;
|
| 520 |
|
| 521 | |
| 522 | * without skipping any whitespaces. So we need to strip
|
| 523 | * leading whitespaces manually here
|
| 524 |
|
| 525 | i = 0;
|
| 526 | while (isblank(line[i]))
|
| 527 | i++;
|
| 528 | if (strcmp(line + i, path) != 0)
|
| 529 | continue;
|
| 530 |
|
| 531 | pr_debug("usdt: discovered segment for lib '%s': addrs %zx-%zx mode %s offset %zx\n",
|
| 532 | path, seg_start, seg_end, mode, seg_off);
|
| 533 |
|
| 534 |
|
| 535 | if (mode[2] != 'x')
|
| 536 | continue;
|
| 537 |
|
| 538 | tmp = libbpf_reallocarray(*segs, *seg_cnt + 1, sizeof(**segs));
|
| 539 | if (!tmp) {
|
| 540 | err = -ENOMEM;
|
| 541 | goto err_out;
|
| 542 | }
|
| 543 |
|
| 544 | *segs = tmp;
|
| 545 | seg = *segs + *seg_cnt;
|
| 546 | *seg_cnt += 1;
|
| 547 |
|
| 548 | seg->start = seg_start;
|
| 549 | seg->end = seg_end;
|
| 550 | seg->offset = seg_off;
|
| 551 | seg->is_exec = true;
|
| 552 | }
|
| 553 |
|
| 554 | if (*seg_cnt == 0) {
|
| 555 | pr_warn("usdt: failed to find '%s' (resolved to '%s') within PID %d memory mappings\n",
|
| 556 | lib_path, path, pid);
|
| 557 | err = -ESRCH;
|
| 558 | goto err_out;
|
| 559 | }
|
| 560 |
|
| 561 | qsort(*segs, *seg_cnt, sizeof(**segs), cmp_elf_segs);
|
| 562 | err = 0;
|
| 563 | err_out:
|
| 564 | fclose(f);
|
| 565 | return err;
|
| 566 | }
|
| 567 |
|
| 568 | static struct elf_seg *find_elf_seg(struct elf_seg *segs, size_t seg_cnt, long virtaddr)
|
| 569 | {
|
| 570 | struct elf_seg *seg;
|
| 571 | int i;
|
| 572 |
|
| 573 | |
| 574 | * given virtual address (absolute for executables, relative for
|
| 575 | * libraries) which should match address range of [seg_start, seg_end)
|
| 576 |
|
| 577 | for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
|
| 578 | if (seg->start <= virtaddr && virtaddr < seg->end)
|
| 579 | return seg;
|
| 580 | }
|
| 581 | return NULL;
|
| 582 | }
|
| 583 |
|
| 584 | static struct elf_seg *find_vma_seg(struct elf_seg *segs, size_t seg_cnt, long offset)
|
| 585 | {
|
| 586 | struct elf_seg *seg;
|
| 587 | int i;
|
| 588 |
|
| 589 | |
| 590 | * actually a file offset, so should be fall within logical
|
| 591 | * offset-based range of [offset_start, offset_end)
|
| 592 |
|
| 593 | for (i = 0, seg = segs; i < seg_cnt; i++, seg++) {
|
| 594 | if (seg->offset <= offset && offset < seg->offset + (seg->end - seg->start))
|
| 595 | return seg;
|
| 596 | }
|
| 597 | return NULL;
|
| 598 | }
|
| 599 |
|
| 600 | static int parse_usdt_note(GElf_Nhdr *nhdr, const char *data, size_t name_off,
|
| 601 | size_t desc_off, struct usdt_note *usdt_note);
|
| 602 |
|
| 603 | static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie);
|
| 604 |
|
| 605 | #if defined(__x86_64__)
|
| 606 | static bool has_nop_combo(int fd, long off)
|
| 607 | {
|
| 608 | unsigned char nop_combo[11] = {
|
| 609 | 0x90, 0x66, 0x2e, 0x0f, 0x1f, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00,
|
| 610 | };
|
| 611 | unsigned char buf[11];
|
| 612 |
|
| 613 | if (pread(fd, buf, 11, off) != 11)
|
| 614 | return false;
|
| 615 | return memcmp(buf, nop_combo, 11) == 0;
|
| 616 | }
|
| 617 | #else
|
| 618 | static bool has_nop_combo(int fd, long off)
|
| 619 | {
|
| 620 | return false;
|
| 621 | }
|
| 622 | #endif
|
| 623 |
|
| 624 | static int collect_usdt_targets(struct usdt_manager *man, struct elf_fd *elf_fd, const char *path,
|
| 625 | pid_t pid, const char *usdt_provider, const char *usdt_name,
|
| 626 | __u64 usdt_cookie, struct usdt_target **out_targets,
|
| 627 | size_t *out_target_cnt)
|
| 628 | {
|
| 629 | size_t off, name_off, desc_off, seg_cnt = 0, vma_seg_cnt = 0, target_cnt = 0;
|
| 630 | struct elf_seg *segs = NULL, *vma_segs = NULL;
|
| 631 | struct usdt_target *targets = NULL, *target;
|
| 632 | Elf *elf = elf_fd->elf;
|
| 633 | long base_addr = 0;
|
| 634 | Elf_Scn *notes_scn, *base_scn;
|
| 635 | GElf_Shdr base_shdr, notes_shdr;
|
| 636 | GElf_Ehdr ehdr;
|
| 637 | GElf_Nhdr nhdr;
|
| 638 | Elf_Data *data;
|
| 639 | int err;
|
| 640 |
|
| 641 | *out_targets = NULL;
|
| 642 | *out_target_cnt = 0;
|
| 643 |
|
| 644 | err = find_elf_sec_by_name(elf, USDT_NOTE_SEC, ¬es_shdr, ¬es_scn);
|
| 645 | if (err) {
|
| 646 | pr_warn("usdt: no USDT notes section (%s) found in '%s'\n", USDT_NOTE_SEC, path);
|
| 647 | return err;
|
| 648 | }
|
| 649 |
|
| 650 | if (notes_shdr.sh_type != SHT_NOTE || !gelf_getehdr(elf, &ehdr)) {
|
| 651 | pr_warn("usdt: invalid USDT notes section (%s) in '%s'\n", USDT_NOTE_SEC, path);
|
| 652 | return -EINVAL;
|
| 653 | }
|
| 654 |
|
| 655 | err = parse_elf_segs(elf, path, &segs, &seg_cnt);
|
| 656 | if (err) {
|
| 657 | pr_warn("usdt: failed to process ELF program segments for '%s': %s\n",
|
| 658 | path, errstr(err));
|
| 659 | goto err_out;
|
| 660 | }
|
| 661 |
|
| 662 | |
| 663 | * offset compensation (see a big comment further below)
|
| 664 |
|
| 665 | if (find_elf_sec_by_name(elf, USDT_BASE_SEC, &base_shdr, &base_scn) == 0)
|
| 666 | base_addr = base_shdr.sh_addr;
|
| 667 |
|
| 668 | data = elf_getdata(notes_scn, 0);
|
| 669 | off = 0;
|
| 670 | while ((off = gelf_getnote(data, off, &nhdr, &name_off, &desc_off)) > 0) {
|
| 671 | long usdt_abs_ip, usdt_rel_ip, usdt_sema_off = 0;
|
| 672 | struct usdt_note note;
|
| 673 | struct elf_seg *seg = NULL;
|
| 674 | void *tmp;
|
| 675 |
|
| 676 | err = parse_usdt_note(&nhdr, data->d_buf, name_off, desc_off, ¬e);
|
| 677 | if (err)
|
| 678 | goto err_out;
|
| 679 |
|
| 680 | if (strcmp(note.provider, usdt_provider) != 0 || strcmp(note.name, usdt_name) != 0)
|
| 681 | continue;
|
| 682 |
|
| 683 | |
| 684 | * relevant parts quoted here:
|
| 685 | *
|
| 686 | * Each SDT probe also expands into a non-allocated ELF note. You can
|
| 687 | * find this by looking at SHT_NOTE sections and decoding the format;
|
| 688 | * see below for details. Because the note is non-allocated, it means
|
| 689 | * there is no runtime cost, and also preserved in both stripped files
|
| 690 | * and .debug files.
|
| 691 | *
|
| 692 | * However, this means that prelink won't adjust the note's contents
|
| 693 | * for address offsets. Instead, this is done via the .stapsdt.base
|
| 694 | * section. This is a special section that is added to the text. We
|
| 695 | * will only ever have one of these sections in a final link and it
|
| 696 | * will only ever be one byte long. Nothing about this section itself
|
| 697 | * matters, we just use it as a marker to detect prelink address
|
| 698 | * adjustments.
|
| 699 | *
|
| 700 | * Each probe note records the link-time address of the .stapsdt.base
|
| 701 | * section alongside the probe PC address. The decoder compares the
|
| 702 | * base address stored in the note with the .stapsdt.base section's
|
| 703 | * sh_addr. Initially these are the same, but the section header will
|
| 704 | * be adjusted by prelink. So the decoder applies the difference to
|
| 705 | * the probe PC address to get the correct prelinked PC address; the
|
| 706 | * same adjustment is applied to the semaphore address, if any.
|
| 707 | *
|
| 708 | * [0] https://sourceware.org/systemtap/wiki/UserSpaceProbeImplementation
|
| 709 |
|
| 710 | usdt_abs_ip = note.loc_addr;
|
| 711 | if (base_addr && note.base_addr)
|
| 712 | usdt_abs_ip += base_addr - note.base_addr;
|
| 713 |
|
| 714 | |
| 715 | * kernel expects file offset to be specified, not a relative
|
| 716 | * virtual address, so we need to translate virtual address to
|
| 717 | * file offset, for both ET_EXEC and ET_DYN binaries.
|
| 718 |
|
| 719 | seg = find_elf_seg(segs, seg_cnt, usdt_abs_ip);
|
| 720 | if (!seg) {
|
| 721 | err = -ESRCH;
|
| 722 | pr_warn("usdt: failed to find ELF program segment for '%s:%s' in '%s' at IP 0x%lx\n",
|
| 723 | usdt_provider, usdt_name, path, (unsigned long)usdt_abs_ip);
|
| 724 | goto err_out;
|
| 725 | }
|
| 726 | if (!seg->is_exec) {
|
| 727 | err = -ESRCH;
|
| 728 | pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx) for '%s:%s' at IP 0x%lx is not executable\n",
|
| 729 | path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name,
|
| 730 | (unsigned long)usdt_abs_ip);
|
| 731 | goto err_out;
|
| 732 | }
|
| 733 |
|
| 734 | usdt_rel_ip = usdt_abs_ip - seg->start + seg->offset;
|
| 735 |
|
| 736 | if (ehdr.e_type == ET_DYN && !man->has_bpf_cookie) {
|
| 737 | |
| 738 | * attach to a shared library, we'll need to know and
|
| 739 | * record absolute addresses of attach points due to
|
| 740 | * the need to lookup USDT spec by absolute IP of
|
| 741 | * triggered uprobe. Doing this resolution is only
|
| 742 | * possible when we have a specific PID of the process
|
| 743 | * that's using specified shared library. BPF cookie
|
| 744 | * removes the absolute address limitation as we don't
|
| 745 | * need to do this lookup (we just use BPF cookie as
|
| 746 | * an index of USDT spec), so for newer kernels with
|
| 747 | * BPF cookie support libbpf supports USDT attachment
|
| 748 | * to shared libraries with no PID filter.
|
| 749 |
|
| 750 | if (pid < 0) {
|
| 751 | pr_warn("usdt: attaching to shared libraries without specific PID is not supported on current kernel\n");
|
| 752 | err = -ENOTSUP;
|
| 753 | goto err_out;
|
| 754 | }
|
| 755 |
|
| 756 |
|
| 757 | if (vma_seg_cnt == 0) {
|
| 758 | err = parse_vma_segs(pid, path, &vma_segs, &vma_seg_cnt);
|
| 759 | if (err) {
|
| 760 | pr_warn("usdt: failed to get memory segments in PID %d for shared library '%s': %s\n",
|
| 761 | pid, path, errstr(err));
|
| 762 | goto err_out;
|
| 763 | }
|
| 764 | }
|
| 765 |
|
| 766 | seg = find_vma_seg(vma_segs, vma_seg_cnt, usdt_rel_ip);
|
| 767 | if (!seg) {
|
| 768 | err = -ESRCH;
|
| 769 | pr_warn("usdt: failed to find shared lib memory segment for '%s:%s' in '%s' at relative IP 0x%lx\n",
|
| 770 | usdt_provider, usdt_name, path, (unsigned long)usdt_rel_ip);
|
| 771 | goto err_out;
|
| 772 | }
|
| 773 |
|
| 774 | usdt_abs_ip = seg->start - seg->offset + usdt_rel_ip;
|
| 775 | }
|
| 776 |
|
| 777 | pr_debug("usdt: probe for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved abs_ip 0x%lx rel_ip 0x%lx) args '%s' in segment [0x%lx, 0x%lx) at offset 0x%lx\n",
|
| 778 | usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ", path,
|
| 779 | (unsigned long)note.loc_addr, (unsigned long)note.base_addr,
|
| 780 | (unsigned long)usdt_abs_ip, (unsigned long)usdt_rel_ip, note.args,
|
| 781 | (unsigned long)(seg ? seg->start : 0), (unsigned long)(seg ? seg->end : 0),
|
| 782 | (unsigned long)(seg ? seg->offset : 0));
|
| 783 |
|
| 784 |
|
| 785 | if (note.sema_addr) {
|
| 786 | if (!man->has_sema_refcnt) {
|
| 787 | pr_warn("usdt: kernel doesn't support USDT semaphore refcounting for '%s:%s' in '%s'\n",
|
| 788 | usdt_provider, usdt_name, path);
|
| 789 | err = -ENOTSUP;
|
| 790 | goto err_out;
|
| 791 | }
|
| 792 |
|
| 793 | seg = find_elf_seg(segs, seg_cnt, note.sema_addr);
|
| 794 | if (!seg) {
|
| 795 | err = -ESRCH;
|
| 796 | pr_warn("usdt: failed to find ELF loadable segment with semaphore of '%s:%s' in '%s' at 0x%lx\n",
|
| 797 | usdt_provider, usdt_name, path, (unsigned long)note.sema_addr);
|
| 798 | goto err_out;
|
| 799 | }
|
| 800 | if (seg->is_exec) {
|
| 801 | err = -ESRCH;
|
| 802 | pr_warn("usdt: matched ELF binary '%s' segment [0x%lx, 0x%lx] for semaphore of '%s:%s' at 0x%lx is executable\n",
|
| 803 | path, (unsigned long)seg->start, (unsigned long)seg->end, usdt_provider, usdt_name,
|
| 804 | (unsigned long)note.sema_addr);
|
| 805 | goto err_out;
|
| 806 | }
|
| 807 |
|
| 808 | usdt_sema_off = note.sema_addr - seg->start + seg->offset;
|
| 809 |
|
| 810 | pr_debug("usdt: sema for '%s:%s' in %s '%s': addr 0x%lx base 0x%lx (resolved 0x%lx) in segment [0x%lx, 0x%lx] at offset 0x%lx\n",
|
| 811 | usdt_provider, usdt_name, ehdr.e_type == ET_EXEC ? "exec" : "lib ",
|
| 812 | path, (unsigned long)note.sema_addr, (unsigned long)note.base_addr, (unsigned long)usdt_sema_off,
|
| 813 | (unsigned long)seg->start, (unsigned long)seg->end, (unsigned long)seg->offset);
|
| 814 | }
|
| 815 |
|
| 816 |
|
| 817 | tmp = libbpf_reallocarray(targets, target_cnt + 1, sizeof(*targets));
|
| 818 | if (!tmp) {
|
| 819 | err = -ENOMEM;
|
| 820 | goto err_out;
|
| 821 | }
|
| 822 | targets = tmp;
|
| 823 |
|
| 824 | target = &targets[target_cnt];
|
| 825 | memset(target, 0, sizeof(*target));
|
| 826 |
|
| 827 | |
| 828 | * We have uprobe syscall and usdt with nop,nop10 instructions combo,
|
| 829 | * so we can place the uprobe directly on nop10 (+1) and get this probe
|
| 830 | * optimized.
|
| 831 |
|
| 832 | if (man->has_uprobe_syscall && has_nop_combo(elf_fd->fd, usdt_rel_ip)) {
|
| 833 | usdt_abs_ip++;
|
| 834 | usdt_rel_ip++;
|
| 835 | }
|
| 836 |
|
| 837 | target->abs_ip = usdt_abs_ip;
|
| 838 | target->rel_ip = usdt_rel_ip;
|
| 839 | target->sema_off = usdt_sema_off;
|
| 840 |
|
| 841 | |
| 842 | * be referenced safely until elf_end() call
|
| 843 |
|
| 844 | target->spec_str = note.args;
|
| 845 |
|
| 846 | err = parse_usdt_spec(&target->spec, ¬e, usdt_cookie);
|
| 847 | if (err)
|
| 848 | goto err_out;
|
| 849 |
|
| 850 | target_cnt++;
|
| 851 | }
|
| 852 |
|
| 853 | *out_targets = targets;
|
| 854 | *out_target_cnt = target_cnt;
|
| 855 | err = target_cnt;
|
| 856 |
|
| 857 | err_out:
|
| 858 | free(segs);
|
| 859 | free(vma_segs);
|
| 860 | if (err < 0)
|
| 861 | free(targets);
|
| 862 | return err;
|
| 863 | }
|
| 864 |
|
| 865 | struct bpf_link_usdt {
|
| 866 | struct bpf_link link;
|
| 867 |
|
| 868 | struct usdt_manager *usdt_man;
|
| 869 |
|
| 870 | size_t spec_cnt;
|
| 871 | int *spec_ids;
|
| 872 |
|
| 873 | size_t uprobe_cnt;
|
| 874 | struct {
|
| 875 | long abs_ip;
|
| 876 | struct bpf_link *link;
|
| 877 | } *uprobes;
|
| 878 |
|
| 879 | struct bpf_link *multi_link;
|
| 880 | };
|
| 881 |
|
| 882 | static int bpf_link_usdt_detach(struct bpf_link *link)
|
| 883 | {
|
| 884 | struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link);
|
| 885 | struct usdt_manager *man = usdt_link->usdt_man;
|
| 886 | int i;
|
| 887 |
|
| 888 | bpf_link__destroy(usdt_link->multi_link);
|
| 889 |
|
| 890 |
|
| 891 | for (i = 0; i < usdt_link->uprobe_cnt; i++) {
|
| 892 |
|
| 893 | bpf_link__destroy(usdt_link->uprobes[i].link);
|
| 894 | |
| 895 | * unconditionally overwritten on subsequent USDT attaches,
|
| 896 | * but if BPF cookies are not used we need to remove entry
|
| 897 | * from ip_to_spec_id map, otherwise we'll run into false
|
| 898 | * conflicting IP errors
|
| 899 |
|
| 900 | if (!man->has_bpf_cookie) {
|
| 901 |
|
| 902 | (void)bpf_map_delete_elem(bpf_map__fd(man->ip_to_spec_id_map),
|
| 903 | &usdt_link->uprobes[i].abs_ip);
|
| 904 | }
|
| 905 | }
|
| 906 |
|
| 907 | |
| 908 | * for future reuse for subsequent USDT attaches
|
| 909 |
|
| 910 | if (!man->free_spec_ids) {
|
| 911 |
|
| 912 | man->free_spec_ids = usdt_link->spec_ids;
|
| 913 | man->free_spec_cnt = usdt_link->spec_cnt;
|
| 914 | usdt_link->spec_ids = NULL;
|
| 915 | } else {
|
| 916 |
|
| 917 | size_t new_cnt = man->free_spec_cnt + usdt_link->spec_cnt;
|
| 918 | int *new_free_ids;
|
| 919 |
|
| 920 | new_free_ids = libbpf_reallocarray(man->free_spec_ids, new_cnt,
|
| 921 | sizeof(*new_free_ids));
|
| 922 | |
| 923 | * a bunch of free IDs; this is very unlikely to happen and if
|
| 924 | * system is so exhausted on memory, it's the least of user's
|
| 925 | * concerns, probably.
|
| 926 | * So just do our best here to return those IDs to usdt_manager.
|
| 927 | * Another edge case when we can legitimately get NULL is when
|
| 928 | * new_cnt is zero, which can happen in some edge cases, so we
|
| 929 | * need to be careful about that.
|
| 930 |
|
| 931 | if (new_free_ids || new_cnt == 0) {
|
| 932 | memcpy(new_free_ids + man->free_spec_cnt, usdt_link->spec_ids,
|
| 933 | usdt_link->spec_cnt * sizeof(*usdt_link->spec_ids));
|
| 934 | man->free_spec_ids = new_free_ids;
|
| 935 | man->free_spec_cnt = new_cnt;
|
| 936 | }
|
| 937 | }
|
| 938 |
|
| 939 | return 0;
|
| 940 | }
|
| 941 |
|
| 942 | static void bpf_link_usdt_dealloc(struct bpf_link *link)
|
| 943 | {
|
| 944 | struct bpf_link_usdt *usdt_link = container_of(link, struct bpf_link_usdt, link);
|
| 945 |
|
| 946 | free(usdt_link->spec_ids);
|
| 947 | free(usdt_link->uprobes);
|
| 948 | free(usdt_link);
|
| 949 | }
|
| 950 |
|
| 951 | static size_t specs_hash_fn(long key, void *ctx)
|
| 952 | {
|
| 953 | return str_hash((char *)key);
|
| 954 | }
|
| 955 |
|
| 956 | static bool specs_equal_fn(long key1, long key2, void *ctx)
|
| 957 | {
|
| 958 | return strcmp((char *)key1, (char *)key2) == 0;
|
| 959 | }
|
| 960 |
|
| 961 | static int allocate_spec_id(struct usdt_manager *man, struct hashmap *specs_hash,
|
| 962 | struct bpf_link_usdt *link, struct usdt_target *target,
|
| 963 | int *spec_id, bool *is_new)
|
| 964 | {
|
| 965 | long tmp;
|
| 966 | void *new_ids;
|
| 967 | int err;
|
| 968 |
|
| 969 |
|
| 970 | if (hashmap__find(specs_hash, target->spec_str, &tmp)) {
|
| 971 | *spec_id = tmp;
|
| 972 | *is_new = false;
|
| 973 | return 0;
|
| 974 | }
|
| 975 |
|
| 976 | |
| 977 | * returned back to usdt_manager when USDT link is detached
|
| 978 |
|
| 979 | new_ids = libbpf_reallocarray(link->spec_ids, link->spec_cnt + 1, sizeof(*link->spec_ids));
|
| 980 | if (!new_ids)
|
| 981 | return -ENOMEM;
|
| 982 | link->spec_ids = new_ids;
|
| 983 |
|
| 984 |
|
| 985 | if (man->free_spec_cnt) {
|
| 986 | *spec_id = man->free_spec_ids[man->free_spec_cnt - 1];
|
| 987 |
|
| 988 |
|
| 989 | err = hashmap__add(specs_hash, target->spec_str, *spec_id);
|
| 990 | if (err)
|
| 991 | return err;
|
| 992 |
|
| 993 | man->free_spec_cnt--;
|
| 994 | } else {
|
| 995 |
|
| 996 | if (man->next_free_spec_id >= bpf_map__max_entries(man->specs_map))
|
| 997 | return -E2BIG;
|
| 998 |
|
| 999 | *spec_id = man->next_free_spec_id;
|
| 1000 |
|
| 1001 |
|
| 1002 | err = hashmap__add(specs_hash, target->spec_str, *spec_id);
|
| 1003 | if (err)
|
| 1004 | return err;
|
| 1005 |
|
| 1006 | man->next_free_spec_id++;
|
| 1007 | }
|
| 1008 |
|
| 1009 |
|
| 1010 | link->spec_ids[link->spec_cnt] = *spec_id;
|
| 1011 | link->spec_cnt++;
|
| 1012 | *is_new = true;
|
| 1013 | return 0;
|
| 1014 | }
|
| 1015 |
|
| 1016 | struct bpf_link *usdt_manager_attach_usdt(struct usdt_manager *man, const struct bpf_program *prog,
|
| 1017 | pid_t pid, const char *path,
|
| 1018 | const char *usdt_provider, const char *usdt_name,
|
| 1019 | __u64 usdt_cookie)
|
| 1020 | {
|
| 1021 | unsigned long *offsets = NULL, *ref_ctr_offsets = NULL;
|
| 1022 | int i, err, spec_map_fd, ip_map_fd;
|
| 1023 | LIBBPF_OPTS(bpf_uprobe_opts, opts);
|
| 1024 | struct hashmap *specs_hash = NULL;
|
| 1025 | struct bpf_link_usdt *link = NULL;
|
| 1026 | struct usdt_target *targets = NULL;
|
| 1027 | __u64 *cookies = NULL;
|
| 1028 | struct elf_fd elf_fd;
|
| 1029 | size_t target_cnt;
|
| 1030 |
|
| 1031 | spec_map_fd = bpf_map__fd(man->specs_map);
|
| 1032 | ip_map_fd = bpf_map__fd(man->ip_to_spec_id_map);
|
| 1033 |
|
| 1034 | err = elf_open(path, &elf_fd);
|
| 1035 | if (err)
|
| 1036 | return libbpf_err_ptr(err);
|
| 1037 |
|
| 1038 | err = sanity_check_usdt_elf(elf_fd.elf, path);
|
| 1039 | if (err)
|
| 1040 | goto err_out;
|
| 1041 |
|
| 1042 |
|
| 1043 | if (pid < 0)
|
| 1044 | pid = -1;
|
| 1045 | else if (pid == 0)
|
| 1046 | pid = getpid();
|
| 1047 |
|
| 1048 | |
| 1049 | * activations to a given PID, if pid > 0
|
| 1050 |
|
| 1051 | err = collect_usdt_targets(man, &elf_fd, path, pid, usdt_provider, usdt_name,
|
| 1052 | usdt_cookie, &targets, &target_cnt);
|
| 1053 | if (err <= 0) {
|
| 1054 | err = (err == 0) ? -ENOENT : err;
|
| 1055 | goto err_out;
|
| 1056 | }
|
| 1057 |
|
| 1058 | specs_hash = hashmap__new(specs_hash_fn, specs_equal_fn, NULL);
|
| 1059 | if (IS_ERR(specs_hash)) {
|
| 1060 | err = PTR_ERR(specs_hash);
|
| 1061 | goto err_out;
|
| 1062 | }
|
| 1063 |
|
| 1064 | link = calloc(1, sizeof(*link));
|
| 1065 | if (!link) {
|
| 1066 | err = -ENOMEM;
|
| 1067 | goto err_out;
|
| 1068 | }
|
| 1069 |
|
| 1070 | link->usdt_man = man;
|
| 1071 | link->link.detach = &bpf_link_usdt_detach;
|
| 1072 | link->link.dealloc = &bpf_link_usdt_dealloc;
|
| 1073 |
|
| 1074 | if (man->has_uprobe_multi) {
|
| 1075 | offsets = calloc(target_cnt, sizeof(*offsets));
|
| 1076 | cookies = calloc(target_cnt, sizeof(*cookies));
|
| 1077 | ref_ctr_offsets = calloc(target_cnt, sizeof(*ref_ctr_offsets));
|
| 1078 |
|
| 1079 | if (!offsets || !ref_ctr_offsets || !cookies) {
|
| 1080 | err = -ENOMEM;
|
| 1081 | goto err_out;
|
| 1082 | }
|
| 1083 | } else {
|
| 1084 | link->uprobes = calloc(target_cnt, sizeof(*link->uprobes));
|
| 1085 | if (!link->uprobes) {
|
| 1086 | err = -ENOMEM;
|
| 1087 | goto err_out;
|
| 1088 | }
|
| 1089 | }
|
| 1090 |
|
| 1091 | for (i = 0; i < target_cnt; i++) {
|
| 1092 | struct usdt_target *target = &targets[i];
|
| 1093 | struct bpf_link *uprobe_link;
|
| 1094 | bool is_new;
|
| 1095 | int spec_id;
|
| 1096 |
|
| 1097 | |
| 1098 | * newly allocated, we'll need to fill out spec map, otherwise
|
| 1099 | * entire spec should be valid and can be just used by a new
|
| 1100 | * uprobe. We reuse spec when USDT arg spec is identical. We
|
| 1101 | * also never share specs between two different USDT
|
| 1102 | * attachments ("links"), so all the reused specs already
|
| 1103 | * share USDT cookie value implicitly.
|
| 1104 |
|
| 1105 | err = allocate_spec_id(man, specs_hash, link, target, &spec_id, &is_new);
|
| 1106 | if (err)
|
| 1107 | goto err_out;
|
| 1108 |
|
| 1109 | if (is_new && bpf_map_update_elem(spec_map_fd, &spec_id, &target->spec, BPF_ANY)) {
|
| 1110 | err = -errno;
|
| 1111 | pr_warn("usdt: failed to set USDT spec #%d for '%s:%s' in '%s': %s\n",
|
| 1112 | spec_id, usdt_provider, usdt_name, path, errstr(err));
|
| 1113 | goto err_out;
|
| 1114 | }
|
| 1115 | if (!man->has_bpf_cookie &&
|
| 1116 | bpf_map_update_elem(ip_map_fd, &target->abs_ip, &spec_id, BPF_NOEXIST)) {
|
| 1117 | err = -errno;
|
| 1118 | if (err == -EEXIST) {
|
| 1119 | pr_warn("usdt: IP collision detected for spec #%d for '%s:%s' in '%s'\n",
|
| 1120 | spec_id, usdt_provider, usdt_name, path);
|
| 1121 | } else {
|
| 1122 | pr_warn("usdt: failed to map IP 0x%lx to spec #%d for '%s:%s' in '%s': %s\n",
|
| 1123 | (unsigned long)target->abs_ip, spec_id, usdt_provider, usdt_name,
|
| 1124 | path, errstr(err));
|
| 1125 | }
|
| 1126 | goto err_out;
|
| 1127 | }
|
| 1128 |
|
| 1129 | if (man->has_uprobe_multi) {
|
| 1130 | offsets[i] = target->rel_ip;
|
| 1131 | ref_ctr_offsets[i] = target->sema_off;
|
| 1132 | cookies[i] = spec_id;
|
| 1133 | } else {
|
| 1134 | opts.ref_ctr_offset = target->sema_off;
|
| 1135 | opts.bpf_cookie = man->has_bpf_cookie ? spec_id : 0;
|
| 1136 | uprobe_link = bpf_program__attach_uprobe_opts(prog, pid, path,
|
| 1137 | target->rel_ip, &opts);
|
| 1138 | err = libbpf_get_error(uprobe_link);
|
| 1139 | if (err) {
|
| 1140 | pr_warn("usdt: failed to attach uprobe #%d for '%s:%s' in '%s': %s\n",
|
| 1141 | i, usdt_provider, usdt_name, path, errstr(err));
|
| 1142 | goto err_out;
|
| 1143 | }
|
| 1144 |
|
| 1145 | link->uprobes[i].link = uprobe_link;
|
| 1146 | link->uprobes[i].abs_ip = target->abs_ip;
|
| 1147 | link->uprobe_cnt++;
|
| 1148 | }
|
| 1149 | }
|
| 1150 |
|
| 1151 | if (man->has_uprobe_multi) {
|
| 1152 | LIBBPF_OPTS(bpf_uprobe_multi_opts, opts_multi,
|
| 1153 | .ref_ctr_offsets = ref_ctr_offsets,
|
| 1154 | .offsets = offsets,
|
| 1155 | .cookies = cookies,
|
| 1156 | .cnt = target_cnt,
|
| 1157 | );
|
| 1158 |
|
| 1159 | link->multi_link = bpf_program__attach_uprobe_multi(prog, pid, path,
|
| 1160 | NULL, &opts_multi);
|
| 1161 | if (!link->multi_link) {
|
| 1162 | err = -errno;
|
| 1163 | pr_warn("usdt: failed to attach uprobe multi for '%s:%s' in '%s': %s\n",
|
| 1164 | usdt_provider, usdt_name, path, errstr(err));
|
| 1165 | goto err_out;
|
| 1166 | }
|
| 1167 |
|
| 1168 | free(offsets);
|
| 1169 | free(ref_ctr_offsets);
|
| 1170 | free(cookies);
|
| 1171 | }
|
| 1172 |
|
| 1173 | free(targets);
|
| 1174 | hashmap__free(specs_hash);
|
| 1175 | elf_close(&elf_fd);
|
| 1176 | return &link->link;
|
| 1177 |
|
| 1178 | err_out:
|
| 1179 | free(offsets);
|
| 1180 | free(ref_ctr_offsets);
|
| 1181 | free(cookies);
|
| 1182 |
|
| 1183 | if (link)
|
| 1184 | bpf_link__destroy(&link->link);
|
| 1185 | free(targets);
|
| 1186 | hashmap__free(specs_hash);
|
| 1187 | elf_close(&elf_fd);
|
| 1188 | return libbpf_err_ptr(err);
|
| 1189 | }
|
| 1190 |
|
| 1191 | |
| 1192 | * Logic inspired by perf's code.
|
| 1193 |
|
| 1194 | static int parse_usdt_note(GElf_Nhdr *nhdr, const char *data, size_t name_off, size_t desc_off,
|
| 1195 | struct usdt_note *note)
|
| 1196 | {
|
| 1197 | const char *provider, *name, *args;
|
| 1198 | long addrs[3];
|
| 1199 | size_t len;
|
| 1200 |
|
| 1201 |
|
| 1202 | if (strncmp(data + name_off, USDT_NOTE_NAME, nhdr->n_namesz) != 0)
|
| 1203 | return -EINVAL;
|
| 1204 | if (nhdr->n_type != USDT_NOTE_TYPE)
|
| 1205 | return -EINVAL;
|
| 1206 |
|
| 1207 |
|
| 1208 | len = nhdr->n_descsz;
|
| 1209 | data = data + desc_off;
|
| 1210 |
|
| 1211 |
|
| 1212 | if (len < sizeof(addrs) + 3)
|
| 1213 | return -EINVAL;
|
| 1214 |
|
| 1215 |
|
| 1216 | memcpy(&addrs, data, sizeof(addrs));
|
| 1217 |
|
| 1218 |
|
| 1219 | provider = data + sizeof(addrs);
|
| 1220 |
|
| 1221 | name = (const char *)memchr(provider, '\0', data + len - provider);
|
| 1222 | if (!name)
|
| 1223 | return -EINVAL;
|
| 1224 | name++;
|
| 1225 | if (name >= data + len || *name == '\0')
|
| 1226 | return -EINVAL;
|
| 1227 |
|
| 1228 | args = memchr(name, '\0', data + len - name);
|
| 1229 | if (!args)
|
| 1230 | return -EINVAL;
|
| 1231 | ++args;
|
| 1232 | if (args >= data + len)
|
| 1233 | return -EINVAL;
|
| 1234 |
|
| 1235 | note->provider = provider;
|
| 1236 | note->name = name;
|
| 1237 | if (*args == '\0' || *args == ':')
|
| 1238 | note->args = "";
|
| 1239 | else
|
| 1240 | note->args = args;
|
| 1241 | note->loc_addr = addrs[0];
|
| 1242 | note->base_addr = addrs[1];
|
| 1243 | note->sema_addr = addrs[2];
|
| 1244 |
|
| 1245 | return 0;
|
| 1246 | }
|
| 1247 |
|
| 1248 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz);
|
| 1249 |
|
| 1250 | static int parse_usdt_spec(struct usdt_spec *spec, const struct usdt_note *note, __u64 usdt_cookie)
|
| 1251 | {
|
| 1252 | struct usdt_arg_spec *arg;
|
| 1253 | const char *s;
|
| 1254 | int arg_sz, len;
|
| 1255 |
|
| 1256 | spec->usdt_cookie = usdt_cookie;
|
| 1257 | spec->arg_cnt = 0;
|
| 1258 |
|
| 1259 | s = note->args;
|
| 1260 | while (s[0]) {
|
| 1261 | if (spec->arg_cnt >= USDT_MAX_ARG_CNT) {
|
| 1262 | pr_warn("usdt: too many USDT arguments (> %d) for '%s:%s' with args spec '%s'\n",
|
| 1263 | USDT_MAX_ARG_CNT, note->provider, note->name, note->args);
|
| 1264 | return -E2BIG;
|
| 1265 | }
|
| 1266 |
|
| 1267 | arg = &spec->args[spec->arg_cnt];
|
| 1268 | len = parse_usdt_arg(s, spec->arg_cnt, arg, &arg_sz);
|
| 1269 | if (len < 0)
|
| 1270 | return len;
|
| 1271 |
|
| 1272 | arg->arg_signed = arg_sz < 0;
|
| 1273 | if (arg_sz < 0)
|
| 1274 | arg_sz = -arg_sz;
|
| 1275 |
|
| 1276 | switch (arg_sz) {
|
| 1277 | case 1: case 2: case 4: case 8:
|
| 1278 | arg->arg_bitshift = 64 - arg_sz * 8;
|
| 1279 | break;
|
| 1280 | default:
|
| 1281 | pr_warn("usdt: unsupported arg #%d (spec '%s') size: %d\n",
|
| 1282 | spec->arg_cnt, s, arg_sz);
|
| 1283 | return -EINVAL;
|
| 1284 | }
|
| 1285 |
|
| 1286 | s += len;
|
| 1287 | spec->arg_cnt++;
|
| 1288 | }
|
| 1289 |
|
| 1290 | return 0;
|
| 1291 | }
|
| 1292 |
|
| 1293 |
|
| 1294 |
|
| 1295 | #if defined(__x86_64__) || defined(__i386__)
|
| 1296 |
|
| 1297 | static int calc_pt_regs_off(const char *reg_name)
|
| 1298 | {
|
| 1299 | static struct {
|
| 1300 | const char *names[4];
|
| 1301 | size_t pt_regs_off;
|
| 1302 | } reg_map[] = {
|
| 1303 | #ifdef __x86_64__
|
| 1304 | #define reg_off(reg64, reg32) offsetof(struct pt_regs, reg64)
|
| 1305 | #else
|
| 1306 | #define reg_off(reg64, reg32) offsetof(struct pt_regs, reg32)
|
| 1307 | #endif
|
| 1308 | { {"rip", "eip", "", ""}, reg_off(rip, eip) },
|
| 1309 | { {"rax", "eax", "ax", "al"}, reg_off(rax, eax) },
|
| 1310 | { {"rbx", "ebx", "bx", "bl"}, reg_off(rbx, ebx) },
|
| 1311 | { {"rcx", "ecx", "cx", "cl"}, reg_off(rcx, ecx) },
|
| 1312 | { {"rdx", "edx", "dx", "dl"}, reg_off(rdx, edx) },
|
| 1313 | { {"rsi", "esi", "si", "sil"}, reg_off(rsi, esi) },
|
| 1314 | { {"rdi", "edi", "di", "dil"}, reg_off(rdi, edi) },
|
| 1315 | { {"rbp", "ebp", "bp", "bpl"}, reg_off(rbp, ebp) },
|
| 1316 | { {"rsp", "esp", "sp", "spl"}, reg_off(rsp, esp) },
|
| 1317 | #undef reg_off
|
| 1318 | #ifdef __x86_64__
|
| 1319 | { {"r8", "r8d", "r8w", "r8b"}, offsetof(struct pt_regs, r8) },
|
| 1320 | { {"r9", "r9d", "r9w", "r9b"}, offsetof(struct pt_regs, r9) },
|
| 1321 | { {"r10", "r10d", "r10w", "r10b"}, offsetof(struct pt_regs, r10) },
|
| 1322 | { {"r11", "r11d", "r11w", "r11b"}, offsetof(struct pt_regs, r11) },
|
| 1323 | { {"r12", "r12d", "r12w", "r12b"}, offsetof(struct pt_regs, r12) },
|
| 1324 | { {"r13", "r13d", "r13w", "r13b"}, offsetof(struct pt_regs, r13) },
|
| 1325 | { {"r14", "r14d", "r14w", "r14b"}, offsetof(struct pt_regs, r14) },
|
| 1326 | { {"r15", "r15d", "r15w", "r15b"}, offsetof(struct pt_regs, r15) },
|
| 1327 | #endif
|
| 1328 | };
|
| 1329 | int i, j;
|
| 1330 |
|
| 1331 | for (i = 0; i < ARRAY_SIZE(reg_map); i++) {
|
| 1332 | for (j = 0; j < ARRAY_SIZE(reg_map[i].names); j++) {
|
| 1333 | if (strcmp(reg_name, reg_map[i].names[j]) == 0)
|
| 1334 | return reg_map[i].pt_regs_off;
|
| 1335 | }
|
| 1336 | }
|
| 1337 |
|
| 1338 | pr_warn("usdt: unrecognized register '%s'\n", reg_name);
|
| 1339 | return -ENOENT;
|
| 1340 | }
|
| 1341 |
|
| 1342 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)
|
| 1343 | {
|
| 1344 | char reg_name[16] = {0}, idx_reg_name[16] = {0};
|
| 1345 | int len, reg_off, idx_reg_off, scale = 1;
|
| 1346 | long off = 0;
|
| 1347 |
|
| 1348 | if (sscanf(arg_str, " %d @ %ld ( %%%15[^,] , %%%15[^,] , %d ) %n",
|
| 1349 | arg_sz, &off, reg_name, idx_reg_name, &scale, &len) == 5 ||
|
| 1350 | sscanf(arg_str, " %d @ ( %%%15[^,] , %%%15[^,] , %d ) %n",
|
| 1351 | arg_sz, reg_name, idx_reg_name, &scale, &len) == 4 ||
|
| 1352 | sscanf(arg_str, " %d @ %ld ( %%%15[^,] , %%%15[^)] ) %n",
|
| 1353 | arg_sz, &off, reg_name, idx_reg_name, &len) == 4 ||
|
| 1354 | sscanf(arg_str, " %d @ ( %%%15[^,] , %%%15[^)] ) %n",
|
| 1355 | arg_sz, reg_name, idx_reg_name, &len) == 3
|
| 1356 | ) {
|
| 1357 | |
| 1358 | * Scale Index Base case:
|
| 1359 | * 1@-96(%rbp,%rax,8)
|
| 1360 | * 1@(%rbp,%rax,8)
|
| 1361 | * 1@-96(%rbp,%rax)
|
| 1362 | * 1@(%rbp,%rax)
|
| 1363 |
|
| 1364 | arg->arg_type = USDT_ARG_SIB;
|
| 1365 | arg->val_off = off;
|
| 1366 |
|
| 1367 | reg_off = calc_pt_regs_off(reg_name);
|
| 1368 | if (reg_off < 0)
|
| 1369 | return reg_off;
|
| 1370 | arg->reg_off = reg_off;
|
| 1371 |
|
| 1372 | idx_reg_off = calc_pt_regs_off(idx_reg_name);
|
| 1373 | if (idx_reg_off < 0)
|
| 1374 | return idx_reg_off;
|
| 1375 | arg->idx_reg_off = idx_reg_off;
|
| 1376 |
|
| 1377 |
|
| 1378 | switch (scale) {
|
| 1379 | case 1: arg->scale_bitshift = 0; break;
|
| 1380 | case 2: arg->scale_bitshift = 1; break;
|
| 1381 | case 4: arg->scale_bitshift = 2; break;
|
| 1382 | case 8: arg->scale_bitshift = 3; break;
|
| 1383 | default:
|
| 1384 | pr_warn("usdt: invalid SIB scale %d, expected 1, 2, 4, 8\n", scale);
|
| 1385 | return -EINVAL;
|
| 1386 | }
|
| 1387 | } else if (sscanf(arg_str, " %d @ %ld ( %%%15[^)] ) %n",
|
| 1388 | arg_sz, &off, reg_name, &len) == 3) {
|
| 1389 |
|
| 1390 | arg->arg_type = USDT_ARG_REG_DEREF;
|
| 1391 | arg->val_off = off;
|
| 1392 | reg_off = calc_pt_regs_off(reg_name);
|
| 1393 | if (reg_off < 0)
|
| 1394 | return reg_off;
|
| 1395 | arg->reg_off = reg_off;
|
| 1396 | } else if (sscanf(arg_str, " %d @ ( %%%15[^)] ) %n", arg_sz, reg_name, &len) == 2) {
|
| 1397 |
|
| 1398 | arg->arg_type = USDT_ARG_REG_DEREF;
|
| 1399 | arg->val_off = 0;
|
| 1400 | reg_off = calc_pt_regs_off(reg_name);
|
| 1401 | if (reg_off < 0)
|
| 1402 | return reg_off;
|
| 1403 | arg->reg_off = reg_off;
|
| 1404 | } else if (sscanf(arg_str, " %d @ %%%15s %n", arg_sz, reg_name, &len) == 2) {
|
| 1405 |
|
| 1406 | arg->arg_type = USDT_ARG_REG;
|
| 1407 |
|
| 1408 | arg->val_off = 0;
|
| 1409 |
|
| 1410 | reg_off = calc_pt_regs_off(reg_name);
|
| 1411 | if (reg_off < 0)
|
| 1412 | return reg_off;
|
| 1413 | arg->reg_off = reg_off;
|
| 1414 | } else if (sscanf(arg_str, " %d @ $%ld %n", arg_sz, &off, &len) == 2) {
|
| 1415 |
|
| 1416 | arg->arg_type = USDT_ARG_CONST;
|
| 1417 | arg->val_off = off;
|
| 1418 | arg->reg_off = 0;
|
| 1419 | } else {
|
| 1420 | pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);
|
| 1421 | return -EINVAL;
|
| 1422 | }
|
| 1423 |
|
| 1424 | return len;
|
| 1425 | }
|
| 1426 |
|
| 1427 | #elif defined(__s390x__)
|
| 1428 |
|
| 1429 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)
|
| 1430 | {
|
| 1431 | unsigned int reg;
|
| 1432 | int len;
|
| 1433 | long off;
|
| 1434 |
|
| 1435 | if (sscanf(arg_str, " %d @ %ld ( %%r%u ) %n", arg_sz, &off, ®, &len) == 3) {
|
| 1436 |
|
| 1437 | arg->arg_type = USDT_ARG_REG_DEREF;
|
| 1438 | arg->val_off = off;
|
| 1439 | if (reg > 15) {
|
| 1440 | pr_warn("usdt: unrecognized register '%%r%u'\n", reg);
|
| 1441 | return -EINVAL;
|
| 1442 | }
|
| 1443 | arg->reg_off = offsetof(user_pt_regs, gprs[reg]);
|
| 1444 | } else if (sscanf(arg_str, " %d @ %%r%u %n", arg_sz, ®, &len) == 2) {
|
| 1445 |
|
| 1446 | arg->arg_type = USDT_ARG_REG;
|
| 1447 | arg->val_off = 0;
|
| 1448 | if (reg > 15) {
|
| 1449 | pr_warn("usdt: unrecognized register '%%r%u'\n", reg);
|
| 1450 | return -EINVAL;
|
| 1451 | }
|
| 1452 | arg->reg_off = offsetof(user_pt_regs, gprs[reg]);
|
| 1453 | } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) {
|
| 1454 |
|
| 1455 | arg->arg_type = USDT_ARG_CONST;
|
| 1456 | arg->val_off = off;
|
| 1457 | arg->reg_off = 0;
|
| 1458 | } else {
|
| 1459 | pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);
|
| 1460 | return -EINVAL;
|
| 1461 | }
|
| 1462 |
|
| 1463 | return len;
|
| 1464 | }
|
| 1465 |
|
| 1466 | #elif defined(__aarch64__)
|
| 1467 |
|
| 1468 | static int calc_pt_regs_off(const char *reg_name)
|
| 1469 | {
|
| 1470 | int reg_num;
|
| 1471 |
|
| 1472 | if (sscanf(reg_name, "x%d", ®_num) == 1) {
|
| 1473 | if (reg_num >= 0 && reg_num < 31)
|
| 1474 | return offsetof(struct user_pt_regs, regs[reg_num]);
|
| 1475 | } else if (strcmp(reg_name, "sp") == 0) {
|
| 1476 | return offsetof(struct user_pt_regs, sp);
|
| 1477 | }
|
| 1478 | pr_warn("usdt: unrecognized register '%s'\n", reg_name);
|
| 1479 | return -ENOENT;
|
| 1480 | }
|
| 1481 |
|
| 1482 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)
|
| 1483 | {
|
| 1484 | char reg_name[16];
|
| 1485 | int len, reg_off;
|
| 1486 | long off;
|
| 1487 |
|
| 1488 | if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , %ld ] %n", arg_sz, reg_name, &off, &len) == 3) {
|
| 1489 |
|
| 1490 | arg->arg_type = USDT_ARG_REG_DEREF;
|
| 1491 | arg->val_off = off;
|
| 1492 | reg_off = calc_pt_regs_off(reg_name);
|
| 1493 | if (reg_off < 0)
|
| 1494 | return reg_off;
|
| 1495 | arg->reg_off = reg_off;
|
| 1496 | } else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) {
|
| 1497 |
|
| 1498 | arg->arg_type = USDT_ARG_REG_DEREF;
|
| 1499 | arg->val_off = 0;
|
| 1500 | reg_off = calc_pt_regs_off(reg_name);
|
| 1501 | if (reg_off < 0)
|
| 1502 | return reg_off;
|
| 1503 | arg->reg_off = reg_off;
|
| 1504 | } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) {
|
| 1505 |
|
| 1506 | arg->arg_type = USDT_ARG_CONST;
|
| 1507 | arg->val_off = off;
|
| 1508 | arg->reg_off = 0;
|
| 1509 | } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) {
|
| 1510 |
|
| 1511 | arg->arg_type = USDT_ARG_REG;
|
| 1512 | arg->val_off = 0;
|
| 1513 | reg_off = calc_pt_regs_off(reg_name);
|
| 1514 | if (reg_off < 0)
|
| 1515 | return reg_off;
|
| 1516 | arg->reg_off = reg_off;
|
| 1517 | } else {
|
| 1518 | pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);
|
| 1519 | return -EINVAL;
|
| 1520 | }
|
| 1521 |
|
| 1522 | return len;
|
| 1523 | }
|
| 1524 |
|
| 1525 | #elif defined(__riscv)
|
| 1526 |
|
| 1527 | static int calc_pt_regs_off(const char *reg_name)
|
| 1528 | {
|
| 1529 | static struct {
|
| 1530 | const char *name;
|
| 1531 | size_t pt_regs_off;
|
| 1532 | } reg_map[] = {
|
| 1533 | { "ra", offsetof(struct user_regs_struct, ra) },
|
| 1534 | { "sp", offsetof(struct user_regs_struct, sp) },
|
| 1535 | { "gp", offsetof(struct user_regs_struct, gp) },
|
| 1536 | { "tp", offsetof(struct user_regs_struct, tp) },
|
| 1537 | { "a0", offsetof(struct user_regs_struct, a0) },
|
| 1538 | { "a1", offsetof(struct user_regs_struct, a1) },
|
| 1539 | { "a2", offsetof(struct user_regs_struct, a2) },
|
| 1540 | { "a3", offsetof(struct user_regs_struct, a3) },
|
| 1541 | { "a4", offsetof(struct user_regs_struct, a4) },
|
| 1542 | { "a5", offsetof(struct user_regs_struct, a5) },
|
| 1543 | { "a6", offsetof(struct user_regs_struct, a6) },
|
| 1544 | { "a7", offsetof(struct user_regs_struct, a7) },
|
| 1545 | { "s0", offsetof(struct user_regs_struct, s0) },
|
| 1546 | { "s1", offsetof(struct user_regs_struct, s1) },
|
| 1547 | { "s2", offsetof(struct user_regs_struct, s2) },
|
| 1548 | { "s3", offsetof(struct user_regs_struct, s3) },
|
| 1549 | { "s4", offsetof(struct user_regs_struct, s4) },
|
| 1550 | { "s5", offsetof(struct user_regs_struct, s5) },
|
| 1551 | { "s6", offsetof(struct user_regs_struct, s6) },
|
| 1552 | { "s7", offsetof(struct user_regs_struct, s7) },
|
| 1553 | { "s8", offsetof(struct user_regs_struct, rv_s8) },
|
| 1554 | { "s9", offsetof(struct user_regs_struct, s9) },
|
| 1555 | { "s10", offsetof(struct user_regs_struct, s10) },
|
| 1556 | { "s11", offsetof(struct user_regs_struct, s11) },
|
| 1557 | { "t0", offsetof(struct user_regs_struct, t0) },
|
| 1558 | { "t1", offsetof(struct user_regs_struct, t1) },
|
| 1559 | { "t2", offsetof(struct user_regs_struct, t2) },
|
| 1560 | { "t3", offsetof(struct user_regs_struct, t3) },
|
| 1561 | { "t4", offsetof(struct user_regs_struct, t4) },
|
| 1562 | { "t5", offsetof(struct user_regs_struct, t5) },
|
| 1563 | { "t6", offsetof(struct user_regs_struct, t6) },
|
| 1564 | };
|
| 1565 | int i;
|
| 1566 |
|
| 1567 | for (i = 0; i < ARRAY_SIZE(reg_map); i++) {
|
| 1568 | if (strcmp(reg_name, reg_map[i].name) == 0)
|
| 1569 | return reg_map[i].pt_regs_off;
|
| 1570 | }
|
| 1571 |
|
| 1572 | pr_warn("usdt: unrecognized register '%s'\n", reg_name);
|
| 1573 | return -ENOENT;
|
| 1574 | }
|
| 1575 |
|
| 1576 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)
|
| 1577 | {
|
| 1578 | char reg_name[16];
|
| 1579 | int len, reg_off;
|
| 1580 | long off;
|
| 1581 |
|
| 1582 | if (sscanf(arg_str, " %d @ %ld ( %15[a-z0-9] ) %n", arg_sz, &off, reg_name, &len) == 3) {
|
| 1583 |
|
| 1584 | arg->arg_type = USDT_ARG_REG_DEREF;
|
| 1585 | arg->val_off = off;
|
| 1586 | reg_off = calc_pt_regs_off(reg_name);
|
| 1587 | if (reg_off < 0)
|
| 1588 | return reg_off;
|
| 1589 | arg->reg_off = reg_off;
|
| 1590 | } else if (sscanf(arg_str, " %d @ %ld %n", arg_sz, &off, &len) == 2) {
|
| 1591 |
|
| 1592 | arg->arg_type = USDT_ARG_CONST;
|
| 1593 | arg->val_off = off;
|
| 1594 | arg->reg_off = 0;
|
| 1595 | } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) {
|
| 1596 |
|
| 1597 | arg->arg_type = USDT_ARG_REG;
|
| 1598 | arg->val_off = 0;
|
| 1599 | reg_off = calc_pt_regs_off(reg_name);
|
| 1600 | if (reg_off < 0)
|
| 1601 | return reg_off;
|
| 1602 | arg->reg_off = reg_off;
|
| 1603 | } else {
|
| 1604 | pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);
|
| 1605 | return -EINVAL;
|
| 1606 | }
|
| 1607 |
|
| 1608 | return len;
|
| 1609 | }
|
| 1610 |
|
| 1611 | #elif defined(__arm__)
|
| 1612 |
|
| 1613 | static int calc_pt_regs_off(const char *reg_name)
|
| 1614 | {
|
| 1615 | static struct {
|
| 1616 | const char *name;
|
| 1617 | size_t pt_regs_off;
|
| 1618 | } reg_map[] = {
|
| 1619 | { "r0", offsetof(struct pt_regs, uregs[0]) },
|
| 1620 | { "r1", offsetof(struct pt_regs, uregs[1]) },
|
| 1621 | { "r2", offsetof(struct pt_regs, uregs[2]) },
|
| 1622 | { "r3", offsetof(struct pt_regs, uregs[3]) },
|
| 1623 | { "r4", offsetof(struct pt_regs, uregs[4]) },
|
| 1624 | { "r5", offsetof(struct pt_regs, uregs[5]) },
|
| 1625 | { "r6", offsetof(struct pt_regs, uregs[6]) },
|
| 1626 | { "r7", offsetof(struct pt_regs, uregs[7]) },
|
| 1627 | { "r8", offsetof(struct pt_regs, uregs[8]) },
|
| 1628 | { "r9", offsetof(struct pt_regs, uregs[9]) },
|
| 1629 | { "r10", offsetof(struct pt_regs, uregs[10]) },
|
| 1630 | { "fp", offsetof(struct pt_regs, uregs[11]) },
|
| 1631 | { "ip", offsetof(struct pt_regs, uregs[12]) },
|
| 1632 | { "sp", offsetof(struct pt_regs, uregs[13]) },
|
| 1633 | { "lr", offsetof(struct pt_regs, uregs[14]) },
|
| 1634 | { "pc", offsetof(struct pt_regs, uregs[15]) },
|
| 1635 | };
|
| 1636 | int i;
|
| 1637 |
|
| 1638 | for (i = 0; i < ARRAY_SIZE(reg_map); i++) {
|
| 1639 | if (strcmp(reg_name, reg_map[i].name) == 0)
|
| 1640 | return reg_map[i].pt_regs_off;
|
| 1641 | }
|
| 1642 |
|
| 1643 | pr_warn("usdt: unrecognized register '%s'\n", reg_name);
|
| 1644 | return -ENOENT;
|
| 1645 | }
|
| 1646 |
|
| 1647 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)
|
| 1648 | {
|
| 1649 | char reg_name[16];
|
| 1650 | int len, reg_off;
|
| 1651 | long off;
|
| 1652 |
|
| 1653 | if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] , #%ld ] %n",
|
| 1654 | arg_sz, reg_name, &off, &len) == 3) {
|
| 1655 |
|
| 1656 | arg->arg_type = USDT_ARG_REG_DEREF;
|
| 1657 | arg->val_off = off;
|
| 1658 | reg_off = calc_pt_regs_off(reg_name);
|
| 1659 | if (reg_off < 0)
|
| 1660 | return reg_off;
|
| 1661 | arg->reg_off = reg_off;
|
| 1662 | } else if (sscanf(arg_str, " %d @ \[ %15[a-z0-9] ] %n", arg_sz, reg_name, &len) == 2) {
|
| 1663 |
|
| 1664 | arg->arg_type = USDT_ARG_REG_DEREF;
|
| 1665 | arg->val_off = 0;
|
| 1666 | reg_off = calc_pt_regs_off(reg_name);
|
| 1667 | if (reg_off < 0)
|
| 1668 | return reg_off;
|
| 1669 | arg->reg_off = reg_off;
|
| 1670 | } else if (sscanf(arg_str, " %d @ #%ld %n", arg_sz, &off, &len) == 2) {
|
| 1671 |
|
| 1672 | arg->arg_type = USDT_ARG_CONST;
|
| 1673 | arg->val_off = off;
|
| 1674 | arg->reg_off = 0;
|
| 1675 | } else if (sscanf(arg_str, " %d @ %15[a-z0-9] %n", arg_sz, reg_name, &len) == 2) {
|
| 1676 |
|
| 1677 | arg->arg_type = USDT_ARG_REG;
|
| 1678 | arg->val_off = 0;
|
| 1679 | reg_off = calc_pt_regs_off(reg_name);
|
| 1680 | if (reg_off < 0)
|
| 1681 | return reg_off;
|
| 1682 | arg->reg_off = reg_off;
|
| 1683 | } else {
|
| 1684 | pr_warn("usdt: unrecognized arg #%d spec '%s'\n", arg_num, arg_str);
|
| 1685 | return -EINVAL;
|
| 1686 | }
|
| 1687 |
|
| 1688 | return len;
|
| 1689 | }
|
| 1690 |
|
| 1691 | #else
|
| 1692 |
|
| 1693 | static int parse_usdt_arg(const char *arg_str, int arg_num, struct usdt_arg_spec *arg, int *arg_sz)
|
| 1694 | {
|
| 1695 | pr_warn("usdt: libbpf doesn't support USDTs on current architecture\n");
|
| 1696 | return -ENOTSUP;
|
| 1697 | }
|
| 1698 |
|
| 1699 | #endif
|