diff --git a/fs/verity/measure.c b/fs/verity/measure.c index cfe2d5e..68dfccb 100644 --- a/fs/verity/measure.c +++ b/fs/verity/measure.c @@ -122,11 +122,11 @@ __bpf_kfunc int bpf_get_fsverity_digest(struct file *file, const struct bpf_dynp { const struct bpf_dynptr_kern *digest_ptr = (struct bpf_dynptr_kern *)digest_p; const struct inode *inode = file_inode(file); - u32 dynptr_sz = __bpf_dynptr_size(digest_ptr); + u64 dynptr_sz = __bpf_dynptr_size(digest_ptr); struct fsverity_digest *arg; const struct fsverity_info *vi; const struct fsverity_hash_alg *hash_alg; - int out_digest_sz; + u64 out_digest_sz; if (dynptr_sz < sizeof(struct fsverity_digest)) return -EINVAL; @@ -144,17 +144,20 @@ __bpf_kfunc int bpf_get_fsverity_digest(struct file *file, const struct bpf_dynp hash_alg = vi->tree_params.hash_alg; + out_digest_sz = dynptr_sz - sizeof(struct fsverity_digest); + if (out_digest_sz < hash_alg->digest_size) + return -EOVERFLOW; + arg->digest_algorithm = hash_alg - fsverity_hash_algs; arg->digest_size = hash_alg->digest_size; - out_digest_sz = dynptr_sz - sizeof(struct fsverity_digest); - /* copy digest */ - memcpy(arg->digest, vi->file_digest, min_t(int, hash_alg->digest_size, out_digest_sz)); + memcpy(arg->digest, vi->file_digest, hash_alg->digest_size); /* fill the extra buffer with zeros */ if (out_digest_sz > hash_alg->digest_size) - memset(arg->digest + arg->digest_size, 0, out_digest_sz - hash_alg->digest_size); + memset(arg->digest + hash_alg->digest_size, 0, + out_digest_sz - hash_alg->digest_size); return 0; } diff --git a/kernel/bpf/rqspinlock.c b/kernel/bpf/rqspinlock.c index e4e338c..2129def 100644 --- a/kernel/bpf/rqspinlock.c +++ b/kernel/bpf/rqspinlock.c @@ -572,9 +572,10 @@ queue: /* Disable queue destruction when we detect deadlocks. */ if (ret == -EDEADLK) { - if (!next) + if (!try_cmpxchg_tail(lock, tail, 0)) { next = smp_cond_load_relaxed(&node->next, (VAL)); - arch_mcs_spin_unlock_contended(&next->locked); + arch_mcs_spin_unlock_contended(&next->locked); + } goto err_release_node; } diff --git a/kernel/bpf/verifier.c b/kernel/bpf/verifier.c index 7aa4734..fdc5fbb 100644 --- a/kernel/bpf/verifier.c +++ b/kernel/bpf/verifier.c @@ -13557,23 +13557,21 @@ static void sanitize_mark_insn_seen(struct bpf_verifier_env *env) env->insn_aux_data[env->insn_idx].seen = env->pass_cnt; } -static int sanitize_err(struct bpf_verifier_env *env, - const struct bpf_insn *insn, int reason, - const struct bpf_reg_state *off_reg, - const struct bpf_reg_state *dst_reg) +static int sanitize_err(struct bpf_verifier_env *env, const struct bpf_insn *insn, int reason) { static const char *err = "pointer arithmetic with it prohibited for !root"; const char *op = BPF_OP(insn->code) == BPF_ADD ? "add" : "sub"; u32 dst = insn->dst_reg, src = insn->src_reg; + struct bpf_reg_state *regs = cur_regs(env); switch (reason) { case REASON_BOUNDS: verbose(env, "R%d has unknown scalar with mixed signed bounds, %s\n", - off_reg == dst_reg ? dst : src, err); + regs[src].type == SCALAR_VALUE ? src : dst, err); break; case REASON_TYPE: verbose(env, "R%d has pointer with unsupported alu operation, %s\n", - off_reg == dst_reg ? src : dst, err); + regs[src].type == SCALAR_VALUE ? dst : src, err); break; case REASON_PATHS: verbose(env, "R%d tried to %s from different maps, paths or scalars, %s\n", @@ -13709,13 +13707,6 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EACCES; } - /* - * Accesses to untrusted PTR_TO_MEM are done through probe - * instructions, hence no need to track offsets. - */ - if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) - return 0; - switch (base_type(ptr_reg->type)) { case PTR_TO_CTX: case PTR_TO_MAP_VALUE: @@ -13745,11 +13736,19 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EACCES; } - /* In case of 'scalar += pointer', dst_reg inherits pointer type and id. - * The id may be overwritten later if we create a new variable offset. + /* For 'scalar += pointer', dst_reg inherits the complete pointer + * register state. Individual fields may be adjusted later by pointer + * arithmetic. Callers guarantee that below does not overwrite off_reg. */ - dst_reg->type = ptr_reg->type; - dst_reg->id = ptr_reg->id; + if (dst_reg != ptr_reg) + *dst_reg = *ptr_reg; + + /* + * Accesses to untrusted PTR_TO_MEM are done through probe + * instructions, hence no need to track offsets. + */ + if (base_type(ptr_reg->type) == PTR_TO_MEM && (ptr_reg->type & PTR_UNTRUSTED)) + return 0; if (!check_reg_sane_offset_scalar(env, off_reg, ptr_reg->type) || !check_reg_sane_offset_ptr(env, ptr_reg, ptr_reg->type)) @@ -13762,7 +13761,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, ret = sanitize_ptr_alu(env, insn, ptr_reg, off_reg, dst_reg, &info, false); if (ret < 0) - return sanitize_err(env, insn, ret, off_reg, dst_reg); + return sanitize_err(env, insn, ret); } switch (opcode) { @@ -13792,7 +13791,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, } break; case BPF_SUB: - if (dst_reg == off_reg) { + if (dst_reg != ptr_reg) { /* scalar -= pointer. Creates an unknown scalar */ verbose(env, "R%d tried to subtract pointer from scalar\n", dst); @@ -13855,7 +13854,7 @@ static int adjust_ptr_min_max_vals(struct bpf_verifier_env *env, return -EFAULT; } if (ret < 0) - return sanitize_err(env, insn, ret, off_reg, dst_reg); + return sanitize_err(env, insn, ret); } return 0; @@ -14607,7 +14606,7 @@ static int adjust_scalar_min_max_vals(struct bpf_verifier_env *env, if (sanitize_needed(opcode)) { ret = sanitize_val_alu(env, insn); if (ret < 0) - return sanitize_err(env, insn, ret, NULL, NULL); + return sanitize_err(env, insn, ret); } /* Calculate sign/unsigned bounds and tnum for alu32 and alu64 bit ops. @@ -14810,8 +14809,8 @@ static int adjust_reg_min_max_vals(struct bpf_verifier_env *env, err = mark_chain_precision(env, insn->dst_reg); if (err) return err; - return adjust_ptr_min_max_vals(env, insn, - src_reg, dst_reg); + off_reg = *dst_reg; + return adjust_ptr_min_max_vals(env, insn, src_reg, &off_reg); } } else if (ptr_reg) { /* pointer += scalar */ diff --git a/net/core/filter.c b/net/core/filter.c index 11bb0d2..1684598 100644 --- a/net/core/filter.c +++ b/net/core/filter.c @@ -7684,7 +7684,7 @@ BPF_CALL_5(bpf_tcp_check_syncookie, struct sock *, sk, void *, iph, u32, iph_len return -EINVAL; /* sk_listener() allows TCP_NEW_SYN_RECV, which makes no sense here. */ - if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN) + if (sk->sk_state != TCP_LISTEN || sk->sk_protocol != IPPROTO_TCP) return -EINVAL; if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies)) @@ -7757,7 +7757,7 @@ BPF_CALL_5(bpf_tcp_gen_syncookie, struct sock *, sk, void *, iph, u32, iph_len, if (unlikely(!sk || th_len < sizeof(*th) || th_len != th->doff * 4)) return -EINVAL; - if (sk->sk_protocol != IPPROTO_TCP || sk->sk_state != TCP_LISTEN) + if (sk->sk_state != TCP_LISTEN || sk->sk_protocol != IPPROTO_TCP) return -EINVAL; if (!READ_ONCE(sock_net(sk)->ipv4.sysctl_tcp_syncookies)) diff --git a/net/ipv4/tcp_bpf.c b/net/ipv4/tcp_bpf.c index a30475a..2e234d1 100644 --- a/net/ipv4/tcp_bpf.c +++ b/net/ipv4/tcp_bpf.c @@ -469,6 +469,7 @@ more_data: case __SK_REDIRECT: redir_ingress = psock->redir_ingress; sk_redir = psock->sk_redir; + sock_hold(sk_redir); sk_msg_apply_bytes(psock, tosend); if (!psock->apply_bytes) { /* Clean up before releasing the sock lock. */ @@ -489,6 +490,7 @@ more_data: if (eval == __SK_REDIRECT) sock_put(sk_redir); + sock_put(sk_redir); lock_sock(sk); sk_mem_uncharge(sk, sent); diff --git a/net/ipv4/tcp_ipv4.c b/net/ipv4/tcp_ipv4.c index b8887cd..7f413f5 100644 --- a/net/ipv4/tcp_ipv4.c +++ b/net/ipv4/tcp_ipv4.c @@ -3078,24 +3078,24 @@ static unsigned int bpf_iter_tcp_established_batch(struct seq_file *seq, { struct bpf_tcp_iter_state *iter = seq->private; struct hlist_nulls_node *node; - unsigned int expected = 1; - struct sock *sk; - - sock_hold(*start_sk); - iter->batch[iter->end_sk++].sk = *start_sk; + struct sock *sk = *start_sk; + unsigned int expected = 0; - sk = sk_nulls_next(*start_sk); *start_sk = NULL; sk_nulls_for_each_from(sk, node) { - if (seq_sk_match(seq, sk)) { - if (iter->end_sk < iter->max_sk) { - sock_hold(sk); - iter->batch[iter->end_sk++].sk = sk; - } else if (!*start_sk) { - /* Remember where we left off. */ - *start_sk = sk; - } - expected++; + if (!seq_sk_match(seq, sk)) + continue; + expected++; + if (iter->end_sk < iter->max_sk) { + /* reqsk_queue_hash_req() inserts with sk_refcnt == 0 + * and refcount_set()s it after the bucket lock drops. + */ + if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt))) + continue; + iter->batch[iter->end_sk++].sk = sk; + } else if (!*start_sk) { + /* Remember where we left off. */ + *start_sk = sk; } } @@ -3133,12 +3133,13 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) struct sock *sk; int err; +again: sk = bpf_iter_tcp_resume(seq); if (!sk) return NULL; /* Done */ expected = bpf_iter_fill_batch(seq, &sk); - if (likely(iter->end_sk == expected)) + if (likely(!sk)) goto done; /* Batch size was too small. */ @@ -3157,7 +3158,7 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) return NULL; /* Done */ expected = bpf_iter_fill_batch(seq, &sk); - if (likely(iter->end_sk == expected)) + if (likely(!sk)) goto done; /* Batch size was still too small. Hold onto the lock while we try @@ -3170,10 +3171,14 @@ static struct sock *bpf_iter_tcp_batch(struct seq_file *seq) return ERR_PTR(err); } - expected = bpf_iter_fill_batch(seq, &sk); - WARN_ON_ONCE(iter->end_sk != expected); + bpf_iter_fill_batch(seq, &sk); + WARN_ON_ONCE(sk); done: bpf_iter_tcp_unlock_bucket(seq); + if (unlikely(!iter->end_sk)) { + ++iter->state.bucket; + goto again; + } return iter->batch[0].sk; } diff --git a/net/netfilter/nf_conntrack_bpf.c b/net/netfilter/nf_conntrack_bpf.c index f98d1d4..c2df7c9 100644 --- a/net/netfilter/nf_conntrack_bpf.c +++ b/net/netfilter/nf_conntrack_bpf.c @@ -122,42 +122,54 @@ __bpf_nf_ct_alloc_entry(struct net *net, struct bpf_sock_tuple *bpf_tuple, struct nf_conntrack_tuple otuple, rtuple; struct nf_conntrack_zone ct_zone; struct nf_conn *ct; + u8 ct_zone_dir = 0; + u16 ct_zone_id; + s32 netns_id; + u8 l4proto; int err; if (!(opts_len == NF_BPF_CT_OPTS_SZ || opts_len == 12)) return ERR_PTR(-EINVAL); + + netns_id = READ_ONCE(opts->netns_id); + l4proto = READ_ONCE(opts->l4proto); + ct_zone_id = READ_ONCE(opts->ct_zone_id); if (opts_len == NF_BPF_CT_OPTS_SZ) { - if (opts->reserved[0] || opts->reserved[1] || opts->reserved[2]) + ct_zone_dir = READ_ONCE(opts->ct_zone_dir); + if (READ_ONCE(opts->reserved[0]) || + READ_ONCE(opts->reserved[1]) || + READ_ONCE(opts->reserved[2])) return ERR_PTR(-EINVAL); } else { - if (opts->ct_zone_id) + if (ct_zone_id) return ERR_PTR(-EINVAL); } - if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS)) + if (unlikely(netns_id < BPF_F_CURRENT_NETNS)) return ERR_PTR(-EINVAL); - err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto, + err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, l4proto, IP_CT_DIR_ORIGINAL, &otuple); if (err < 0) return ERR_PTR(err); - err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto, + err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, l4proto, IP_CT_DIR_REPLY, &rtuple); if (err < 0) return ERR_PTR(err); - if (opts->netns_id >= 0) { - net = get_net_ns_by_id(net, opts->netns_id); + if (netns_id >= 0) { + net = get_net_ns_by_id(net, netns_id); if (unlikely(!net)) return ERR_PTR(-ENONET); } if (opts_len == NF_BPF_CT_OPTS_SZ) { - if (opts->ct_zone_dir == 0) - opts->ct_zone_dir = NF_CT_DEFAULT_ZONE_DIR; - nf_ct_zone_init(&ct_zone, - opts->ct_zone_id, opts->ct_zone_dir, 0); + if (ct_zone_dir == 0) { + ct_zone_dir = NF_CT_DEFAULT_ZONE_DIR; + opts->ct_zone_dir = ct_zone_dir; + } + nf_ct_zone_init(&ct_zone, ct_zone_id, ct_zone_dir, 0); } else { ct_zone = nf_ct_zone_dflt; } @@ -171,7 +183,7 @@ __bpf_nf_ct_alloc_entry(struct net *net, struct bpf_sock_tuple *bpf_tuple, __nf_ct_set_timeout(ct, timeout * HZ); out: - if (opts->netns_id >= 0) + if (netns_id >= 0) put_net(net); return ct; @@ -186,46 +198,58 @@ static struct nf_conn *__bpf_nf_ct_lookup(struct net *net, struct nf_conntrack_tuple tuple; struct nf_conntrack_zone ct_zone; struct nf_conn *ct; + u8 ct_zone_dir = 0; + u16 ct_zone_id; + s32 netns_id; + u8 l4proto; int err; if (!opts || !bpf_tuple) return ERR_PTR(-EINVAL); if (!(opts_len == NF_BPF_CT_OPTS_SZ || opts_len == 12)) return ERR_PTR(-EINVAL); + + netns_id = READ_ONCE(opts->netns_id); + l4proto = READ_ONCE(opts->l4proto); + ct_zone_id = READ_ONCE(opts->ct_zone_id); if (opts_len == NF_BPF_CT_OPTS_SZ) { - if (opts->reserved[0] || opts->reserved[1] || opts->reserved[2]) + ct_zone_dir = READ_ONCE(opts->ct_zone_dir); + if (READ_ONCE(opts->reserved[0]) || + READ_ONCE(opts->reserved[1]) || + READ_ONCE(opts->reserved[2])) return ERR_PTR(-EINVAL); } else { - if (opts->ct_zone_id) + if (ct_zone_id) return ERR_PTR(-EINVAL); } - if (unlikely(opts->l4proto != IPPROTO_TCP && opts->l4proto != IPPROTO_UDP)) + if (unlikely(l4proto != IPPROTO_TCP && l4proto != IPPROTO_UDP)) return ERR_PTR(-EPROTO); - if (unlikely(opts->netns_id < BPF_F_CURRENT_NETNS)) + if (unlikely(netns_id < BPF_F_CURRENT_NETNS)) return ERR_PTR(-EINVAL); - err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, opts->l4proto, + err = bpf_nf_ct_tuple_parse(bpf_tuple, tuple_len, l4proto, IP_CT_DIR_ORIGINAL, &tuple); if (err < 0) return ERR_PTR(err); - if (opts->netns_id >= 0) { - net = get_net_ns_by_id(net, opts->netns_id); + if (netns_id >= 0) { + net = get_net_ns_by_id(net, netns_id); if (unlikely(!net)) return ERR_PTR(-ENONET); } if (opts_len == NF_BPF_CT_OPTS_SZ) { - if (opts->ct_zone_dir == 0) - opts->ct_zone_dir = NF_CT_DEFAULT_ZONE_DIR; - nf_ct_zone_init(&ct_zone, - opts->ct_zone_id, opts->ct_zone_dir, 0); + if (ct_zone_dir == 0) { + ct_zone_dir = NF_CT_DEFAULT_ZONE_DIR; + opts->ct_zone_dir = ct_zone_dir; + } + nf_ct_zone_init(&ct_zone, ct_zone_id, ct_zone_dir, 0); } else { ct_zone = nf_ct_zone_dflt; } hash = nf_conntrack_find_get(net, &ct_zone, &tuple); - if (opts->netns_id >= 0) + if (netns_id >= 0) put_net(net); if (!hash) return ERR_PTR(-ENOENT); diff --git a/tools/testing/selftests/bpf/progs/dynptr_fail.c b/tools/testing/selftests/bpf/progs/dynptr_fail.c index 344fb2a..29c6361 100644 --- a/tools/testing/selftests/bpf/progs/dynptr_fail.c +++ b/tools/testing/selftests/bpf/progs/dynptr_fail.c @@ -1635,6 +1635,36 @@ static int callback(__u32 index, void *data) return 0; } +/* A commuted add should preserve the parent id of a dynptr data slice. */ +SEC("?raw_tp") +__failure __msg("invalid mem access 'scalar'") +int dynptr_slice_commuted_invalidate(void *ctx) +{ + struct bpf_dynptr ptr; + __u32 *slice, *derived; + + bpf_ringbuf_reserve_dynptr(&ringbuf, sizeof(__u32), 0, &ptr); + + slice = bpf_dynptr_data(&ptr, 0, sizeof(__u32)); + if (!slice) + goto done; + + asm volatile ("%[dst] = 0;" + "%[dst] += %[src];" + "%[src] = 0;" + : [dst]"=&r"(derived), [src]"+r"(slice) + : + : "memory"); + + bpf_ringbuf_discard_dynptr(&ptr, 0); + val = *derived; + return 0; + +done: + bpf_ringbuf_discard_dynptr(&ptr, 0); + return 0; +} + /* If the dynptr is written into in a callback function, its data * slices should be invalidated as well. */ diff --git a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c index 5b44537..f166fff 100644 --- a/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c +++ b/tools/testing/selftests/bpf/progs/mem_rdonly_untrusted.c @@ -226,4 +226,21 @@ int null_check(void *ctx) return 0; } +SEC("socket") +__success +__retval(1) +int ldx_is_ok_commuted_addr(void *ctx) +{ + int v, *p, *derived; + + v = 1; + p = bpf_rdonly_cast(&v, 0); + asm volatile ("%[dst] = 0;" + "%[dst] += %[src];" + : [dst]"=&r"(derived) + : [src]"r"(p) + : "memory"); + return *derived; +} + char _license[] SEC("license") = "GPL"; diff --git a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c index fb62e09..d3df7a9 100644 --- a/tools/testing/selftests/bpf/progs/verifier_basic_stack.c +++ b/tools/testing/selftests/bpf/progs/verifier_basic_stack.c @@ -97,4 +97,45 @@ __naked void misaligned_read_from_stack(void) " ::: __clobber_all); } +SEC("socket") +__description("stack pointer arithmetic preserves frame number") +__failure __msg("R7 invalid mem access 'scalar'") +__naked void stack_ptr_arith_preserves_frameno(void) +{ + asm volatile ("\ + r3 = 0; \ + *(u64 *)(r10 - 8) = r3; \ + r1 = %[map_hash_8b] ll; \ + r2 = r10; \ + r2 += -8; \ + call %[bpf_map_lookup_elem]; \ + if r0 != 0 goto +2; \ + r0 = 0; \ + exit; \ + r1 = r0; \ + r2 = 0; \ + r3 = 0; \ + call stack_ptr_arith_preserves_frameno_subprog;\ + r0 = 0; \ + exit; \ + ": + : __imm(bpf_map_lookup_elem), + __imm_addr(map_hash_8b) + : __clobber_all); +} + +static __used __naked void stack_ptr_arith_preserves_frameno_subprog(void) +{ + asm volatile ("\ + *(u64 *)(r10 - 8) = r1; \ + r6 = -8; \ + r6 += r10; \ + *(u64 *)(r6 + 0) = r2; \ + r7 = *(u64 *)(r10 - 8); \ + *(u64 *)(r7 + 0) = r3; \ + r0 = 0; \ + exit; \ + "::: __clobber_all); +} + char _license[] SEC("license") = "GPL";