projects/linux·blob master · refs

256 lines · 6.3 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (c) 2020 Facebook
4 * Copyright 2020 Google LLC.
5 */
6
7#include <linux/pid.h>
8#include <linux/sched.h>
9#include <linux/rculist.h>
10#include <linux/list.h>
11#include <linux/hash.h>
12#include <linux/types.h>
13#include <linux/spinlock.h>
14#include <linux/bpf.h>
15#include <linux/bpf_local_storage.h>
16#include <linux/filter.h>
17#include <uapi/linux/btf.h>
18#include <linux/btf_ids.h>
19#include <linux/rcupdate_trace.h>
20
21DEFINE_BPF_STORAGE_CACHE(task_cache);
22
23static struct bpf_local_storage __rcu **task_storage_ptr(void *owner)
24{
25 struct task_struct *task = owner;
26
27 return &task->bpf_storage;
28}
29
30static struct bpf_local_storage_data *
31task_storage_lookup(struct task_struct *task, struct bpf_map *map,
32 bool cacheit_lockit)
33{
34 struct bpf_local_storage *task_storage;
35 struct bpf_local_storage_map *smap;
36
37 task_storage =
38 rcu_dereference_check(task->bpf_storage, bpf_rcu_lock_held());
39 if (!task_storage)
40 return NULL;
41
42 smap = (struct bpf_local_storage_map *)map;
43 return bpf_local_storage_lookup(task_storage, smap, cacheit_lockit);
44}
45
46void bpf_task_storage_free(struct task_struct *task)
47{
48 struct bpf_local_storage *local_storage;
49
50 rcu_read_lock();
51
52 local_storage = rcu_dereference(task->bpf_storage);
53 if (!local_storage)
54 goto out;
55
56 bpf_local_storage_destroy(local_storage);
57out:
58 rcu_read_unlock();
59}
60
61static void *bpf_pid_task_storage_lookup_elem(struct bpf_map *map, void *key)
62{
63 struct bpf_local_storage_data *sdata;
64 struct task_struct *task;
65 unsigned int f_flags;
66 struct pid *pid;
67 int fd, err;
68
69 fd = *(int *)key;
70 pid = pidfd_get_pid(fd, &f_flags);
71 if (IS_ERR(pid))
72 return ERR_CAST(pid);
73
74 /* We should be in an RCU read side critical section, it should be safe
75 * to call pid_task.
76 */
77 WARN_ON_ONCE(!rcu_read_lock_held());
78 task = pid_task(pid, PIDTYPE_PID);
79 if (!task) {
80 err = -ENOENT;
81 goto out;
82 }
83
84 sdata = task_storage_lookup(task, map, true);
85 put_pid(pid);
86 return sdata ? sdata->data : NULL;
87out:
88 put_pid(pid);
89 return ERR_PTR(err);
90}
91
92static long bpf_pid_task_storage_update_elem(struct bpf_map *map, void *key,
93 void *value, u64 map_flags)
94{
95 struct bpf_local_storage_data *sdata;
96 struct task_struct *task;
97 unsigned int f_flags;
98 struct pid *pid;
99 int fd, err;
100
101 if ((map_flags & BPF_F_LOCK) && btf_record_has_field(map->record, BPF_UPTR))
102 return -EOPNOTSUPP;
103
104 fd = *(int *)key;
105 pid = pidfd_get_pid(fd, &f_flags);
106 if (IS_ERR(pid))
107 return PTR_ERR(pid);
108
109 /* We should be in an RCU read side critical section, it should be safe
110 * to call pid_task.
111 */
112 WARN_ON_ONCE(!rcu_read_lock_held());
113 task = pid_task(pid, PIDTYPE_PID);
114 if (!task) {
115 err = -ENOENT;
116 goto out;
117 }
118
119 sdata = bpf_local_storage_update(
120 task, (struct bpf_local_storage_map *)map, value, map_flags,
121 true);
122
123 err = PTR_ERR_OR_ZERO(sdata);
124out:
125 put_pid(pid);
126 return err;
127}
128
129static int task_storage_delete(struct task_struct *task, struct bpf_map *map)
130{
131 struct bpf_local_storage_data *sdata;
132
133 sdata = task_storage_lookup(task, map, false);
134 if (!sdata)
135 return -ENOENT;
136
137 return bpf_selem_unlink(SELEM(sdata));
138}
139
140static long bpf_pid_task_storage_delete_elem(struct bpf_map *map, void *key)
141{
142 struct task_struct *task;
143 unsigned int f_flags;
144 struct pid *pid;
145 int fd, err;
146
147 fd = *(int *)key;
148 pid = pidfd_get_pid(fd, &f_flags);
149 if (IS_ERR(pid))
150 return PTR_ERR(pid);
151
152 /* We should be in an RCU read side critical section, it should be safe
153 * to call pid_task.
154 */
155 WARN_ON_ONCE(!rcu_read_lock_held());
156 task = pid_task(pid, PIDTYPE_PID);
157 if (!task) {
158 err = -ENOENT;
159 goto out;
160 }
161
162 err = task_storage_delete(task, map);
163out:
164 put_pid(pid);
165 return err;
166}
167
168BPF_CALL_4(bpf_task_storage_get, struct bpf_map *, map, struct task_struct *,
169 task, void *, value, u64, flags)
170{
171 struct bpf_local_storage_data *sdata;
172
173 WARN_ON_ONCE(!bpf_rcu_lock_held());
174 if (flags & ~BPF_LOCAL_STORAGE_GET_F_CREATE || !task)
175 return (unsigned long)NULL;
176
177 sdata = task_storage_lookup(task, map, true);
178 if (sdata)
179 return (unsigned long)sdata->data;
180
181 /* only allocate new storage, when the task is refcounted */
182 if (refcount_read(&task->usage) &&
183 (flags & BPF_LOCAL_STORAGE_GET_F_CREATE)) {
184 sdata = bpf_local_storage_update(
185 task, (struct bpf_local_storage_map *)map, value,
186 BPF_NOEXIST, false);
187 return IS_ERR(sdata) ? (unsigned long)NULL : (unsigned long)sdata->data;
188 }
189
190 return (unsigned long)NULL;
191}
192
193BPF_CALL_2(bpf_task_storage_delete, struct bpf_map *, map, struct task_struct *,
194 task)
195{
196 WARN_ON_ONCE(!bpf_rcu_lock_held());
197 if (!task)
198 return -EINVAL;
199
200 /* This helper must only be called from places where the lifetime of the task
201 * is guaranteed. Either by being refcounted or by being protected
202 * by an RCU read-side critical section.
203 */
204 return task_storage_delete(task, map);
205}
206
207static int notsupp_get_next_key(struct bpf_map *map, void *key, void *next_key)
208{
209 return -ENOTSUPP;
210}
211
212static struct bpf_map *task_storage_map_alloc(union bpf_attr *attr)
213{
214 return bpf_local_storage_map_alloc(attr, &task_cache);
215}
216
217static void task_storage_map_free(struct bpf_map *map)
218{
219 bpf_local_storage_map_free(map, &task_cache);
220}
221
222BTF_ID_LIST_GLOBAL_SINGLE(bpf_local_storage_map_btf_id, struct, bpf_local_storage_map)
223const struct bpf_map_ops task_storage_map_ops = {
224 .map_meta_equal = bpf_map_meta_equal,
225 .map_alloc_check = bpf_local_storage_map_alloc_check,
226 .map_alloc = task_storage_map_alloc,
227 .map_free = task_storage_map_free,
228 .map_get_next_key = notsupp_get_next_key,
229 .map_lookup_elem = bpf_pid_task_storage_lookup_elem,
230 .map_update_elem = bpf_pid_task_storage_update_elem,
231 .map_delete_elem = bpf_pid_task_storage_delete_elem,
232 .map_check_btf = bpf_local_storage_map_check_btf,
233 .map_mem_usage = bpf_local_storage_map_mem_usage,
234 .map_btf_id = &bpf_local_storage_map_btf_id[0],
235 .map_owner_storage_ptr = task_storage_ptr,
236};
237
238const struct bpf_func_proto bpf_task_storage_get_proto = {
239 .func = bpf_task_storage_get,
240 .gpl_only = false,
241 .ret_type = RET_PTR_TO_MAP_VALUE_OR_NULL,
242 .arg1_type = ARG_CONST_MAP_PTR,
243 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
244 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
245 .arg3_type = ARG_PTR_TO_MAP_VALUE_OR_NULL,
246 .arg4_type = ARG_ANYTHING,
247};
248
249const struct bpf_func_proto bpf_task_storage_delete_proto = {
250 .func = bpf_task_storage_delete,
251 .gpl_only = false,
252 .ret_type = RET_INTEGER,
253 .arg1_type = ARG_CONST_MAP_PTR,
254 .arg2_type = ARG_PTR_TO_BTF_ID_OR_NULL,
255 .arg2_btf_id = &btf_tracing_ids[BTF_TRACING_TYPE_TASK],
256};