projects/linux·blob master · refs

504 lines · 12.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2/*
3 * Copyright (C) 2021 VMware Inc, Steven Rostedt <[email protected]>
4 */
5#include <linux/spinlock.h>
6#include <linux/seqlock.h>
7#include <linux/irq_work.h>
8#include <linux/slab.h>
9#include "trace.h"
10
11/* See pid_list.h for details */
12
13static inline union lower_chunk *get_lower_chunk(struct trace_pid_list *pid_list)
14{
15 union lower_chunk *chunk;
16
17 lockdep_assert_held(&pid_list->lock);
18
19 if (!pid_list->lower_list)
20 return NULL;
21
22 chunk = pid_list->lower_list;
23 pid_list->lower_list = chunk->next;
24 pid_list->free_lower_chunks--;
25 WARN_ON_ONCE(pid_list->free_lower_chunks < 0);
26 chunk->next = NULL;
27 /*
28 * If a refill needs to happen, it can not happen here
29 * as the scheduler run queue locks are held.
30 */
31 if (pid_list->free_lower_chunks <= CHUNK_REALLOC)
32 irq_work_queue(&pid_list->refill_irqwork);
33
34 return chunk;
35}
36
37static inline union upper_chunk *get_upper_chunk(struct trace_pid_list *pid_list)
38{
39 union upper_chunk *chunk;
40
41 lockdep_assert_held(&pid_list->lock);
42
43 if (!pid_list->upper_list)
44 return NULL;
45
46 chunk = pid_list->upper_list;
47 pid_list->upper_list = chunk->next;
48 pid_list->free_upper_chunks--;
49 WARN_ON_ONCE(pid_list->free_upper_chunks < 0);
50 chunk->next = NULL;
51 /*
52 * If a refill needs to happen, it can not happen here
53 * as the scheduler run queue locks are held.
54 */
55 if (pid_list->free_upper_chunks <= CHUNK_REALLOC)
56 irq_work_queue(&pid_list->refill_irqwork);
57
58 return chunk;
59}
60
61static inline void put_lower_chunk(struct trace_pid_list *pid_list,
62 union lower_chunk *chunk)
63{
64 lockdep_assert_held(&pid_list->lock);
65
66 chunk->next = pid_list->lower_list;
67 pid_list->lower_list = chunk;
68 pid_list->free_lower_chunks++;
69}
70
71static inline void put_upper_chunk(struct trace_pid_list *pid_list,
72 union upper_chunk *chunk)
73{
74 lockdep_assert_held(&pid_list->lock);
75
76 chunk->next = pid_list->upper_list;
77 pid_list->upper_list = chunk;
78 pid_list->free_upper_chunks++;
79}
80
81static inline bool upper_empty(union upper_chunk *chunk)
82{
83 /*
84 * If chunk->data has no lower chunks, it will be the same
85 * as a zeroed bitmask.
86 */
87 return bitmap_empty((unsigned long *)chunk->data, BITS_PER_TYPE(chunk->data));
88}
89
90static inline int pid_split(unsigned int pid, unsigned int *upper1,
91 unsigned int *upper2, unsigned int *lower)
92{
93 /* MAX_PID should cover all pids */
94 BUILD_BUG_ON(MAX_PID < PID_MAX_LIMIT);
95
96 /* In case a bad pid is passed in, then fail */
97 if (unlikely(pid >= MAX_PID))
98 return -1;
99
100 *upper1 = (pid >> UPPER1_SHIFT) & UPPER_MASK;
101 *upper2 = (pid >> UPPER2_SHIFT) & UPPER_MASK;
102 *lower = pid & LOWER_MASK;
103
104 return 0;
105}
106
107static inline unsigned int pid_join(unsigned int upper1,
108 unsigned int upper2, unsigned int lower)
109{
110 return ((upper1 & UPPER_MASK) << UPPER1_SHIFT) |
111 ((upper2 & UPPER_MASK) << UPPER2_SHIFT) |
112 (lower & LOWER_MASK);
113}
114
115/**
116 * trace_pid_list_is_set - test if the pid is set in the list
117 * @pid_list: The pid list to test
118 * @pid: The pid to see if set in the list.
119 *
120 * Tests if @pid is set in the @pid_list. This is usually called
121 * from the scheduler when a task is scheduled. Its pid is checked
122 * if it should be traced or not.
123 *
124 * Return true if the pid is in the list, false otherwise.
125 */
126bool trace_pid_list_is_set(struct trace_pid_list *pid_list, unsigned int pid)
127{
128 union upper_chunk *upper_chunk;
129 union lower_chunk *lower_chunk;
130 unsigned int seq;
131 unsigned int upper1;
132 unsigned int upper2;
133 unsigned int lower;
134 bool ret = false;
135
136 if (!pid_list)
137 return false;
138
139 if (pid_split(pid, &upper1, &upper2, &lower) < 0)
140 return false;
141
142 do {
143 seq = read_seqcount_begin(&pid_list->seqcount);
144 ret = false;
145 upper_chunk = pid_list->upper[upper1];
146 if (upper_chunk) {
147 lower_chunk = upper_chunk->data[upper2];
148 if (lower_chunk)
149 ret = test_bit(lower, lower_chunk->data);
150 }
151 } while (read_seqcount_retry(&pid_list->seqcount, seq));
152
153 return ret;
154}
155
156/**
157 * trace_pid_list_set - add a pid to the list
158 * @pid_list: The pid list to add the @pid to.
159 * @pid: The pid to add.
160 *
161 * Adds @pid to @pid_list. This is usually done explicitly by a user
162 * adding a task to be traced, or indirectly by the fork function
163 * when children should be traced and a task's pid is in the list.
164 *
165 * Return 0 on success, negative otherwise.
166 */
167int trace_pid_list_set(struct trace_pid_list *pid_list, unsigned int pid)
168{
169 union upper_chunk *upper_chunk;
170 union lower_chunk *lower_chunk;
171 unsigned long flags;
172 unsigned int upper1;
173 unsigned int upper2;
174 unsigned int lower;
175 int ret;
176
177 if (!pid_list)
178 return -ENODEV;
179
180 if (pid_split(pid, &upper1, &upper2, &lower) < 0)
181 return -EINVAL;
182
183 raw_spin_lock_irqsave(&pid_list->lock, flags);
184 write_seqcount_begin(&pid_list->seqcount);
185 upper_chunk = pid_list->upper[upper1];
186 if (!upper_chunk) {
187 upper_chunk = get_upper_chunk(pid_list);
188 if (!upper_chunk) {
189 ret = -ENOMEM;
190 goto out;
191 }
192 pid_list->upper[upper1] = upper_chunk;
193 }
194 lower_chunk = upper_chunk->data[upper2];
195 if (!lower_chunk) {
196 lower_chunk = get_lower_chunk(pid_list);
197 if (!lower_chunk) {
198 ret = -ENOMEM;
199 goto out;
200 }
201 upper_chunk->data[upper2] = lower_chunk;
202 }
203 set_bit(lower, lower_chunk->data);
204 ret = 0;
205 out:
206 write_seqcount_end(&pid_list->seqcount);
207 raw_spin_unlock_irqrestore(&pid_list->lock, flags);
208 return ret;
209}
210
211/**
212 * trace_pid_list_clear - remove a pid from the list
213 * @pid_list: The pid list to remove the @pid from.
214 * @pid: The pid to remove.
215 *
216 * Removes @pid from @pid_list. This is usually done explicitly by a user
217 * removing tasks from tracing, or indirectly by the exit function
218 * when a task that is set to be traced exits.
219 *
220 * Return 0 on success, negative otherwise.
221 */
222int trace_pid_list_clear(struct trace_pid_list *pid_list, unsigned int pid)
223{
224 union upper_chunk *upper_chunk;
225 union lower_chunk *lower_chunk;
226 unsigned long flags;
227 unsigned int upper1;
228 unsigned int upper2;
229 unsigned int lower;
230
231 if (!pid_list)
232 return -ENODEV;
233
234 if (pid_split(pid, &upper1, &upper2, &lower) < 0)
235 return -EINVAL;
236
237 raw_spin_lock_irqsave(&pid_list->lock, flags);
238 write_seqcount_begin(&pid_list->seqcount);
239 upper_chunk = pid_list->upper[upper1];
240 if (!upper_chunk)
241 goto out;
242
243 lower_chunk = upper_chunk->data[upper2];
244 if (!lower_chunk)
245 goto out;
246
247 clear_bit(lower, lower_chunk->data);
248
249 /* if there's no more bits set, add it to the free list */
250 if (find_first_bit(lower_chunk->data, LOWER_MAX) >= LOWER_MAX) {
251 put_lower_chunk(pid_list, lower_chunk);
252 upper_chunk->data[upper2] = NULL;
253 if (upper_empty(upper_chunk)) {
254 put_upper_chunk(pid_list, upper_chunk);
255 pid_list->upper[upper1] = NULL;
256 }
257 }
258 out:
259 write_seqcount_end(&pid_list->seqcount);
260 raw_spin_unlock_irqrestore(&pid_list->lock, flags);
261 return 0;
262}
263
264/**
265 * trace_pid_list_next - return the next pid in the list
266 * @pid_list: The pid list to examine.
267 * @pid: The pid to start from
268 * @next: The pointer to place the pid that is set starting from @pid.
269 *
270 * Looks for the next consecutive pid that is in @pid_list starting
271 * at the pid specified by @pid. If one is set (including @pid), then
272 * that pid is placed into @next.
273 *
274 * Return 0 when a pid is found, -1 if there are no more pids included.
275 */
276int trace_pid_list_next(struct trace_pid_list *pid_list, unsigned int pid,
277 unsigned int *next)
278{
279 union upper_chunk *upper_chunk;
280 union lower_chunk *lower_chunk;
281 unsigned long flags;
282 unsigned int upper1;
283 unsigned int upper2;
284 unsigned int lower;
285
286 if (!pid_list)
287 return -ENODEV;
288
289 if (pid_split(pid, &upper1, &upper2, &lower) < 0)
290 return -EINVAL;
291
292 raw_spin_lock_irqsave(&pid_list->lock, flags);
293 for (; upper1 <= UPPER_MASK; upper1++, upper2 = 0) {
294 upper_chunk = pid_list->upper[upper1];
295
296 if (!upper_chunk)
297 continue;
298
299 for (; upper2 <= UPPER_MASK; upper2++, lower = 0) {
300 lower_chunk = upper_chunk->data[upper2];
301 if (!lower_chunk)
302 continue;
303
304 lower = find_next_bit(lower_chunk->data, LOWER_MAX,
305 lower);
306 if (lower < LOWER_MAX)
307 goto found;
308 }
309 }
310
311 found:
312 raw_spin_unlock_irqrestore(&pid_list->lock, flags);
313 if (upper1 > UPPER_MASK)
314 return -1;
315
316 *next = pid_join(upper1, upper2, lower);
317 return 0;
318}
319
320/**
321 * trace_pid_list_first - return the first pid in the list
322 * @pid_list: The pid list to examine.
323 * @pid: The pointer to place the pid first found pid that is set.
324 *
325 * Looks for the first pid that is set in @pid_list, and places it
326 * into @pid if found.
327 *
328 * Return 0 when a pid is found, -1 if there are no pids set.
329 */
330int trace_pid_list_first(struct trace_pid_list *pid_list, unsigned int *pid)
331{
332 return trace_pid_list_next(pid_list, 0, pid);
333}
334
335static void pid_list_refill_irq(struct irq_work *iwork)
336{
337 struct trace_pid_list *pid_list = container_of(iwork, struct trace_pid_list,
338 refill_irqwork);
339 union upper_chunk *upper = NULL;
340 union lower_chunk *lower = NULL;
341 union upper_chunk **upper_next = &upper;
342 union lower_chunk **lower_next = &lower;
343 int upper_count;
344 int lower_count;
345 int ucnt = 0;
346 int lcnt = 0;
347
348 again:
349 raw_spin_lock(&pid_list->lock);
350 write_seqcount_begin(&pid_list->seqcount);
351 upper_count = CHUNK_ALLOC - pid_list->free_upper_chunks;
352 lower_count = CHUNK_ALLOC - pid_list->free_lower_chunks;
353 write_seqcount_end(&pid_list->seqcount);
354 raw_spin_unlock(&pid_list->lock);
355
356 if (upper_count <= 0 && lower_count <= 0)
357 return;
358
359 while (upper_count-- > 0) {
360 union upper_chunk *chunk;
361
362 chunk = kzalloc_obj(*chunk, GFP_NOWAIT);
363 if (!chunk)
364 break;
365 *upper_next = chunk;
366 upper_next = &chunk->next;
367 ucnt++;
368 }
369
370 while (lower_count-- > 0) {
371 union lower_chunk *chunk;
372
373 chunk = kzalloc_obj(*chunk, GFP_NOWAIT);
374 if (!chunk)
375 break;
376 *lower_next = chunk;
377 lower_next = &chunk->next;
378 lcnt++;
379 }
380
381 raw_spin_lock(&pid_list->lock);
382 write_seqcount_begin(&pid_list->seqcount);
383 if (upper) {
384 *upper_next = pid_list->upper_list;
385 pid_list->upper_list = upper;
386 pid_list->free_upper_chunks += ucnt;
387 }
388 if (lower) {
389 *lower_next = pid_list->lower_list;
390 pid_list->lower_list = lower;
391 pid_list->free_lower_chunks += lcnt;
392 }
393 write_seqcount_end(&pid_list->seqcount);
394 raw_spin_unlock(&pid_list->lock);
395
396 /*
397 * On success of allocating all the chunks, both counters
398 * will be less than zero. If they are not, then an allocation
399 * failed, and we should not try again.
400 */
401 if (upper_count >= 0 || lower_count >= 0)
402 return;
403 /*
404 * When the locks were released, free chunks could have
405 * been used and allocation needs to be done again. Might as
406 * well allocate it now.
407 */
408 goto again;
409}
410
411/**
412 * trace_pid_list_alloc - create a new pid_list
413 *
414 * Allocates a new pid_list to store pids into.
415 *
416 * Returns the pid_list on success, NULL otherwise.
417 */
418struct trace_pid_list *trace_pid_list_alloc(void)
419{
420 struct trace_pid_list *pid_list;
421 int i;
422
423 /* According to linux/thread.h, pids can be no bigger that 30 bits */
424 WARN_ON_ONCE(init_pid_ns.pid_max > (1 << 30));
425
426 pid_list = kzalloc_obj(*pid_list);
427 if (!pid_list)
428 return NULL;
429
430 init_irq_work(&pid_list->refill_irqwork, pid_list_refill_irq);
431
432 raw_spin_lock_init(&pid_list->lock);
433 seqcount_raw_spinlock_init(&pid_list->seqcount, &pid_list->lock);
434
435 for (i = 0; i < CHUNK_ALLOC; i++) {
436 union upper_chunk *chunk;
437
438 chunk = kzalloc_obj(*chunk);
439 if (!chunk)
440 break;
441 chunk->next = pid_list->upper_list;
442 pid_list->upper_list = chunk;
443 pid_list->free_upper_chunks++;
444 }
445
446 for (i = 0; i < CHUNK_ALLOC; i++) {
447 union lower_chunk *chunk;
448
449 chunk = kzalloc_obj(*chunk);
450 if (!chunk)
451 break;
452 chunk->next = pid_list->lower_list;
453 pid_list->lower_list = chunk;
454 pid_list->free_lower_chunks++;
455 }
456
457 return pid_list;
458}
459
460/**
461 * trace_pid_list_free - Frees an allocated pid_list.
462 * @pid_list: The pid list to free.
463 *
464 * Frees the memory for a pid_list that was allocated.
465 */
466void trace_pid_list_free(struct trace_pid_list *pid_list)
467{
468 union upper_chunk *upper;
469 union lower_chunk *lower;
470 int i, j;
471
472 if (!pid_list)
473 return;
474
475 irq_work_sync(&pid_list->refill_irqwork);
476
477 while (pid_list->lower_list) {
478 union lower_chunk *chunk;
479
480 chunk = pid_list->lower_list;
481 pid_list->lower_list = pid_list->lower_list->next;
482 kfree(chunk);
483 }
484
485 while (pid_list->upper_list) {
486 union upper_chunk *chunk;
487
488 chunk = pid_list->upper_list;
489 pid_list->upper_list = pid_list->upper_list->next;
490 kfree(chunk);
491 }
492
493 for (i = 0; i < UPPER1_SIZE; i++) {
494 upper = pid_list->upper[i];
495 if (upper) {
496 for (j = 0; j < UPPER2_SIZE; j++) {
497 lower = upper->data[j];
498 kfree(lower);
499 }
500 kfree(upper);
501 }
502 }
503 kfree(pid_list);
504}