projects/linux·blob master · refs

639 lines · 17.6 KiB

1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Landlock - Cross-thread ruleset enforcement
4 *
5 * Copyright © 2025 Google LLC
6 */
7
8#include <linux/atomic.h>
9#include <linux/cleanup.h>
10#include <linux/completion.h>
11#include <linux/cred.h>
12#include <linux/errno.h>
13#include <linux/overflow.h>
14#include <linux/rcupdate.h>
15#include <linux/sched.h>
16#include <linux/sched/signal.h>
17#include <linux/sched/task.h>
18#include <linux/slab.h>
19#include <linux/task_work.h>
20#include <uapi/linux/landlock.h>
21
22#include "cred.h"
23#include "tsync.h"
24
25#include <trace/events/landlock.h>
26
27/*
28 * Shared state between multiple threads which are enforcing Landlock rulesets
29 * in lockstep with each other.
30 */
31struct tsync_shared_context {
32 /* The old and tentative new creds of the calling thread. */
33 const struct cred *old_cred;
34 const struct cred *new_cred;
35
36 /* True if sibling tasks need to set the no_new_privs flag. */
37 bool set_no_new_privs;
38
39 /* An error encountered in preparation step, or 0. */
40 atomic_t preparation_error;
41
42 /*
43 * Barrier after preparation step in restrict_one_thread.
44 * The calling thread waits for completion.
45 *
46 * Re-initialized on every round of looking for newly spawned threads.
47 */
48 atomic_t num_preparing;
49 struct completion all_prepared;
50
51 /* Sibling threads wait for completion. */
52 struct completion ready_to_commit;
53
54 /*
55 * Barrier after commit step (used by syscall impl to wait for
56 * completion).
57 */
58 atomic_t num_unfinished;
59 struct completion all_finished;
60};
61
62struct tsync_work {
63 struct callback_head work;
64 struct task_struct *task;
65 struct tsync_shared_context *shared_ctx;
66};
67
68/*
69 * restrict_one_thread - update a thread's Landlock domain in lockstep with the
70 * other threads in the same process
71 *
72 * When this is run, the same function gets run in all other threads in the same
73 * process (except for the calling thread which called landlock_restrict_self).
74 * The concurrently running invocations of restrict_one_thread coordinate
75 * through the shared ctx object to do their work in lockstep to implement
76 * all-or-nothing semantics for enforcing the new Landlock domain.
77 *
78 * Afterwards, depending on the presence of an error, all threads either commit
79 * or abort the prepared credentials. The commit operation can not fail any
80 * more.
81 */
82static void restrict_one_thread(struct tsync_shared_context *ctx)
83{
84 const struct landlock_domain *new_dom =
85 landlock_cred(ctx->new_cred)->domain;
86 int err;
87 struct cred *cred = NULL;
88
89 if (current_cred() == ctx->old_cred) {
90 /*
91 * Switch out old_cred with new_cred, if possible.
92 *
93 * In the common case, where all threads initially point to the
94 * same struct cred, this optimization avoids creating separate
95 * redundant credentials objects for each, which would all have
96 * the same contents.
97 *
98 * Note: We are intentionally dropping the const qualifier
99 * here, because it is required by commit_creds() and
100 * abort_creds().
101 */
102 cred = (struct cred *)get_cred(ctx->new_cred);
103 } else {
104 /* Else, prepare new creds and populate them. */
105 cred = prepare_creds();
106
107 if (!cred) {
108 atomic_set(&ctx->preparation_error, -ENOMEM);
109
110 /*
111 * Even on error, we need to adhere to the protocol and
112 * coordinate with concurrently running invocations.
113 */
114 if (atomic_dec_return(&ctx->num_preparing) == 0)
115 complete_all(&ctx->all_prepared);
116
117 goto out;
118 }
119
120 landlock_cred_copy(landlock_cred(cred),
121 landlock_cred(ctx->new_cred));
122 }
123
124 /*
125 * Barrier: Wait until all threads are done preparing.
126 * After this point, we can have no more failures.
127 */
128 if (atomic_dec_return(&ctx->num_preparing) == 0)
129 complete_all(&ctx->all_prepared);
130
131 /*
132 * Wait for signal from calling thread that it's safe to read the
133 * preparation error now and we are ready to commit (or abort).
134 */
135 wait_for_completion(&ctx->ready_to_commit);
136
137 /* Abort the commit if any of the other threads had an error. */
138 err = atomic_read(&ctx->preparation_error);
139 if (err) {
140 abort_creds(cred);
141 goto out;
142 }
143
144 /*
145 * Make sure that all sibling tasks fulfill the no_new_privs
146 * prerequisite. (This is in line with Seccomp's
147 * SECCOMP_FILTER_FLAG_TSYNC logic in kernel/seccomp.c)
148 */
149 if (ctx->set_no_new_privs)
150 task_set_no_new_privs(current);
151
152 commit_creds(cred);
153
154 /*
155 * Emitted strictly after commit_creds() and before the out: label, so
156 * it fires only for a thread now enforcing new_dom, and every
157 * non-concluding (complete == false) event happens-before the
158 * operation's single concluding one. Skipped on the flags-only path,
159 * where old_cred and new_cred carry the same domain. A sibling never
160 * concludes the operation and its enforcement is always process-wide.
161 */
162 if (new_dom != landlock_cred(ctx->old_cred)->domain)
163 trace_landlock_enforce_domain(new_dom, false, true,
164 task_no_new_privs(current));
165
166out:
167 /* Notify the calling thread once all threads are done */
168 if (atomic_dec_return(&ctx->num_unfinished) == 0)
169 complete_all(&ctx->all_finished);
170}
171
172/*
173 * restrict_one_thread_callback - task_work callback for restricting a thread
174 *
175 * Calls restrict_one_thread with the struct landlock_shared_tsync_context.
176 */
177static void restrict_one_thread_callback(struct callback_head *work)
178{
179 struct tsync_work *ctx = container_of(work, struct tsync_work, work);
180
181 restrict_one_thread(ctx->shared_ctx);
182}
183
184/*
185 * struct tsync_works - a growable array of per-task contexts
186 *
187 * The zero-initialized struct represents the empty array.
188 */
189struct tsync_works {
190 struct tsync_work **works;
191 size_t size;
192 size_t capacity;
193};
194
195/*
196 * tsync_works_provide - provides a preallocated tsync_work for the given task
197 *
198 * This also stores a task pointer in the context and increments the reference
199 * count of the task.
200 *
201 * This function may fail in the case where we did not preallocate sufficient
202 * capacity. This can legitimately happen if new threads get started after we
203 * grew the capacity.
204 *
205 * Return: A pointer to the preallocated context struct with task filled in, or
206 * NULL if preallocated context structs ran out.
207 */
208static struct tsync_work *tsync_works_provide(struct tsync_works *s,
209 struct task_struct *task)
210{
211 struct tsync_work *ctx;
212
213 if (s->size >= s->capacity)
214 return NULL;
215
216 ctx = s->works[s->size];
217 s->size++;
218
219 ctx->task = get_task_struct(task);
220 return ctx;
221}
222
223/**
224 * tsync_works_trim - Put the last tsync_work element
225 *
226 * @s: TSYNC works to trim.
227 *
228 * Put the last task and decrement the size of @s.
229 *
230 * This helper does not cancel a running task, but just reset the last element
231 * to zero.
232 */
233static void tsync_works_trim(struct tsync_works *s)
234{
235 struct tsync_work *ctx;
236
237 if (WARN_ON_ONCE(s->size <= 0))
238 return;
239
240 ctx = s->works[s->size - 1];
241
242 /*
243 * For consistency, remove the task from ctx so that it does not look
244 * like we handed it a task_work.
245 */
246 put_task_struct(ctx->task);
247 *ctx = (typeof(*ctx)){};
248
249 /*
250 * Cancel the tsync_works_provide() change to recycle the reserved
251 * memory for the next thread, if any. This also ensures that
252 * cancel_tsync_works() and tsync_works_release() do not see any NULL
253 * task pointers.
254 */
255 s->size--;
256}
257
258/*
259 * tsync_works_grow_by - preallocates space for n more contexts in s
260 *
261 * On a successful return, the subsequent n calls to tsync_works_provide() are
262 * guaranteed to succeed. (size + n <= capacity)
263 *
264 * Return: 0 if sufficient space for n more elements could be provided, -ENOMEM
265 * on allocation errors, -EOVERFLOW in case of integer overflow.
266 */
267static int tsync_works_grow_by(struct tsync_works *s, size_t n, gfp_t flags)
268{
269 size_t i;
270 size_t new_capacity;
271 struct tsync_work **works;
272 struct tsync_work *work;
273
274 if (check_add_overflow(s->size, n, &new_capacity))
275 return -EOVERFLOW;
276
277 /* No need to reallocate if s already has sufficient capacity. */
278 if (new_capacity <= s->capacity)
279 return 0;
280
281 works = krealloc_array(s->works, new_capacity, sizeof(s->works[0]),
282 flags);
283 if (!works)
284 return -ENOMEM;
285
286 s->works = works;
287
288 for (i = s->capacity; i < new_capacity; i++) {
289 work = kzalloc_obj(*work, flags);
290 if (!work) {
291 /*
292 * Leave the object in a consistent state,
293 * but return an error.
294 */
295 s->capacity = i;
296 return -ENOMEM;
297 }
298 s->works[i] = work;
299 }
300 s->capacity = new_capacity;
301 return 0;
302}
303
304/*
305 * tsync_works_contains - checks for presence of task in s
306 */
307static bool tsync_works_contains_task(const struct tsync_works *s,
308 const struct task_struct *task)
309{
310 size_t i;
311
312 for (i = 0; i < s->size; i++)
313 if (s->works[i]->task == task)
314 return true;
315
316 return false;
317}
318
319/*
320 * tsync_works_release - frees memory held by s and drops all task references
321 *
322 * This does not free s itself, only the data structures held by it.
323 */
324static void tsync_works_release(struct tsync_works *s)
325{
326 size_t i;
327
328 for (i = 0; i < s->size; i++) {
329 if (WARN_ON_ONCE(!s->works[i]->task))
330 continue;
331
332 put_task_struct(s->works[i]->task);
333 }
334
335 for (i = 0; i < s->capacity; i++)
336 kfree(s->works[i]);
337
338 kfree(s->works);
339 s->works = NULL;
340 s->size = 0;
341 s->capacity = 0;
342}
343
344/*
345 * count_additional_threads - counts the sibling threads that are not in works
346 */
347static size_t count_additional_threads(const struct tsync_works *works)
348{
349 const struct task_struct *caller, *thread;
350 size_t n = 0;
351
352 caller = current;
353
354 guard(rcu)();
355
356 for_each_thread(caller, thread) {
357 /* Skip current, since it is initiating the sync. */
358 if (thread == caller)
359 continue;
360
361 /* Skip exited threads. */
362 if (thread->flags & PF_EXITING)
363 continue;
364
365 /* Skip threads that we have already seen. */
366 if (tsync_works_contains_task(works, thread))
367 continue;
368
369 n++;
370 }
371 return n;
372}
373
374/*
375 * schedule_task_work - adds task_work for all eligible sibling threads
376 * which have not been scheduled yet
377 *
378 * For each added task_work, atomically increments shared_ctx->num_preparing and
379 * shared_ctx->num_unfinished.
380 *
381 * Return: True if at least one eligible sibling thread was found, false
382 * otherwise.
383 */
384static bool schedule_task_work(struct tsync_works *works,
385 struct tsync_shared_context *shared_ctx)
386{
387 int err;
388 const struct task_struct *caller;
389 struct task_struct *thread;
390 struct tsync_work *ctx;
391 bool found_more_threads = false;
392
393 caller = current;
394
395 guard(rcu)();
396
397 for_each_thread(caller, thread) {
398 /* Skip current, since it is initiating the sync. */
399 if (thread == caller)
400 continue;
401
402 /* Skip exited threads. */
403 if (thread->flags & PF_EXITING)
404 continue;
405
406 /* Skip threads that we already looked at. */
407 if (tsync_works_contains_task(works, thread))
408 continue;
409
410 /*
411 * We found a sibling thread that is not doing its task_work
412 * yet, and which might spawn new threads before our task work
413 * runs, so we need at least one more round in the outer loop.
414 */
415 found_more_threads = true;
416
417 ctx = tsync_works_provide(works, thread);
418 if (!ctx) {
419 /*
420 * We ran out of preallocated contexts -- we need to
421 * try again with this thread at a later time!
422 * found_more_threads is already true at this point.
423 */
424 break;
425 }
426
427 ctx->shared_ctx = shared_ctx;
428
429 atomic_inc(&shared_ctx->num_preparing);
430 atomic_inc(&shared_ctx->num_unfinished);
431
432 init_task_work(&ctx->work, restrict_one_thread_callback);
433 err = task_work_add(thread, &ctx->work, TWA_SIGNAL);
434 if (unlikely(err)) {
435 /*
436 * task_work_add() only fails if the task is about to
437 * exit. We checked that earlier, but it can happen as
438 * a race. Resume without setting an error, as the
439 * task is probably gone in the next loop iteration.
440 */
441 tsync_works_trim(works);
442
443 atomic_dec(&shared_ctx->num_preparing);
444 atomic_dec(&shared_ctx->num_unfinished);
445 }
446 }
447
448 return found_more_threads;
449}
450
451/*
452 * cancel_tsync_works - cancel all task works where it is possible
453 *
454 * Task works can be canceled as long as they are still queued and have not
455 * started running. If they get canceled, we decrement
456 * shared_ctx->num_preparing and shared_ctx->num_unfished and mark the two
457 * completions if needed, as if the task was never scheduled.
458 */
459static void cancel_tsync_works(const struct tsync_works *works,
460 struct tsync_shared_context *shared_ctx)
461{
462 size_t i;
463
464 for (i = 0; i < works->size; i++) {
465 if (WARN_ON_ONCE(!works->works[i]->task))
466 continue;
467
468 if (!task_work_cancel(works->works[i]->task,
469 &works->works[i]->work))
470 continue;
471
472 /* After dequeueing, act as if the task work had executed. */
473
474 if (atomic_dec_return(&shared_ctx->num_preparing) == 0)
475 complete_all(&shared_ctx->all_prepared);
476
477 if (atomic_dec_return(&shared_ctx->num_unfinished) == 0)
478 complete_all(&shared_ctx->all_finished);
479 }
480}
481
482/*
483 * restrict_sibling_threads - enables a Landlock policy for all sibling threads
484 */
485int landlock_restrict_sibling_threads(const struct cred *old_cred,
486 const struct cred *new_cred,
487 const u32 restrict_flags)
488{
489 int err;
490 struct tsync_shared_context shared_ctx;
491 struct tsync_works works = {};
492 size_t newly_discovered_threads;
493 bool found_more_threads;
494
495 atomic_set(&shared_ctx.preparation_error, 0);
496 init_completion(&shared_ctx.all_prepared);
497 init_completion(&shared_ctx.ready_to_commit);
498 atomic_set(&shared_ctx.num_unfinished, 1);
499 init_completion(&shared_ctx.all_finished);
500 shared_ctx.old_cred = old_cred;
501 shared_ctx.new_cred = new_cred;
502 shared_ctx.set_no_new_privs =
503 (restrict_flags & LANDLOCK_RESTRICT_SELF_NO_NEW_PRIVS) ||
504 task_no_new_privs(current);
505
506 /*
507 * Serialize concurrent TSYNC operations to prevent deadlocks when
508 * multiple threads call landlock_restrict_self() simultaneously.
509 * If the lock is already held, we gracefully yield by restarting the
510 * syscall. This allows the current thread to process pending
511 * task_works before retrying.
512 */
513 if (!down_write_trylock(&current->signal->exec_update_lock))
514 return restart_syscall();
515
516 /*
517 * We schedule a pseudo-signal task_work for each of the calling task's
518 * sibling threads. In the task work, each thread:
519 *
520 * 1) runs prepare_creds() and writes back the error to
521 * shared_ctx.preparation_error, if needed.
522 *
523 * 2) signals that it's done with prepare_creds() to the calling task.
524 * (completion "all_prepared").
525 *
526 * 3) waits for the completion "ready_to_commit". This is sent by the
527 * calling task after ensuring that all sibling threads have done
528 * with the "preparation" stage.
529 *
530 * After this barrier is reached, it's safe to read
531 * shared_ctx.preparation_error.
532 *
533 * 4) reads shared_ctx.preparation_error and then either does
534 * commit_creds() or abort_creds().
535 *
536 * 5) signals that it's done altogether (barrier synchronization
537 * "all_finished")
538 *
539 * Unlike seccomp, which modifies sibling tasks directly, we do not
540 * need to acquire the cred_guard_mutex and sighand->siglock:
541 *
542 * - As in our case, all threads are themselves exchanging their own
543 * struct cred through the credentials API, no locks are needed for
544 * that.
545 * - Our for_each_thread() loops are protected by RCU.
546 * - We do not acquire a lock to keep the list of sibling threads
547 * stable between our for_each_thread loops. If the list of
548 * available sibling threads changes between these for_each_thread
549 * loops, we make up for that by continuing to look for threads until
550 * they are all discovered and have entered their task_work, where
551 * they are unable to spawn new threads.
552 */
553 do {
554 /* In RCU read-lock, count the threads we need. */
555 newly_discovered_threads = count_additional_threads(&works);
556
557 if (newly_discovered_threads == 0)
558 break; /* done */
559
560 err = tsync_works_grow_by(&works, newly_discovered_threads,
561 GFP_KERNEL_ACCOUNT);
562 if (err) {
563 atomic_set(&shared_ctx.preparation_error, err);
564 break;
565 }
566
567 /*
568 * The "all_prepared" barrier is used locally to the loop body,
569 * this use of for_each_thread(). We can reset it on each loop
570 * iteration because all previous loop iterations are done with
571 * it already.
572 *
573 * num_preparing is initialized to 1 so that the counter can
574 * not go to 0 and mark the completion as done before all task
575 * works are registered. We decrement it at the end of the
576 * loop body.
577 */
578 atomic_set(&shared_ctx.num_preparing, 1);
579 reinit_completion(&shared_ctx.all_prepared);
580
581 /*
582 * In RCU read-lock, schedule task work on newly discovered
583 * sibling tasks.
584 */
585 found_more_threads = schedule_task_work(&works, &shared_ctx);
586
587 /*
588 * Decrement num_preparing for current, to undo that we
589 * initialized it to 1 a few lines above.
590 */
591 if (atomic_dec_return(&shared_ctx.num_preparing) > 0) {
592 if (wait_for_completion_interruptible(
593 &shared_ctx.all_prepared)) {
594 /*
595 * In case of interruption, we need to retry
596 * the system call.
597 */
598 atomic_set(&shared_ctx.preparation_error,
599 -ERESTARTNOINTR);
600
601 /*
602 * Opportunistic improvement: try to cancel task
603 * works for tasks that did not start running
604 * yet. We do not have a guarantee that it
605 * cancels any of the enqueued task works
606 * because task_work_run() might already have
607 * dequeued them.
608 */
609 cancel_tsync_works(&works, &shared_ctx);
610
611 /*
612 * Break the loop with error. The cleanup code
613 * after the loop unblocks the remaining
614 * task_works.
615 */
616 break;
617 }
618 }
619 } while (found_more_threads &&
620 !atomic_read(&shared_ctx.preparation_error));
621
622 /*
623 * We now have either (a) all sibling threads blocking and in "prepared"
624 * state in the task work, or (b) the preparation error is set. Ask all
625 * threads to commit (or abort).
626 */
627 complete_all(&shared_ctx.ready_to_commit);
628
629 /*
630 * Decrement num_unfinished for current, to undo that we initialized it
631 * to 1 at the beginning.
632 */
633 if (atomic_dec_return(&shared_ctx.num_unfinished) > 0)
634 wait_for_completion(&shared_ctx.all_finished);
635
636 tsync_works_release(&works);
637 up_write(&current->signal->exec_update_lock);
638 return atomic_read(&shared_ctx.preparation_error);
639}