projects/linux·blob master · refs

1872 lines · 63.6 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3// Copyright (C) 2025 Google LLC.
4
5//! This module defines the `Process` type, which represents a process using a particular binder
6//! context.
7//!
8//! The `Process` object keeps track of all of the resources that this process owns in the binder
9//! context.
10//!
11//! There is one `Process` object for each binder fd that a process has opened, so processes using
12//! several binder contexts have several `Process` objects. This ensures that the contexts are
13//! fully separated.
14
15use core::mem::take;
16
17use kernel::{
18 bindings,
19 bits::bit_u8,
20 cred::Credential,
21 error::Error,
22 fs::file::{self, File},
23 id_pool::IdPool,
24 list::{List, ListArc, ListArcField, ListLinks},
25 mm,
26 prelude::*,
27 rbtree::{self, RBTree, RBTreeNode, RBTreeNodeReservation},
28 seq_file::SeqFile,
29 seq_print,
30 sync::poll::PollTable,
31 sync::{
32 aref::ARef,
33 lock::{spinlock::SpinLockBackend, Guard},
34 poll::PollCondVarBox,
35 Arc, ArcBorrow, CondVar, CondVarTimeoutResult, SetOnce, SpinLock, UniqueArc,
36 },
37 task::{Pid, Task},
38 uaccess::{UserSlice, UserSliceReader},
39 uapi,
40 workqueue::{self, Work},
41};
42
43use crate::{
44 allocation::{Allocation, AllocationInfo, NewAllocation},
45 context::Context,
46 defs::*,
47 error::{BinderError, BinderResult},
48 node::{CouldNotDeliverCriticalIncrement, CritIncrWrapper, Node, NodeDeath, NodeRef},
49 page_range::ShrinkablePageRange,
50 range_alloc::{RangeAllocator, ReserveNew, ReserveNewArgs},
51 stats::BinderStats,
52 thread::{PushWorkRes, Thread},
53 transaction::TransactionInfo,
54 BinderfsProcFile, DArc, DLArc, DTRWrap, DeliverToRead,
55};
56
57#[path = "freeze.rs"]
58mod freeze;
59use self::freeze::{FreezeCookie, FreezeListener};
60
61struct Mapping {
62 address: usize,
63 alloc: RangeAllocator<AllocationInfo>,
64}
65
66impl Mapping {
67 fn new(address: usize, size: usize) -> Self {
68 Self {
69 address,
70 alloc: RangeAllocator::new(size),
71 }
72 }
73}
74
75kernel::impl_flags!(
76 /// Represents multiple deferred work flags.
77 #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)]
78 pub struct DeferWorks(u8);
79
80 /// Represents a single deferred work category.
81 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
82 pub enum DeferWork {
83 Flush = bit_u8(0),
84 Release = bit_u8(1),
85 }
86);
87
88#[derive(Copy, Clone)]
89pub(crate) enum IsFrozen {
90 Yes,
91 No,
92 InProgress,
93}
94
95impl IsFrozen {
96 /// Whether incoming transactions should be rejected due to freeze.
97 pub(crate) fn is_frozen(self) -> bool {
98 match self {
99 IsFrozen::Yes => true,
100 IsFrozen::No => false,
101 IsFrozen::InProgress => true,
102 }
103 }
104
105 /// Whether freeze notifications consider this process frozen.
106 pub(crate) fn is_fully_frozen(self) -> bool {
107 match self {
108 IsFrozen::Yes => true,
109 IsFrozen::No => false,
110 IsFrozen::InProgress => false,
111 }
112 }
113}
114
115/// The fields of `Process` protected by the spinlock.
116pub(crate) struct ProcessInner {
117 is_manager: bool,
118 pub(crate) is_dead: bool,
119 threads: RBTree<i32, Arc<Thread>>,
120 /// INVARIANT: Threads pushed to this list must be owned by this process.
121 ready_threads: List<Thread>,
122 nodes: RBTree<u64, DArc<Node>>,
123 mapping: Option<Mapping>,
124 work: List<DTRWrap<dyn DeliverToRead>>,
125 delivered_deaths: List<DTRWrap<NodeDeath>, 2>,
126
127 /// The number of requested threads that haven't registered yet.
128 requested_thread_count: u32,
129 /// The maximum number of threads used by the process thread pool.
130 max_threads: u32,
131 /// The number of threads the started and registered with the thread pool.
132 started_thread_count: u32,
133
134 /// Bitmap of deferred work to do.
135 defer_work: DeferWorks,
136
137 /// Number of transactions to be transmitted before processes in freeze_wait
138 /// are woken up.
139 outstanding_txns: u32,
140 /// Process is frozen and unable to service binder transactions.
141 pub(crate) is_frozen: IsFrozen,
142 /// Process received sync transactions since last frozen.
143 pub(crate) sync_recv: bool,
144 /// Process received async transactions since last frozen.
145 pub(crate) async_recv: bool,
146 pub(crate) binderfs_file: Option<BinderfsProcFile>,
147 /// Check for oneway spam
148 oneway_spam_detection_enabled: bool,
149}
150
151impl ProcessInner {
152 fn new() -> Self {
153 Self {
154 is_manager: false,
155 is_dead: false,
156 threads: RBTree::new(),
157 ready_threads: List::new(),
158 mapping: None,
159 nodes: RBTree::new(),
160 work: List::new(),
161 delivered_deaths: List::new(),
162 requested_thread_count: 0,
163 max_threads: 0,
164 started_thread_count: 0,
165 defer_work: DeferWorks::default(),
166 outstanding_txns: 0,
167 is_frozen: IsFrozen::No,
168 sync_recv: false,
169 async_recv: false,
170 binderfs_file: None,
171 oneway_spam_detection_enabled: false,
172 }
173 }
174
175 /// Schedule the work item for execution on this process.
176 ///
177 /// If any threads are ready for work, then the work item is given directly to that thread and
178 /// it is woken up. Otherwise, it is pushed to the process work list.
179 ///
180 /// This call can fail only if the process is dead. In this case, the work item is returned to
181 /// the caller so that the caller can drop it after releasing the inner process lock. This is
182 /// necessary since the destructor of `Transaction` will take locks that can't necessarily be
183 /// taken while holding the inner process lock.
184 pub(crate) fn push_work(
185 &mut self,
186 proc: &Process,
187 work: DLArc<dyn DeliverToRead>,
188 ) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> {
189 let sync = work.should_sync_wakeup();
190
191 // Try to find a ready thread to which to push the work.
192 if let Some(thread) = self.ready_threads.pop_front() {
193 // Push to thread while holding state lock. This prevents the thread from giving up
194 // (for example, because of a signal) when we're about to deliver work.
195 match thread.push_work_inner(work, sync) {
196 PushWorkRes::Ok => Ok(()),
197 PushWorkRes::OkNotifyPoll => {
198 proc.notify_poll(sync);
199 Ok(())
200 }
201 PushWorkRes::FailedDead(work) => Err((BinderError::new_dead(), work)),
202 }
203 } else if self.is_dead {
204 Err((BinderError::new_dead(), work))
205 } else {
206 // Didn't find a thread waiting for proc work; this can happen
207 // in two scenarios:
208 // 1. All threads are busy handling transactions
209 // In that case, one of those threads should call back into
210 // the kernel driver soon and pick up this work.
211 // 2. Threads are using the (e)poll interface, in which case
212 // they may be blocked on the waitqueue without having been
213 // added to waiting_threads. For this case, we wake it up
214 // directly.
215 self.work.push_back(work);
216
217 // Wake up polling threads, if any.
218 proc.notify_poll(sync);
219
220 Ok(())
221 }
222 }
223
224 pub(crate) fn remove_node(&mut self, ptr: u64) {
225 self.nodes.remove(&ptr);
226 }
227
228 /// Updates the reference count on the given node.
229 pub(crate) fn update_node_refcount(
230 &mut self,
231 node: &DArc<Node>,
232 inc: bool,
233 strong: bool,
234 count: usize,
235 othread: Option<&Thread>,
236 ) {
237 let push = node.update_refcount_locked(inc, strong, count, self);
238
239 // If we decided that we need to push work, push either to the process or to a thread if
240 // one is specified.
241 if let Some(pnode) = push {
242 if let Some(thread) = othread {
243 thread.push_work_deferred(pnode);
244 } else {
245 let _ = self.push_work(&node.owner, pnode);
246 // Nothing to do: `push_work` may fail if the process is dead, but that's ok as in
247 // that case, it doesn't care about the notification.
248 }
249 }
250 }
251
252 pub(crate) fn new_node_ref(
253 &mut self,
254 node: DArc<Node>,
255 strong: bool,
256 thread: Option<&Thread>,
257 ) -> NodeRef {
258 self.update_node_refcount(&node, true, strong, 1, thread);
259 let strong_count = if strong { 1 } else { 0 };
260 NodeRef::new(node, strong_count, 1 - strong_count)
261 }
262
263 pub(crate) fn new_node_ref_with_thread(
264 &mut self,
265 node: DArc<Node>,
266 strong: bool,
267 thread: &Thread,
268 wrapper: Option<CritIncrWrapper>,
269 ) -> Result<NodeRef, CouldNotDeliverCriticalIncrement> {
270 let push = match wrapper {
271 None => node
272 .incr_refcount_allow_zero2one(strong, self)?
273 .map(|node| node as DLArc<dyn DeliverToRead>),
274 Some(wrapper) => node.incr_refcount_allow_zero2one_with_wrapper(strong, wrapper, self),
275 };
276 if let Some(node) = push {
277 thread.push_work_deferred(node);
278 }
279 let strong_count = if strong { 1 } else { 0 };
280 Ok(NodeRef::new(node, strong_count, 1 - strong_count))
281 }
282
283 /// Returns an existing node with the given pointer and cookie, if one exists.
284 ///
285 /// Returns an error if a node with the given pointer but a different cookie exists.
286 fn get_existing_node(&self, ptr: u64, cookie: u64) -> Result<Option<DArc<Node>>> {
287 match self.nodes.get(&ptr) {
288 None => Ok(None),
289 Some(node) => {
290 let (_, node_cookie) = node.get_id();
291 if node_cookie == cookie {
292 Ok(Some(node.clone()))
293 } else {
294 Err(EINVAL)
295 }
296 }
297 }
298 }
299
300 fn register_thread(&mut self) -> bool {
301 if self.requested_thread_count == 0 {
302 return false;
303 }
304
305 self.requested_thread_count -= 1;
306 self.started_thread_count += 1;
307 true
308 }
309
310 /// Finds a delivered death notification with the given cookie, removes it from the thread's
311 /// delivered list, and returns it.
312 fn pull_delivered_death(&mut self, cookie: u64) -> Option<DArc<NodeDeath>> {
313 let mut cursor = self.delivered_deaths.cursor_front();
314 while let Some(next) = cursor.peek_next() {
315 if next.cookie == cookie {
316 return Some(next.remove().into_arc());
317 }
318 cursor.move_next();
319 }
320 None
321 }
322
323 pub(crate) fn death_delivered(&mut self, death: DArc<NodeDeath>) {
324 if let Some(death) = ListArc::try_from_arc_or_drop(death) {
325 self.delivered_deaths.push_back(death);
326 } else {
327 pr_warn!("Notification added to `delivered_deaths` twice.");
328 }
329 }
330
331 pub(crate) fn add_outstanding_txn(&mut self) {
332 self.outstanding_txns += 1;
333 }
334
335 fn txns_pending_locked(&self) -> bool {
336 if self.outstanding_txns > 0 {
337 return true;
338 }
339 for thread in self.threads.values() {
340 if thread.has_current_transaction() {
341 return true;
342 }
343 }
344 false
345 }
346}
347
348/// Used to keep track of a node that this process has a handle to.
349#[pin_data]
350pub(crate) struct NodeRefInfo {
351 debug_id: usize,
352 /// The refcount that this process owns to the node.
353 node_ref: ListArcField<NodeRef, { Self::LIST_PROC }>,
354 death: ListArcField<Option<DArc<NodeDeath>>, { Self::LIST_PROC }>,
355 /// Cookie of the active freeze listener for this node.
356 freeze: ListArcField<Option<FreezeCookie>, { Self::LIST_PROC }>,
357 /// Used to store this `NodeRefInfo` in the node's `refs` list.
358 #[pin]
359 links: ListLinks<{ Self::LIST_NODE }>,
360 /// The handle for this `NodeRefInfo`.
361 handle: u32,
362 /// The process that has a handle to the node.
363 pub(crate) process: Arc<Process>,
364}
365
366impl NodeRefInfo {
367 /// The id used for the `Node::refs` list.
368 pub(crate) const LIST_NODE: u64 = 0x2da16350fb724a10;
369 /// The id used for the `ListArc` in `ProcessNodeRefs`.
370 const LIST_PROC: u64 = 0xd703a5263dcc8650;
371
372 fn new(node_ref: NodeRef, handle: u32, process: Arc<Process>) -> impl PinInit<Self> {
373 pin_init!(Self {
374 debug_id: super::next_debug_id(),
375 node_ref: ListArcField::new(node_ref),
376 death: ListArcField::new(None),
377 freeze: ListArcField::new(None),
378 links <- ListLinks::new(),
379 handle,
380 process,
381 })
382 }
383
384 kernel::list::define_list_arc_field_getter! {
385 pub(crate) fn death(&mut self<{Self::LIST_PROC}>) -> &mut Option<DArc<NodeDeath>> { death }
386 pub(crate) fn freeze(&mut self<{Self::LIST_PROC}>) -> &mut Option<FreezeCookie> { freeze }
387 pub(crate) fn node_ref(&mut self<{Self::LIST_PROC}>) -> &mut NodeRef { node_ref }
388 pub(crate) fn node_ref2(&self<{Self::LIST_PROC}>) -> &NodeRef { node_ref }
389 }
390}
391
392kernel::list::impl_list_arc_safe! {
393 impl ListArcSafe<{Self::LIST_NODE}> for NodeRefInfo { untracked; }
394 impl ListArcSafe<{Self::LIST_PROC}> for NodeRefInfo { untracked; }
395}
396kernel::list::impl_list_item! {
397 impl ListItem<{Self::LIST_NODE}> for NodeRefInfo {
398 using ListLinks { self.links };
399 }
400}
401
402/// Keeps track of references this process has to nodes owned by other processes.
403///
404/// TODO: Currently, the rbtree requires two allocations per node reference, and two tree
405/// traversals to look up a node by `Node::global_id`. Once the rbtree is more powerful, these
406/// extra costs should be eliminated.
407struct ProcessNodeRefs {
408 /// Used to look up nodes using the 32-bit id that this process knows it by.
409 by_handle: RBTree<u32, ListArc<NodeRefInfo, { NodeRefInfo::LIST_PROC }>>,
410 /// Used to quickly find unused ids in `by_handle`.
411 handle_is_present: IdPool,
412 /// Used to look up nodes without knowing their local 32-bit id. The usize is the address of
413 /// the underlying `Node` struct as returned by `Node::global_id`.
414 by_node: RBTree<usize, u32>,
415 /// Used to look up a `FreezeListener` by cookie.
416 ///
417 /// There might be multiple freeze listeners for the same node, but at most one of them is
418 /// active.
419 freeze_listeners: RBTree<FreezeCookie, FreezeListener>,
420}
421
422impl ProcessNodeRefs {
423 fn new() -> Self {
424 Self {
425 by_handle: RBTree::new(),
426 handle_is_present: IdPool::new(),
427 by_node: RBTree::new(),
428 freeze_listeners: RBTree::new(),
429 }
430 }
431}
432
433use core::mem::offset_of;
434use kernel::bindings::rb_process_layout;
435pub(crate) const PROCESS_LAYOUT: rb_process_layout = rb_process_layout {
436 arc_offset: Arc::<Process>::DATA_OFFSET,
437 task: offset_of!(Process, task),
438};
439
440/// A process using binder.
441///
442/// Strictly speaking, there can be multiple of these per process. There is one for each binder fd
443/// that a process has opened, so processes using several binder contexts have several `Process`
444/// objects. This ensures that the contexts are fully separated.
445#[pin_data]
446pub(crate) struct Process {
447 pub(crate) ctx: Arc<Context>,
448
449 // The task leader (process).
450 pub(crate) task: ARef<Task>,
451
452 // Credential associated with file when `Process` is created.
453 pub(crate) cred: ARef<Credential>,
454
455 #[pin]
456 pub(crate) inner: SpinLock<ProcessInner>,
457
458 #[pin]
459 pub(crate) pages: ShrinkablePageRange,
460
461 // Waitqueue of processes waiting for all outstanding transactions to be
462 // processed.
463 #[pin]
464 freeze_wait: CondVar,
465
466 // Node references are in a different lock to avoid recursive acquisition when
467 // incrementing/decrementing a node in another process.
468 #[pin]
469 node_refs: SpinLock<ProcessNodeRefs>,
470
471 // Synchronizes `register_wait` calls to the `PollCondVarBox`.
472 //
473 // The `PollCondVarBox` is not stored here because synchronization is
474 // done for `register_wait` only. Wakeups do not take this lock.
475 poll: SetOnce<PollCondVarBox>,
476
477 // Work node for deferred work item.
478 #[pin]
479 defer_work: Work<Process>,
480
481 // Links for process list in Context.
482 #[pin]
483 links: ListLinks,
484
485 pub(crate) stats: BinderStats,
486}
487
488kernel::impl_has_work! {
489 impl HasWork<Process> for Process { self.defer_work }
490}
491
492kernel::list::impl_list_arc_safe! {
493 impl ListArcSafe<0> for Process { untracked; }
494}
495kernel::list::impl_list_item! {
496 impl ListItem<0> for Process {
497 using ListLinks { self.links };
498 }
499}
500
501impl workqueue::WorkItem for Process {
502 type Pointer = Arc<Process>;
503
504 fn run(me: Arc<Self>) {
505 let defer;
506 {
507 let mut inner = me.inner.lock();
508 defer = inner.defer_work;
509 inner.defer_work = DeferWorks::default();
510 }
511
512 if defer.contains(DeferWork::Flush) {
513 me.deferred_flush();
514 }
515 if defer.contains(DeferWork::Release) {
516 me.deferred_release();
517 }
518 }
519}
520
521impl Process {
522 fn new(ctx: Arc<Context>, cred: ARef<Credential>) -> Result<Arc<Self>> {
523 let current = kernel::current!();
524 let process = Arc::pin_init::<Error>(
525 try_pin_init!(Process {
526 ctx,
527 cred,
528 inner <- kernel::new_spinlock!(ProcessInner::new(), "Process::inner"),
529 pages <- ShrinkablePageRange::new(&super::BINDER_SHRINKER),
530 node_refs <- kernel::new_spinlock!(ProcessNodeRefs::new(), "Process::node_refs"),
531 freeze_wait <- kernel::new_condvar!("Process::freeze_wait"),
532 task: current.group_leader().into(),
533 defer_work <- kernel::new_work!("Process::defer_work"),
534 links <- ListLinks::new(),
535 stats: BinderStats::new(),
536 poll: SetOnce::new(),
537 }),
538 GFP_KERNEL,
539 )?;
540
541 process.ctx.register_process(process.clone())?;
542
543 Ok(process)
544 }
545
546 pub(crate) fn pid_in_current_ns(&self) -> kernel::task::Pid {
547 self.task.tgid_nr_ns(None)
548 }
549
550 #[inline(never)]
551 pub(crate) fn debug_print_stats(&self, m: &SeqFile, ctx: &Context) -> Result<()> {
552 seq_print!(m, "proc {}\n", self.pid_in_current_ns());
553 seq_print!(m, "context {}\n", &*ctx.name);
554
555 let inner = self.inner.lock();
556 seq_print!(m, " threads: {}\n", inner.threads.iter().count());
557 seq_print!(
558 m,
559 " requested threads: {}+{}/{}\n",
560 inner.requested_thread_count,
561 inner.started_thread_count,
562 inner.max_threads,
563 );
564 if let Some(mapping) = &inner.mapping {
565 seq_print!(
566 m,
567 " free oneway space: {}\n",
568 mapping.alloc.free_oneway_space()
569 );
570 seq_print!(m, " buffers: {}\n", mapping.alloc.count_buffers());
571 }
572 seq_print!(
573 m,
574 " outstanding transactions: {}\n",
575 inner.outstanding_txns
576 );
577 seq_print!(m, " nodes: {}\n", inner.nodes.iter().count());
578 drop(inner);
579
580 {
581 let mut refs = self.node_refs.lock();
582 let (mut count, mut weak, mut strong) = (0, 0, 0);
583 for r in refs.by_handle.values_mut() {
584 let node_ref = r.node_ref();
585 let (nstrong, nweak) = node_ref.get_count();
586 count += 1;
587 weak += nweak;
588 strong += nstrong;
589 }
590 seq_print!(m, " refs: {count} s {strong} w {weak}\n");
591 }
592
593 self.stats.debug_print(" ", m);
594
595 Ok(())
596 }
597
598 #[inline(never)]
599 pub(crate) fn debug_print(&self, m: &SeqFile, ctx: &Context, print_all: bool) -> Result<()> {
600 seq_print!(m, "proc {}\n", self.pid_in_current_ns());
601 seq_print!(m, "context {}\n", &*ctx.name);
602
603 let mut all_threads = KVec::new();
604 let mut all_nodes = KVec::new();
605 loop {
606 let inner = self.inner.lock();
607 let num_threads = inner.threads.iter().count();
608 let num_nodes = inner.nodes.iter().count();
609
610 if all_threads.capacity() < num_threads || all_nodes.capacity() < num_nodes {
611 drop(inner);
612 all_threads.reserve(num_threads, GFP_KERNEL)?;
613 all_nodes.reserve(num_nodes, GFP_KERNEL)?;
614 continue;
615 }
616
617 for thread in inner.threads.values() {
618 assert!(all_threads.len() < all_threads.capacity());
619 let _ = all_threads.push(thread.clone(), GFP_ATOMIC);
620 }
621
622 for node in inner.nodes.values() {
623 assert!(all_nodes.len() < all_nodes.capacity());
624 let _ = all_nodes.push(node.clone(), GFP_ATOMIC);
625 }
626
627 break;
628 }
629
630 for thread in all_threads {
631 thread.debug_print(m, print_all)?;
632 }
633
634 let mut inner = self.inner.lock();
635 for node in all_nodes {
636 if print_all || node.has_oneway_transaction(&mut inner) {
637 node.full_debug_print(m, &mut inner)?;
638 }
639 }
640 drop(inner);
641
642 if print_all {
643 let mut refs = self.node_refs.lock();
644 for r in refs.by_handle.values_mut() {
645 let node_ref = r.node_ref();
646 let dead = node_ref.node.owner.inner.lock().is_dead;
647 let (strong, weak) = node_ref.get_count();
648 let debug_id = node_ref.node.debug_id;
649
650 seq_print!(
651 m,
652 " ref {}: desc {} {}node {debug_id} s {strong} w {weak}",
653 r.debug_id,
654 r.handle,
655 if dead { "dead " } else { "" }
656 );
657 }
658 }
659
660 let inner = self.inner.lock();
661 for work in &inner.work {
662 work.debug_print(m, " ", " pending transaction ")?;
663 }
664 for _death in &inner.delivered_deaths {
665 seq_print!(m, " has delivered dead binder\n");
666 }
667 if let Some(mapping) = &inner.mapping {
668 mapping.alloc.debug_print(m)?;
669 }
670 drop(inner);
671
672 Ok(())
673 }
674
675 /// Attempts to fetch a work item from the process queue.
676 pub(crate) fn get_work(&self) -> Option<DLArc<dyn DeliverToRead>> {
677 self.inner.lock().work.pop_front()
678 }
679
680 /// Attempts to fetch a work item from the process queue. If none is available, it registers the
681 /// given thread as ready to receive work directly.
682 ///
683 /// This must only be called when the thread is not participating in a transaction chain; when
684 /// it is, work will always be delivered directly to the thread (and not through the process
685 /// queue).
686 pub(crate) fn get_work_or_register<'a>(
687 &'a self,
688 thread: &'a Arc<Thread>,
689 ) -> GetWorkOrRegister<'a> {
690 let mut inner = self.inner.lock();
691 // Try to get work from the process queue.
692 if let Some(work) = inner.work.pop_front() {
693 return GetWorkOrRegister::Work(work);
694 }
695
696 // Register the thread as ready.
697 GetWorkOrRegister::Register(Registration::new(thread, &mut inner))
698 }
699
700 fn get_current_thread(self: ArcBorrow<'_, Self>) -> Result<Arc<Thread>> {
701 let id = {
702 let current = kernel::current!();
703 if self.task != current.group_leader() {
704 pr_err!("get_current_thread was called from the wrong process.");
705 return Err(EINVAL);
706 }
707 current.pid()
708 };
709
710 {
711 let inner = self.inner.lock();
712 if let Some(thread) = inner.threads.get(&id) {
713 return Ok(thread.clone());
714 }
715 }
716
717 // Allocate a new `Thread` without holding any locks.
718 let reservation = RBTreeNodeReservation::new(GFP_KERNEL)?;
719 let ta: Arc<Thread> = Thread::new(id, self.into())?;
720
721 let mut inner = self.inner.lock();
722 match inner.threads.entry(id) {
723 rbtree::Entry::Vacant(entry) => {
724 entry.insert(ta.clone(), reservation);
725 Ok(ta)
726 }
727 rbtree::Entry::Occupied(_entry) => {
728 pr_err!("Cannot create two threads with the same id.");
729 Err(EINVAL)
730 }
731 }
732 }
733
734 pub(crate) fn push_work(&self, work: DLArc<dyn DeliverToRead>) -> BinderResult {
735 // If push_work fails, drop the work item outside the lock.
736 let res = self.inner.lock().push_work(self, work);
737 match res {
738 Ok(()) => Ok(()),
739 Err((err, work)) => {
740 drop(work);
741 Err(err)
742 }
743 }
744 }
745
746 fn set_as_manager(
747 self: ArcBorrow<'_, Self>,
748 info: Option<FlatBinderObject>,
749 thread: &Thread,
750 ) -> Result {
751 let (ptr, cookie, flags) = if let Some(obj) = info {
752 (
753 // SAFETY: The object type for this ioctl is implicitly `BINDER_TYPE_BINDER`, so it
754 // is safe to access the `binder` field.
755 unsafe { obj.__bindgen_anon_1.binder },
756 obj.cookie,
757 obj.flags,
758 )
759 } else {
760 (0, 0, 0)
761 };
762 let node_ref = self.get_node(ptr, cookie, flags, true, thread)?;
763 let node = node_ref.node.clone();
764 self.ctx.set_manager_node(node_ref)?;
765 self.inner.lock().is_manager = true;
766
767 // Force the state of the node to prevent the delivery of acquire/increfs.
768 let mut owner_inner = node.owner.inner.lock();
769 node.force_has_count(&mut owner_inner);
770 Ok(())
771 }
772
773 fn get_node_inner(
774 self: ArcBorrow<'_, Self>,
775 ptr: u64,
776 cookie: u64,
777 flags: u32,
778 strong: bool,
779 thread: &Thread,
780 wrapper: Option<CritIncrWrapper>,
781 ) -> Result<Result<NodeRef, CouldNotDeliverCriticalIncrement>> {
782 // Try to find an existing node.
783 {
784 let mut inner = self.inner.lock();
785 if let Some(node) = inner.get_existing_node(ptr, cookie)? {
786 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper));
787 }
788 }
789
790 // Allocate the node before reacquiring the lock.
791 let node = DTRWrap::arc_pin_init(Node::new(ptr, cookie, flags, self.into()))?.into_arc();
792 let rbnode = RBTreeNode::new(ptr, node.clone(), GFP_KERNEL)?;
793 let mut inner = self.inner.lock();
794 if let Some(node) = inner.get_existing_node(ptr, cookie)? {
795 return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper));
796 }
797
798 inner.nodes.insert(rbnode);
799 // This can only fail if someone has already pushed the node to a list, but we just created
800 // it and still hold the lock, so it can't fail right now.
801 let node_ref = inner
802 .new_node_ref_with_thread(node, strong, thread, wrapper)
803 .unwrap();
804
805 Ok(Ok(node_ref))
806 }
807
808 pub(crate) fn get_node(
809 self: ArcBorrow<'_, Self>,
810 ptr: u64,
811 cookie: u64,
812 flags: u32,
813 strong: bool,
814 thread: &Thread,
815 ) -> Result<NodeRef> {
816 let mut wrapper = None;
817 for _ in 0..2 {
818 match self.get_node_inner(ptr, cookie, flags, strong, thread, wrapper) {
819 Err(err) => return Err(err),
820 Ok(Ok(node_ref)) => return Ok(node_ref),
821 Ok(Err(CouldNotDeliverCriticalIncrement)) => {
822 wrapper = Some(CritIncrWrapper::new()?);
823 }
824 }
825 }
826 // We only get a `CouldNotDeliverCriticalIncrement` error if `wrapper` is `None`, so the
827 // loop should run at most twice.
828 unreachable!()
829 }
830
831 pub(crate) fn insert_or_update_handle(
832 self: ArcBorrow<'_, Process>,
833 node_ref: NodeRef,
834 is_manager: bool,
835 ) -> Result<u32> {
836 {
837 let mut refs = self.node_refs.lock();
838
839 // Do a lookup before inserting.
840 if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) {
841 let handle = *handle_ref;
842 let info = refs.by_handle.get_mut(&handle).unwrap();
843 info.node_ref().absorb(node_ref);
844 return Ok(handle);
845 }
846 }
847
848 // Reserve memory for tree nodes.
849 let reserve1 = RBTreeNodeReservation::new(GFP_KERNEL)?;
850 let reserve2 = RBTreeNodeReservation::new(GFP_KERNEL)?;
851 let info = UniqueArc::new_uninit(GFP_KERNEL)?;
852
853 let mut refs_lock = self.node_refs.lock();
854 let mut refs = &mut *refs_lock;
855
856 let (unused_id, by_handle_slot) = loop {
857 // ID 0 may only be used by the manager.
858 let start = if is_manager { 0 } else { 1 };
859
860 if let Some(res) = refs.handle_is_present.find_unused_id(start) {
861 match refs.by_handle.entry(res.as_u32()) {
862 rbtree::Entry::Vacant(entry) => break (res, entry),
863 rbtree::Entry::Occupied(_) => {
864 pr_err!("Detected mismatch between handle_is_present and by_handle");
865 res.acquire();
866 kernel::warn_on!(true);
867 return Err(EINVAL);
868 }
869 }
870 }
871
872 let grow_request = refs.handle_is_present.grow_request().ok_or(ENOMEM)?;
873 drop(refs_lock);
874 let resizer = grow_request.realloc(GFP_KERNEL)?;
875 refs_lock = self.node_refs.lock();
876 refs = &mut *refs_lock;
877 refs.handle_is_present.grow(resizer);
878 };
879 let handle = unused_id.as_u32();
880
881 // Do a lookup again as node may have been inserted before the lock was reacquired.
882 let by_node_slot = match refs.by_node.entry(node_ref.node.global_id()) {
883 rbtree::Entry::Vacant(by_node_slot) => by_node_slot,
884 rbtree::Entry::Occupied(handle_ref) => {
885 // The node was inserted by another thread while we didn't hold the lock.
886 let handle = handle_ref.get();
887 let info = refs.by_handle.get_mut(handle).unwrap();
888 info.node_ref().absorb(node_ref);
889 return Ok(*handle);
890 }
891 };
892
893 let (info_proc, info_node) = {
894 let info_init = NodeRefInfo::new(node_ref, handle, self.into());
895 match info.pin_init_with(info_init) {
896 Ok(info) => ListArc::pair_from_pin_unique(info),
897 // error is infallible
898 Err(err) => match err {},
899 }
900 };
901
902 // Ensure the process is still alive while we insert a new reference.
903 //
904 // This releases the lock before inserting the nodes, but since `is_dead` is set as the
905 // first thing in `deferred_release`, process cleanup will not miss the items inserted into
906 // `refs` below.
907 if self.inner.lock().is_dead {
908 // Explicitly drop the lock so that `info_proc` and `info_node` are dropped outside of
909 // the lock.
910 drop(refs_lock);
911 return Err(ESRCH);
912 }
913
914 // SAFETY: `info_proc` and `info_node` reference the same node, so we are inserting
915 // `info_node` into the right node's `refs` list.
916 unsafe { info_proc.node_ref2().node.insert_node_info(info_node) };
917
918 by_node_slot.insert(handle, reserve1);
919 by_handle_slot.insert(info_proc, reserve2);
920 unused_id.acquire();
921 Ok(handle)
922 }
923
924 pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> {
925 // When handle is zero, try to get the context manager.
926 if handle == 0 {
927 let node_ref = self.ctx.get_manager_node(true)?;
928 if core::ptr::eq(self, &*node_ref.node.owner) {
929 return Err(EINVAL.into());
930 }
931 Ok(node_ref)
932 } else {
933 match self.get_node_from_handle(handle, true) {
934 Ok(node_ref) => Ok(node_ref),
935 Err(err) => {
936 binder_debug!(UserError, "got transaction to invalid handle {handle}");
937 Err(err.into())
938 }
939 }
940 }
941 }
942
943 pub(crate) fn get_node_from_handle(&self, handle: u32, strong: bool) -> Result<NodeRef> {
944 self.node_refs
945 .lock()
946 .by_handle
947 .get_mut(&handle)
948 .ok_or(ENOENT)?
949 .node_ref()
950 .clone(strong)
951 }
952
953 pub(crate) fn remove_from_delivered_deaths(&self, death: &DArc<NodeDeath>) {
954 let mut inner = self.inner.lock();
955 // SAFETY: By the invariant on the `delivered_links` field, this is the right linked list.
956 let removed = unsafe { inner.delivered_deaths.remove(death) };
957 drop(inner);
958 drop(removed);
959 }
960
961 pub(crate) fn update_ref(
962 self: ArcBorrow<'_, Process>,
963 handle: u32,
964 inc: bool,
965 strong: bool,
966 ) -> Result {
967 if inc && handle == 0 {
968 if let Ok(node_ref) = self.ctx.get_manager_node(strong) {
969 if core::ptr::eq(&*self, &*node_ref.node.owner) {
970 return Err(EINVAL);
971 }
972 let _ = self.insert_or_update_handle(node_ref, true);
973 return Ok(());
974 }
975 }
976
977 // To preserve original binder behaviour, we only fail requests where the manager tries to
978 // increment references on itself.
979 let _to_free_by_handle;
980 let _to_free_by_node;
981 let _to_free_freeze_listener;
982 let _to_free_freeze_listener_cleanup;
983 let mut refs = self.node_refs.lock();
984 if let Some(info) = refs.by_handle.get_mut(&handle) {
985 if info.node_ref().update(inc, strong) {
986 // Clean up death if there is one attached to this node reference.
987 //
988 // We remove the entire `info` below, so no need to remove `death` from `info`.
989 if let Some(death) = info.death().as_ref() {
990 death.set_cleared(true);
991 self.remove_from_delivered_deaths(death);
992 }
993
994 // Remove reference from process tables, and from the node's `refs` list.
995
996 // SAFETY: We are removing the `NodeRefInfo` from the right node.
997 unsafe { info.node_ref2().node.remove_node_info(info) };
998
999 let id = info.node_ref().node.global_id();
1000
1001 if let Some(freeze) = *info.freeze() {
1002 if let Some(fl) = refs.freeze_listeners.remove(&freeze) {
1003 _to_free_freeze_listener_cleanup = fl.on_process_cleanup(&self);
1004 _to_free_freeze_listener = fl;
1005 }
1006 }
1007
1008 _to_free_by_handle = refs.by_handle.remove_node(&handle);
1009 _to_free_by_node = refs.by_node.remove_node(&id);
1010 refs.handle_is_present.release_id(handle as usize);
1011
1012 if let Some(shrink) = refs.handle_is_present.shrink_request() {
1013 drop(refs);
1014 // This intentionally ignores allocation failures.
1015 if let Ok(new_bitmap) = shrink.realloc(GFP_KERNEL) {
1016 refs = self.node_refs.lock();
1017 refs.handle_is_present.shrink(new_bitmap);
1018 }
1019 }
1020 }
1021 } else {
1022 // All refs are cleared in process exit, so this warning is expected in that case.
1023 if !self.inner.lock().is_dead {
1024 binder_debug!(UserError, "no such ref {handle}");
1025 }
1026 }
1027 Ok(())
1028 }
1029
1030 /// Decrements the refcount of the given node, if one exists.
1031 pub(crate) fn update_node(&self, ptr: u64, cookie: u64, strong: bool) {
1032 let mut inner = self.inner.lock();
1033 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) {
1034 inner.update_node_refcount(&node, false, strong, 1, None);
1035 }
1036 }
1037
1038 pub(crate) fn inc_ref_done(&self, reader: &mut UserSliceReader, strong: bool) -> Result {
1039 let ptr = reader.read::<u64>()?;
1040 let cookie = reader.read::<u64>()?;
1041 let mut inner = self.inner.lock();
1042 if let Ok(Some(node)) = inner.get_existing_node(ptr, cookie) {
1043 if let Some(node) = node.inc_ref_done_locked(strong, &mut inner) {
1044 // This only fails if the process is dead.
1045 let _ = inner.push_work(self, node);
1046 }
1047 }
1048 Ok(())
1049 }
1050
1051 pub(crate) fn buffer_alloc(
1052 self: &Arc<Self>,
1053 debug_id: usize,
1054 size: usize,
1055 info: &mut TransactionInfo,
1056 ) -> BinderResult<NewAllocation> {
1057 use kernel::page::PAGE_SIZE;
1058
1059 let mut reserve_new_args = ReserveNewArgs {
1060 debug_id,
1061 size,
1062 is_oneway: info.is_oneway(),
1063 pid: info.from_pid,
1064 ..ReserveNewArgs::default()
1065 };
1066
1067 let (new_alloc, addr) = loop {
1068 let mut inner = self.inner.lock();
1069 let mapping = inner.mapping.as_mut().ok_or_else(BinderError::new_dead)?;
1070 let alloc_request = match mapping.alloc.reserve_new(reserve_new_args)? {
1071 ReserveNew::Success(new_alloc) => break (new_alloc, mapping.address),
1072 ReserveNew::NeedAlloc(request) => request,
1073 };
1074 drop(inner);
1075 // We need to allocate memory and then call `reserve_new` again.
1076 reserve_new_args = alloc_request.make_alloc()?;
1077 };
1078
1079 info.oneway_spam_suspect = new_alloc.oneway_spam_detected;
1080 let res = Allocation::new(
1081 self.clone(),
1082 debug_id,
1083 new_alloc.offset,
1084 size,
1085 addr + new_alloc.offset,
1086 );
1087
1088 // This allocation will be marked as in use until the `Allocation` is used to free it.
1089 //
1090 // This method can't be called while holding a lock, so we release the lock first. It's
1091 // okay for several threads to use the method on the same index at the same time. In that
1092 // case, one of the calls will allocate the given page (if missing), and the other call
1093 // will wait for the other call to finish allocating the page.
1094 //
1095 // We will not call `stop_using_range` in parallel with this on the same page, because the
1096 // allocation can only be removed via the destructor of the `Allocation` object that we
1097 // currently own.
1098 match self.pages.use_range(
1099 new_alloc.offset / PAGE_SIZE,
1100 (new_alloc.offset + size).div_ceil(PAGE_SIZE),
1101 ) {
1102 Ok(()) => {}
1103 Err(err) => {
1104 pr_warn!("use_range failure {:?}", err);
1105 return Err(err.into());
1106 }
1107 }
1108
1109 Ok(NewAllocation(res))
1110 }
1111
1112 pub(crate) fn buffer_get(self: &Arc<Self>, ptr: usize) -> Option<Allocation> {
1113 let mut inner = self.inner.lock();
1114 let mapping = inner.mapping.as_mut()?;
1115 let offset = ptr.checked_sub(mapping.address)?;
1116 let (size, debug_id, odata) = mapping.alloc.reserve_existing(offset).ok()?;
1117 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr);
1118 if let Some(data) = odata {
1119 alloc.set_info(data);
1120 }
1121 Some(alloc)
1122 }
1123
1124 pub(crate) fn buffer_raw_free(&self, ptr: usize) {
1125 let mut inner = self.inner.lock();
1126 if let Some(ref mut mapping) = &mut inner.mapping {
1127 let offset = match ptr.checked_sub(mapping.address) {
1128 Some(offset) => offset,
1129 None => return,
1130 };
1131
1132 let freed_range = match mapping.alloc.reservation_abort(offset) {
1133 Ok(freed_range) => freed_range,
1134 Err(_) => {
1135 pr_warn!(
1136 "Pointer {:x} failed to free, base = {:x}\n",
1137 ptr,
1138 mapping.address
1139 );
1140 return;
1141 }
1142 };
1143
1144 // No more allocations in this range. Mark them as not in use.
1145 //
1146 // Must be done before we release the lock so that `use_range` is not used on these
1147 // indices until `stop_using_range` returns.
1148 self.pages
1149 .stop_using_range(freed_range.start_page_idx, freed_range.end_page_idx);
1150 }
1151 }
1152
1153 pub(crate) fn buffer_make_freeable(&self, offset: usize, mut data: Option<AllocationInfo>) {
1154 let mut inner = self.inner.lock();
1155 if let Some(ref mut mapping) = &mut inner.mapping {
1156 if mapping.alloc.reservation_commit(offset, &mut data).is_err() {
1157 pr_warn!("Offset {} failed to be marked freeable\n", offset);
1158 }
1159 }
1160 }
1161
1162 fn create_mapping(&self, vma: &mm::virt::VmaNew) -> Result {
1163 use kernel::page::PAGE_SIZE;
1164 let size = usize::min(vma.end() - vma.start(), bindings::SZ_4M as usize);
1165 let mapping = Mapping::new(vma.start(), size);
1166 let page_count = self.pages.register_with_vma(vma)?;
1167 if page_count * PAGE_SIZE != size {
1168 return Err(EINVAL);
1169 }
1170
1171 // Save range allocator for later.
1172 self.inner.lock().mapping = Some(mapping);
1173
1174 Ok(())
1175 }
1176
1177 fn version(&self, data: UserSlice) -> Result {
1178 data.writer().write(&BinderVersion::current())
1179 }
1180
1181 pub(crate) fn register_thread(&self) -> bool {
1182 self.inner.lock().register_thread()
1183 }
1184
1185 fn remove_thread(&self, thread: Arc<Thread>) {
1186 self.inner.lock().threads.remove(&thread.id);
1187 thread.release();
1188 }
1189
1190 fn set_max_threads(&self, max: u32) {
1191 self.inner.lock().max_threads = max;
1192 }
1193
1194 fn set_oneway_spam_detection_enabled(&self, enabled: u32) {
1195 self.inner.lock().oneway_spam_detection_enabled = enabled != 0;
1196 }
1197
1198 pub(crate) fn is_oneway_spam_detection_enabled(&self) -> bool {
1199 self.inner.lock().oneway_spam_detection_enabled
1200 }
1201
1202 fn get_node_debug_info(&self, data: UserSlice) -> Result {
1203 let (mut reader, mut writer) = data.reader_writer();
1204
1205 // Read the starting point.
1206 let ptr = reader.read::<BinderNodeDebugInfo>()?.ptr;
1207 let mut out = BinderNodeDebugInfo::default();
1208
1209 {
1210 let inner = self.inner.lock();
1211 for (node_ptr, node) in &inner.nodes {
1212 if *node_ptr > ptr {
1213 node.populate_debug_info(&mut out, &inner);
1214 break;
1215 }
1216 }
1217 }
1218
1219 writer.write(&out)
1220 }
1221
1222 fn get_node_info_from_ref(&self, data: UserSlice) -> Result {
1223 let (mut reader, mut writer) = data.reader_writer();
1224 let mut out = reader.read::<BinderNodeInfoForRef>()?;
1225
1226 if out.strong_count != 0
1227 || out.weak_count != 0
1228 || out.reserved1 != 0
1229 || out.reserved2 != 0
1230 || out.reserved3 != 0
1231 {
1232 return Err(EINVAL);
1233 }
1234
1235 // Only the context manager is allowed to use this ioctl.
1236 if !self.inner.lock().is_manager {
1237 return Err(EPERM);
1238 }
1239
1240 {
1241 let mut node_refs = self.node_refs.lock();
1242 let node_info = node_refs.by_handle.get_mut(&out.handle).ok_or(ENOENT)?;
1243 let node_ref = node_info.node_ref();
1244 let owner_inner = node_ref.node.owner.inner.lock();
1245 node_ref.node.populate_counts(&mut out, &owner_inner);
1246 }
1247
1248 // Write the result back.
1249 writer.write(&out)
1250 }
1251
1252 pub(crate) fn needs_thread(&self) -> bool {
1253 let mut inner = self.inner.lock();
1254 let ret = inner.requested_thread_count == 0
1255 && inner.ready_threads.is_empty()
1256 && inner.started_thread_count < inner.max_threads;
1257 if ret {
1258 inner.requested_thread_count += 1
1259 }
1260 ret
1261 }
1262
1263 pub(crate) fn request_death(
1264 self: &Arc<Self>,
1265 reader: &mut UserSliceReader,
1266 thread: &Thread,
1267 ) -> Result {
1268 let handle: u32 = reader.read()?;
1269 let cookie: u64 = reader.read()?;
1270
1271 // Queue BR_ERROR if we can't allocate memory for the death notification.
1272 let death = UniqueArc::new_uninit(GFP_KERNEL).inspect_err(|_| {
1273 thread.push_return_work(BR_ERROR);
1274 binder_debug!(
1275 DeathNotification,
1276 "BC_REQUEST_DEATH_NOTIFICATION failed due to memory allocation failure"
1277 );
1278 })?;
1279 let mut refs = self.node_refs.lock();
1280 let Some(info) = refs.by_handle.get_mut(&handle) else {
1281 binder_debug!(
1282 UserError,
1283 "BC_REQUEST_DEATH_NOTIFICATION invalid ref {handle}"
1284 );
1285 return Ok(());
1286 };
1287
1288 // Nothing to do if there is already a death notification request for this handle.
1289 if info.death().is_some() {
1290 binder_debug!(
1291 UserError,
1292 "BC_REQUEST_DEATH_NOTIFICATION death notification already set"
1293 );
1294 return Ok(());
1295 }
1296
1297 let death = {
1298 let death_init = NodeDeath::new(info.node_ref().node.clone(), self.clone(), cookie);
1299 match death.pin_init_with(death_init) {
1300 Ok(death) => death,
1301 // error is infallible
1302 Err(err) => match err {},
1303 }
1304 };
1305
1306 // Register the death notification.
1307 {
1308 let owner = info.node_ref2().node.owner.clone();
1309 let mut owner_inner = owner.inner.lock();
1310 if owner_inner.is_dead {
1311 let death = Arc::from(death);
1312 *info.death() = Some(death.clone());
1313 drop(owner_inner);
1314 death.set_dead();
1315 } else {
1316 let death = ListArc::from(death);
1317 *info.death() = Some(death.clone_arc());
1318 info.node_ref().node.add_death(death, &mut owner_inner);
1319 }
1320 }
1321 binder_debug!(
1322 DeathNotification,
1323 "BC_REQUEST_DEATH_NOTIFICATION handle {handle} cookie {:016x}",
1324 cookie
1325 );
1326 Ok(())
1327 }
1328
1329 pub(crate) fn clear_death(&self, reader: &mut UserSliceReader, thread: &Thread) -> Result {
1330 let handle: u32 = reader.read()?;
1331 let cookie: u64 = reader.read()?;
1332
1333 let mut refs = self.node_refs.lock();
1334 let Some(info) = refs.by_handle.get_mut(&handle) else {
1335 binder_debug!(
1336 UserError,
1337 "BC_CLEAR_DEATH_NOTIFICATION invalid ref {handle}"
1338 );
1339 return Ok(());
1340 };
1341
1342 let Some(death) = info.death().take() else {
1343 binder_debug!(
1344 UserError,
1345 "BC_CLEAR_DEATH_NOTIFICATION death notification not active"
1346 );
1347 return Ok(());
1348 };
1349 if death.cookie != cookie {
1350 *info.death() = Some(death);
1351 binder_debug!(
1352 UserError,
1353 "BC_CLEAR_DEATH_NOTIFICATION death notification cookie mismatch"
1354 );
1355 return Ok(());
1356 }
1357
1358 // Update state and determine if we need to queue a work item. We only need to do it when
1359 // the node is not dead or if the user already completed the death notification.
1360 let should_schedule = death.set_cleared(false);
1361 drop(refs);
1362
1363 if should_schedule {
1364 if let Some(death) = ListArc::try_from_arc_or_drop(death) {
1365 let _ = thread.push_work_if_looper(death);
1366 }
1367 }
1368
1369 binder_debug!(
1370 DeathNotification,
1371 "BC_CLEAR_DEATH_NOTIFICATION handle {handle} cookie {:016x}",
1372 cookie
1373 );
1374 Ok(())
1375 }
1376
1377 pub(crate) fn dead_binder_done(&self, cookie: u64, thread: &Thread) {
1378 let death = self.inner.lock().pull_delivered_death(cookie);
1379 if let Some(death) = death {
1380 death.set_notification_done(thread);
1381 }
1382 }
1383
1384 /// Locks the spinlock and move the `nodes` rbtree out.
1385 ///
1386 /// This allows you to iterate through `nodes` while also allowing you to give other parts of
1387 /// the codebase exclusive access to `ProcessInner`.
1388 pub(crate) fn lock_with_nodes(&self) -> WithNodes<'_> {
1389 let mut inner = self.inner.lock();
1390 WithNodes {
1391 nodes: take(&mut inner.nodes),
1392 inner,
1393 }
1394 }
1395
1396 fn deferred_flush(&self) {
1397 binder_debug!(pid = self.task.pid(), OpenClose, "flushing process");
1398 let inner = self.inner.lock();
1399 for thread in inner.threads.values() {
1400 thread.exit_looper();
1401 }
1402 }
1403
1404 fn deferred_release(self: Arc<Self>) {
1405 binder_debug!(pid = self.task.pid(), OpenClose, "releasing process");
1406
1407 let is_manager = {
1408 let mut inner = self.inner.lock();
1409 inner.is_dead = true;
1410 inner.is_frozen = IsFrozen::No;
1411 inner.sync_recv = false;
1412 inner.async_recv = false;
1413 inner.is_manager
1414 };
1415
1416 if is_manager {
1417 self.ctx.unset_manager_node();
1418 }
1419
1420 self.ctx.deregister_process(&self);
1421
1422 let binderfs_file = self.inner.lock().binderfs_file.take();
1423 drop(binderfs_file);
1424
1425 // Release threads.
1426 let threads = {
1427 let mut inner = self.inner.lock();
1428 let threads = take(&mut inner.threads);
1429 let ready = take(&mut inner.ready_threads);
1430 drop(inner);
1431 drop(ready);
1432
1433 for thread in threads.values() {
1434 thread.release();
1435 }
1436 threads
1437 };
1438
1439 // Release nodes.
1440 {
1441 while let Some(node) = {
1442 let mut lock = self.inner.lock();
1443 lock.nodes.cursor_front_mut().map(|c| c.remove_current().1)
1444 } {
1445 node.to_key_value().1.release();
1446 }
1447 }
1448
1449 // Clean up death listeners and remove nodes from external node info lists.
1450 for info in self.node_refs.lock().by_handle.values_mut() {
1451 // SAFETY: We are removing the `NodeRefInfo` from the right node.
1452 unsafe { info.node_ref2().node.remove_node_info(info) };
1453
1454 // Clear death notifications from the nodes (that belong to a different process).
1455 // No need to remove them from `info` as we clear info below.
1456 if let Some(death) = info.death().as_ref() {
1457 death.set_cleared(false);
1458 }
1459 }
1460
1461 // Clean up freeze listeners.
1462 let freeze_listeners = take(&mut self.node_refs.lock().freeze_listeners);
1463 for listener in freeze_listeners.values() {
1464 listener.on_process_cleanup(&self);
1465 }
1466 drop(freeze_listeners);
1467
1468 // Release refs on foreign nodes.
1469 {
1470 let mut refs = self.node_refs.lock();
1471 let by_handle = take(&mut refs.by_handle);
1472 let by_node = take(&mut refs.by_node);
1473 drop(refs);
1474 drop(by_node);
1475 drop(by_handle);
1476 }
1477
1478 // Cancel all pending work items.
1479 while let Some(work) = self.get_work() {
1480 work.into_arc().cancel();
1481 }
1482
1483 // Clear delivered_deaths list.
1484 //
1485 // Scope ensures that MutexGuard is dropped while executing the body.
1486 while let Some(delivered_death) = {
1487 // Explicitly bind to avoid tail expression lifetime extension of the lockguard
1488 // Can be removed when the kernel moves to edition 2024
1489 let maybe_death = self.inner.lock().delivered_deaths.pop_front();
1490 maybe_death
1491 } {
1492 drop(delivered_death);
1493 }
1494
1495 // Free any resources kept alive by allocated buffers.
1496 let omapping = self.inner.lock().mapping.take();
1497 if let Some(mut mapping) = omapping {
1498 let address = mapping.address;
1499 mapping
1500 .alloc
1501 .take_for_each(|offset, size, debug_id, odata| {
1502 let ptr = offset + address;
1503 let mut alloc = Allocation::new(self.clone(), debug_id, offset, size, ptr);
1504 if let Some(data) = odata {
1505 alloc.set_info(data);
1506 }
1507 drop(alloc)
1508 });
1509 }
1510
1511 // calls to synchronize_rcu() in thread drop will happen here
1512 drop(threads);
1513 }
1514
1515 pub(crate) fn drop_outstanding_txn(&self) {
1516 let wake = {
1517 let mut inner = self.inner.lock();
1518 if inner.outstanding_txns == 0 {
1519 pr_err!("outstanding_txns underflow");
1520 return;
1521 }
1522 inner.outstanding_txns -= 1;
1523 inner.is_frozen.is_frozen() && inner.outstanding_txns == 0
1524 };
1525
1526 if wake {
1527 self.freeze_wait.notify_all();
1528 }
1529 }
1530
1531 // #[export_name] is a temporary workaround so that ps output does not become unreadable from
1532 // mangled symbol names.
1533 #[export_name = "rust_binder_freeze"]
1534 pub(crate) fn ioctl_freeze(&self, info: &BinderFreezeInfo) -> Result {
1535 if info.enable == 0 {
1536 let msgs = self.prepare_freeze_messages()?;
1537 let mut inner = self.inner.lock();
1538 inner.sync_recv = false;
1539 inner.async_recv = false;
1540 inner.is_frozen = IsFrozen::No;
1541 drop(inner);
1542 msgs.send_messages();
1543 return Ok(());
1544 }
1545
1546 let mut inner = self.inner.lock();
1547 inner.sync_recv = false;
1548 inner.async_recv = false;
1549 inner.is_frozen = IsFrozen::InProgress;
1550
1551 if info.timeout_ms > 0 {
1552 let mut jiffies = kernel::time::msecs_to_jiffies(info.timeout_ms);
1553 while jiffies > 0 {
1554 if inner.outstanding_txns == 0 {
1555 break;
1556 }
1557
1558 match self
1559 .freeze_wait
1560 .wait_interruptible_timeout(&mut inner, jiffies)
1561 {
1562 CondVarTimeoutResult::Signal { .. } => {
1563 inner.is_frozen = IsFrozen::No;
1564 return Err(ERESTARTSYS);
1565 }
1566 CondVarTimeoutResult::Woken { jiffies: remaining } => {
1567 jiffies = remaining;
1568 }
1569 CondVarTimeoutResult::Timeout => {
1570 jiffies = 0;
1571 }
1572 }
1573 }
1574 }
1575
1576 if inner.txns_pending_locked() {
1577 inner.is_frozen = IsFrozen::No;
1578 Err(EAGAIN)
1579 } else {
1580 drop(inner);
1581 match self.prepare_freeze_messages() {
1582 Ok(batch) => {
1583 self.inner.lock().is_frozen = IsFrozen::Yes;
1584 batch.send_messages();
1585 Ok(())
1586 }
1587 Err(kernel::alloc::AllocError) => {
1588 self.inner.lock().is_frozen = IsFrozen::No;
1589 Err(ENOMEM)
1590 }
1591 }
1592 }
1593 }
1594
1595 pub(crate) fn notify_poll(&self, sync: bool) {
1596 if let Some(poll) = self.poll.as_ref() {
1597 if sync {
1598 poll.notify_sync();
1599 }
1600 poll.notify_all();
1601 }
1602 }
1603}
1604
1605fn get_frozen_status(data: UserSlice) -> Result {
1606 let (mut reader, mut writer) = data.reader_writer();
1607
1608 let mut info = reader.read::<BinderFrozenStatusInfo>()?;
1609 info.sync_recv = 0;
1610 info.async_recv = 0;
1611 let mut found = false;
1612
1613 for ctx in crate::context::get_all_contexts()? {
1614 ctx.for_each_proc(|proc| {
1615 if proc.task.pid() == info.pid as Pid {
1616 found = true;
1617 let inner = proc.inner.lock();
1618 let txns_pending = inner.txns_pending_locked();
1619 info.async_recv |= u32::from(inner.async_recv);
1620 info.sync_recv |= u32::from(inner.sync_recv);
1621 info.sync_recv |= u32::from(txns_pending) << 1;
1622 }
1623 });
1624 }
1625
1626 if found {
1627 writer.write(&info)?;
1628 Ok(())
1629 } else {
1630 Err(EINVAL)
1631 }
1632}
1633
1634fn ioctl_freeze(reader: &mut UserSliceReader) -> Result {
1635 let info = reader.read::<BinderFreezeInfo>()?;
1636
1637 // Very unlikely for there to be more than 3, since a process normally uses at most binder and
1638 // hwbinder.
1639 let mut procs = KVec::with_capacity(3, GFP_KERNEL)?;
1640
1641 let ctxs = crate::context::get_all_contexts()?;
1642 for ctx in ctxs {
1643 for proc in ctx.get_procs_with_pid(info.pid as i32)? {
1644 procs.push(proc, GFP_KERNEL)?;
1645 }
1646 }
1647
1648 for proc in procs {
1649 proc.ioctl_freeze(&info)?;
1650 }
1651 Ok(())
1652}
1653
1654/// The ioctl handler.
1655impl Process {
1656 /// Ioctls that are write-only from the perspective of userspace.
1657 ///
1658 /// The kernel will only read from the pointer that userspace provided to us.
1659 fn ioctl_write_only(
1660 this: ArcBorrow<'_, Process>,
1661 _file: &File,
1662 cmd: u32,
1663 reader: &mut UserSliceReader,
1664 ) -> Result {
1665 if cmd == uapi::BINDER_FREEZE {
1666 return ioctl_freeze(reader);
1667 }
1668
1669 let thread = this.get_current_thread()?;
1670 match cmd {
1671 uapi::BINDER_SET_MAX_THREADS => this.set_max_threads(reader.read()?),
1672 uapi::BINDER_THREAD_EXIT => this.remove_thread(thread),
1673 uapi::BINDER_SET_CONTEXT_MGR => this.set_as_manager(None, &thread)?,
1674 uapi::BINDER_SET_CONTEXT_MGR_EXT => {
1675 this.set_as_manager(Some(reader.read()?), &thread)?
1676 }
1677 uapi::BINDER_ENABLE_ONEWAY_SPAM_DETECTION => {
1678 this.set_oneway_spam_detection_enabled(reader.read()?)
1679 }
1680 _ => return Err(EINVAL),
1681 }
1682 Ok(())
1683 }
1684
1685 /// Ioctls that are read/write from the perspective of userspace.
1686 ///
1687 /// The kernel will both read from and write to the pointer that userspace provided to us.
1688 fn ioctl_write_read(
1689 this: ArcBorrow<'_, Process>,
1690 file: &File,
1691 cmd: u32,
1692 data: UserSlice,
1693 ) -> Result {
1694 let blocking = (file.flags() & file::flags::O_NONBLOCK) == 0;
1695 match cmd {
1696 uapi::BINDER_WRITE_READ => this.get_current_thread()?.write_read(data, blocking)?,
1697 uapi::BINDER_GET_NODE_DEBUG_INFO => this.get_node_debug_info(data)?,
1698 uapi::BINDER_GET_NODE_INFO_FOR_REF => this.get_node_info_from_ref(data)?,
1699 uapi::BINDER_VERSION => this.version(data)?,
1700 uapi::BINDER_GET_FROZEN_INFO => get_frozen_status(data)?,
1701 uapi::BINDER_GET_EXTENDED_ERROR => {
1702 this.get_current_thread()?.get_extended_error(data)?
1703 }
1704 _ => return Err(EINVAL),
1705 }
1706 Ok(())
1707 }
1708}
1709
1710/// The file operations supported by `Process`.
1711impl Process {
1712 pub(crate) fn open(ctx: ArcBorrow<'_, Context>, file: &File) -> Result<Arc<Process>> {
1713 let proc = Self::new(ctx.into(), ARef::from(file.cred()))?;
1714 binder_debug!(OpenClose, "opened process");
1715 Ok(proc)
1716 }
1717
1718 pub(crate) fn release(this: Arc<Process>, _file: &File) {
1719 let binderfs_file;
1720 let should_schedule;
1721 {
1722 let mut inner = this.inner.lock();
1723 should_schedule = inner.defer_work == DeferWorks::empty();
1724 inner.defer_work |= DeferWork::Release;
1725 binderfs_file = inner.binderfs_file.take();
1726 }
1727
1728 if should_schedule {
1729 // Ignore failures to schedule to the workqueue. Those just mean that we're already
1730 // scheduled for execution.
1731 let _ = workqueue::system().enqueue(this);
1732 }
1733
1734 drop(binderfs_file);
1735 }
1736
1737 pub(crate) fn flush(this: ArcBorrow<'_, Process>) -> Result {
1738 let should_schedule;
1739 {
1740 let mut inner = this.inner.lock();
1741 should_schedule = inner.defer_work == DeferWorks::empty();
1742 inner.defer_work |= DeferWork::Flush;
1743 }
1744
1745 if should_schedule {
1746 // Ignore failures to schedule to the workqueue. Those just mean that we're already
1747 // scheduled for execution.
1748 let _ = workqueue::system().enqueue(Arc::from(this));
1749 }
1750 Ok(())
1751 }
1752
1753 pub(crate) fn ioctl(this: ArcBorrow<'_, Process>, file: &File, cmd: u32, arg: usize) -> Result {
1754 use kernel::ioctl::{_IOC_DIR, _IOC_SIZE};
1755 use kernel::uapi::{_IOC_READ, _IOC_WRITE};
1756
1757 crate::trace::trace_ioctl(cmd, arg);
1758
1759 let user_slice = UserSlice::new(UserPtr::from_addr(arg), _IOC_SIZE(cmd));
1760
1761 const _IOC_READ_WRITE: u32 = _IOC_READ | _IOC_WRITE;
1762
1763 let res = match _IOC_DIR(cmd) {
1764 _IOC_WRITE => Self::ioctl_write_only(this, file, cmd, &mut user_slice.reader()),
1765 _IOC_READ_WRITE => Self::ioctl_write_read(this, file, cmd, user_slice),
1766 _ => Err(EINVAL),
1767 };
1768
1769 crate::trace::trace_ioctl_done(res);
1770 res
1771 }
1772
1773 pub(crate) fn mmap(
1774 this: ArcBorrow<'_, Process>,
1775 _file: &File,
1776 vma: &mm::virt::VmaNew,
1777 ) -> Result {
1778 // We don't allow mmap to be used in a different process.
1779 if this.task != kernel::current!().group_leader() {
1780 return Err(EINVAL);
1781 }
1782 if vma.start() == 0 {
1783 return Err(EINVAL);
1784 }
1785
1786 vma.try_clear_maywrite().map_err(|_| EPERM)?;
1787 vma.set_dontcopy();
1788 vma.set_mixedmap();
1789
1790 // TODO: Set ops. We need to learn when the user unmaps so that we can stop using it.
1791 this.create_mapping(vma)
1792 }
1793
1794 pub(crate) fn poll(
1795 this: ArcBorrow<'_, Process>,
1796 file: &File,
1797 table: PollTable<'_>,
1798 ) -> Result<u32> {
1799 let thread = this.get_current_thread()?;
1800 {
1801 let poll = loop {
1802 if let Some(poll) = this.poll.as_ref() {
1803 break poll;
1804 }
1805
1806 let poll = PollCondVarBox::new(c"Process::poll", kernel::static_lock_class!())?;
1807 // Reuse our existing lock to synchronize callers initializing.
1808 let _guard = this.node_refs.lock();
1809 this.poll.populate(poll);
1810 };
1811
1812 table.register_wait(file, poll);
1813 }
1814 let (from_proc, mut mask) = thread.poll()?;
1815 if mask == 0 && from_proc && !this.inner.lock().work.is_empty() {
1816 mask |= bindings::POLLIN;
1817 }
1818 Ok(mask)
1819 }
1820}
1821
1822/// Represents that a thread has registered with the `ready_threads` list of its process.
1823///
1824/// The destructor of this type will unregister the thread from the list of ready threads.
1825pub(crate) struct Registration<'a> {
1826 thread: &'a Arc<Thread>,
1827}
1828
1829impl<'a> Registration<'a> {
1830 fn new(thread: &'a Arc<Thread>, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>) -> Self {
1831 assert!(core::ptr::eq(&thread.process.inner, guard.lock_ref()));
1832 // INVARIANT: We are pushing this thread to the right `ready_threads` list.
1833 if let Ok(list_arc) = ListArc::try_from_arc(thread.clone()) {
1834 guard.ready_threads.push_front(list_arc);
1835 } else {
1836 // It is an error to hit this branch, and it should not be reachable. We try to do
1837 // something reasonable when the failure path happens. Most likely, the thread in
1838 // question will sleep forever.
1839 pr_err!("Same thread registered with `ready_threads` twice.");
1840 }
1841 Self { thread }
1842 }
1843}
1844
1845impl Drop for Registration<'_> {
1846 fn drop(&mut self) {
1847 let mut inner = self.thread.process.inner.lock();
1848 // SAFETY: The thread has the invariant that we never push it to any other linked list than
1849 // the `ready_threads` list of its parent process. Therefore, the thread is either in that
1850 // list, or in no list.
1851 unsafe { inner.ready_threads.remove(self.thread) };
1852 }
1853}
1854
1855pub(crate) struct WithNodes<'a> {
1856 pub(crate) inner: Guard<'a, ProcessInner, SpinLockBackend>,
1857 pub(crate) nodes: RBTree<u64, DArc<Node>>,
1858}
1859
1860impl Drop for WithNodes<'_> {
1861 fn drop(&mut self) {
1862 core::mem::swap(&mut self.nodes, &mut self.inner.nodes);
1863 if self.nodes.iter().next().is_some() {
1864 pr_err!("nodes array was modified while using lock_with_nodes\n");
1865 }
1866 }
1867}
1868
1869pub(crate) enum GetWorkOrRegister<'a> {
1870 Work(DLArc<dyn DeliverToRead>),
1871 Register(Registration<'a>),
1872}