projects/linux·blob master · refs

1174 lines · 40.2 KiB

1// SPDX-License-Identifier: GPL-2.0
2
3// Copyright (C) 2025 Google LLC.
4
5use kernel::{
6 list::{AtomicTracker, List, ListArc, ListLinks, TryNewListArc},
7 prelude::*,
8 seq_file::SeqFile,
9 seq_print,
10 sync::lock::{spinlock::SpinLockBackend, Guard},
11 sync::{Arc, LockedBy, SpinLock},
12 uapi,
13};
14
15use crate::{
16 defs::*,
17 error::BinderError,
18 process::{NodeRefInfo, Process, ProcessInner},
19 thread::Thread,
20 transaction::Transaction,
21 BinderReturnWriter, DArc, DLArc, DTRWrap, DeliverToRead,
22};
23
24use core::mem;
25use core::ptr;
26
27mod wrapper;
28pub(crate) use self::wrapper::CritIncrWrapper;
29
30#[derive(Debug)]
31pub(crate) struct CouldNotDeliverCriticalIncrement;
32
33/// Keeps track of how this node is scheduled.
34///
35/// There are two ways to schedule a node to a work list. Just schedule the node itself, or
36/// allocate a wrapper that references the node and schedule the wrapper. These wrappers exists to
37/// make it possible to "move" a node from one list to another - when `do_work` is called directly
38/// on the `Node`, then it's a no-op if there's also a pending wrapper.
39///
40/// Wrappers are generally only needed for zero-to-one refcount increments, and there are two cases
41/// of this: weak increments and strong increments. We call such increments "critical" because it
42/// is critical that they are delivered to the thread doing the increment. Some examples:
43///
44/// * One thread makes a zero-to-one strong increment, and another thread makes a zero-to-one weak
45/// increment. Delivering the node to the thread doing the weak increment is wrong, since the
46/// thread doing the strong increment may have ended a long time ago when the command is actually
47/// processed by userspace.
48///
49/// * We have a weak reference and are about to drop it on one thread. But then another thread does
50/// a zero-to-one strong increment. If the strong increment gets sent to the thread that was
51/// about to drop the weak reference, then the strong increment could be processed after the
52/// other thread has already exited, which would be too late.
53///
54/// Note that trying to create a `ListArc` to the node can succeed even if `has_pushed_node` is
55/// set. This is because another thread might just have popped the node from a todo list, but not
56/// yet called `do_work`. However, if `has_pushed_node` is false, then creating a `ListArc` should
57/// always succeed.
58///
59/// Like the other fields in `NodeInner`, the delivery state is protected by the process lock.
60struct DeliveryState {
61 /// Is the `Node` currently scheduled?
62 has_pushed_node: bool,
63
64 /// Is a wrapper currently scheduled?
65 ///
66 /// The wrapper is used only for strong zero2one increments.
67 has_pushed_wrapper: bool,
68
69 /// Is the currently scheduled `Node` scheduled due to a weak zero2one increment?
70 ///
71 /// Weak zero2one operations are always scheduled using the `Node`.
72 has_weak_zero2one: bool,
73
74 /// Is the currently scheduled wrapper/`Node` scheduled due to a strong zero2one increment?
75 ///
76 /// If `has_pushed_wrapper` is set, then the strong zero2one increment was scheduled using the
77 /// wrapper. Otherwise, `has_pushed_node` must be set and it was scheduled using the `Node`.
78 has_strong_zero2one: bool,
79}
80
81impl DeliveryState {
82 fn should_normal_push(&self) -> bool {
83 !self.has_pushed_node && !self.has_pushed_wrapper
84 }
85
86 fn did_normal_push(&mut self) {
87 assert!(self.should_normal_push());
88 self.has_pushed_node = true;
89 }
90
91 fn should_push_weak_zero2one(&self) -> bool {
92 !self.has_weak_zero2one && !self.has_strong_zero2one
93 }
94
95 fn can_push_weak_zero2one_normally(&self) -> bool {
96 !self.has_pushed_node
97 }
98
99 fn did_push_weak_zero2one(&mut self) {
100 assert!(self.should_push_weak_zero2one());
101 assert!(self.can_push_weak_zero2one_normally());
102 self.has_pushed_node = true;
103 self.has_weak_zero2one = true;
104 }
105
106 fn should_push_strong_zero2one(&self) -> bool {
107 !self.has_strong_zero2one
108 }
109
110 fn can_push_strong_zero2one_normally(&self) -> bool {
111 !self.has_pushed_node
112 }
113
114 fn did_push_strong_zero2one(&mut self) {
115 assert!(self.should_push_strong_zero2one());
116 assert!(self.can_push_strong_zero2one_normally());
117 self.has_pushed_node = true;
118 self.has_strong_zero2one = true;
119 }
120
121 fn did_push_strong_zero2one_wrapper(&mut self) {
122 assert!(self.should_push_strong_zero2one());
123 assert!(!self.can_push_strong_zero2one_normally());
124 self.has_pushed_wrapper = true;
125 self.has_strong_zero2one = true;
126 }
127}
128
129struct CountState {
130 /// The reference count.
131 count: usize,
132 /// Whether the process that owns this node thinks that we hold a refcount on it. (Note that
133 /// even if count is greater than one, we only increment it once in the owning process.)
134 has_count: bool,
135}
136
137impl CountState {
138 fn new() -> Self {
139 Self {
140 count: 0,
141 has_count: false,
142 }
143 }
144}
145
146struct NodeInner {
147 /// Strong refcounts held on this node by `NodeRef` objects.
148 strong: CountState,
149 /// Weak refcounts held on this node by `NodeRef` objects.
150 weak: CountState,
151 delivery_state: DeliveryState,
152 /// The binder driver guarantees that oneway transactions sent to the same node are serialized,
153 /// that is, userspace will not be given the next one until it has finished processing the
154 /// previous oneway transaction. This is done to avoid the case where two oneway transactions
155 /// arrive in opposite order from the order in which they were sent. (E.g., they could be
156 /// delivered to two different threads, which could appear as-if they were sent in opposite
157 /// order.)
158 ///
159 /// To fix that, we store pending oneway transactions in a separate list in the node, and don't
160 /// deliver the next oneway transaction until userspace signals that it has finished processing
161 /// the previous oneway transaction by calling the `BC_FREE_BUFFER` ioctl.
162 oneway_todo: List<DTRWrap<Transaction>>,
163 /// Keeps track of whether this node has a pending oneway transaction.
164 ///
165 /// When this is true, incoming oneway transactions are stored in `oneway_todo`, instead of
166 /// being delivered directly to the process.
167 has_oneway_transaction: bool,
168 /// List of processes to deliver a notification to when this node is destroyed (usually due to
169 /// the process dying).
170 death_list: List<DTRWrap<NodeDeath>, 1>,
171 /// List of processes to deliver freeze notifications to.
172 freeze_list: KVVec<Arc<Process>>,
173 /// The number of active BR_INCREFS or BR_ACQUIRE operations. (should be maximum two)
174 ///
175 /// If this is non-zero, then we postpone any BR_RELEASE or BR_DECREFS notifications until the
176 /// active operations have ended. This avoids the situation an increment and decrement get
177 /// reordered from userspace's perspective.
178 active_inc_refs: u8,
179 /// List of `NodeRefInfo` objects that reference this node.
180 refs: List<NodeRefInfo, { NodeRefInfo::LIST_NODE }>,
181}
182
183use kernel::bindings::rb_node_layout;
184use mem::offset_of;
185pub(crate) const NODE_LAYOUT: rb_node_layout = rb_node_layout {
186 arc_offset: Arc::<Node>::DATA_OFFSET + offset_of!(DTRWrap<Node>, wrapped),
187 debug_id: offset_of!(Node, debug_id),
188 ptr: offset_of!(Node, ptr),
189};
190
191#[pin_data]
192pub(crate) struct Node {
193 pub(crate) debug_id: usize,
194 ptr: u64,
195 pub(crate) cookie: u64,
196 pub(crate) flags: u32,
197 pub(crate) owner: Arc<Process>,
198 inner: LockedBy<NodeInner, ProcessInner>,
199 #[pin]
200 links_track: AtomicTracker,
201}
202
203kernel::list::impl_list_arc_safe! {
204 impl ListArcSafe<0> for Node {
205 tracked_by links_track: AtomicTracker;
206 }
207}
208
209// Make `oneway_todo` work.
210kernel::list::impl_list_item! {
211 impl ListItem<0> for DTRWrap<Transaction> {
212 using ListLinks { self.links.inner };
213 }
214}
215
216impl Node {
217 pub(crate) fn new(
218 ptr: u64,
219 cookie: u64,
220 flags: u32,
221 owner: Arc<Process>,
222 ) -> impl PinInit<Self> {
223 pin_init!(Self {
224 inner: LockedBy::new(
225 &owner.inner,
226 NodeInner {
227 strong: CountState::new(),
228 weak: CountState::new(),
229 delivery_state: DeliveryState {
230 has_pushed_node: false,
231 has_pushed_wrapper: false,
232 has_weak_zero2one: false,
233 has_strong_zero2one: false,
234 },
235 death_list: List::new(),
236 oneway_todo: List::new(),
237 freeze_list: KVVec::new(),
238 has_oneway_transaction: false,
239 active_inc_refs: 0,
240 refs: List::new(),
241 },
242 ),
243 debug_id: super::next_debug_id(),
244 ptr,
245 cookie,
246 flags,
247 owner,
248 links_track <- AtomicTracker::new(),
249 })
250 }
251
252 pub(crate) fn has_oneway_transaction(&self, owner_inner: &mut ProcessInner) -> bool {
253 let inner = self.inner.access_mut(owner_inner);
254 inner.has_oneway_transaction
255 }
256
257 #[inline(never)]
258 pub(crate) fn full_debug_print(
259 &self,
260 m: &SeqFile,
261 owner_inner: &mut ProcessInner,
262 ) -> Result<()> {
263 let inner = self.inner.access_mut(owner_inner);
264 seq_print!(
265 m,
266 " node {}: u{:016x} c{:016x} hs {} hw {} cs {} cw {}",
267 self.debug_id,
268 self.ptr,
269 self.cookie,
270 inner.strong.has_count,
271 inner.weak.has_count,
272 inner.strong.count,
273 inner.weak.count,
274 );
275 if !inner.refs.is_empty() {
276 seq_print!(m, " proc");
277 for node_ref in &inner.refs {
278 seq_print!(m, " {}", node_ref.process.task.pid());
279 }
280 }
281 seq_print!(m, "\n");
282 for t in &inner.oneway_todo {
283 t.debug_print_inner(m, " pending async transaction ");
284 }
285 Ok(())
286 }
287
288 /// Insert the `NodeRef` into this `refs` list.
289 ///
290 /// # Safety
291 ///
292 /// It must be the case that `info.node_ref.node` is this node.
293 pub(crate) unsafe fn insert_node_info(
294 &self,
295 info: ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>,
296 ) {
297 self.inner
298 .access_mut(&mut self.owner.inner.lock())
299 .refs
300 .push_front(info);
301 }
302
303 /// Insert the `NodeRef` into this `refs` list.
304 ///
305 /// # Safety
306 ///
307 /// It must be the case that `info.node_ref.node` is this node.
308 pub(crate) unsafe fn remove_node_info(
309 &self,
310 info: &NodeRefInfo,
311 ) -> Option<ListArc<NodeRefInfo, { NodeRefInfo::LIST_NODE }>> {
312 // SAFETY: We always insert `NodeRefInfo` objects into the `refs` list of the node that it
313 // references in `info.node_ref.node`. That is this node, so `info` cannot possibly be in
314 // the `refs` list of another node.
315 unsafe {
316 self.inner
317 .access_mut(&mut self.owner.inner.lock())
318 .refs
319 .remove(info)
320 }
321 }
322
323 /// An id that is unique across all binder nodes on the system. Used as the key in the
324 /// `by_node` map.
325 pub(crate) fn global_id(&self) -> usize {
326 ptr::from_ref(self).addr()
327 }
328
329 pub(crate) fn get_id(&self) -> (u64, u64) {
330 (self.ptr, self.cookie)
331 }
332
333 pub(crate) fn add_death(
334 &self,
335 death: ListArc<DTRWrap<NodeDeath>, 1>,
336 guard: &mut Guard<'_, ProcessInner, SpinLockBackend>,
337 ) {
338 assert!(
339 core::ptr::eq(self, &**death.node),
340 "attempt to add NodeDeath to the wrong death list"
341 );
342 self.inner.access_mut(guard).death_list.push_back(death);
343 }
344
345 pub(crate) fn inc_ref_done_locked(
346 self: &DArc<Node>,
347 _strong: bool,
348 owner_inner: &mut ProcessInner,
349 ) -> Option<DLArc<Node>> {
350 let inner = self.inner.access_mut(owner_inner);
351 if inner.active_inc_refs == 0 {
352 binder_debug!(UserError, "inc_ref_done called when no active inc_refs");
353 return None;
354 }
355
356 inner.active_inc_refs -= 1;
357 if inner.active_inc_refs == 0 {
358 // Having active inc_refs can inhibit dropping of ref-counts. Calculate whether we
359 // would send a refcount decrement, and if so, tell the caller to schedule us.
360 let strong = inner.strong.count > 0;
361 let has_strong = inner.strong.has_count;
362 let weak = strong || inner.weak.count > 0;
363 let has_weak = inner.weak.has_count;
364
365 let should_drop_weak = !weak && has_weak;
366 let should_drop_strong = !strong && has_strong;
367
368 // If we want to drop the ref-count again, tell the caller to schedule a work node for
369 // that.
370 let need_push = should_drop_weak || should_drop_strong;
371
372 if need_push && inner.delivery_state.should_normal_push() {
373 let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();
374 inner.delivery_state.did_normal_push();
375 Some(list_arc)
376 } else {
377 None
378 }
379 } else {
380 None
381 }
382 }
383
384 pub(crate) fn update_refcount_locked(
385 self: &DArc<Node>,
386 inc: bool,
387 strong: bool,
388 count: usize,
389 owner_inner: &mut ProcessInner,
390 ) -> Option<DLArc<Node>> {
391 let is_dead = owner_inner.is_dead;
392 let inner = self.inner.access_mut(owner_inner);
393
394 // Get a reference to the state we'll update.
395 let state = if strong {
396 &mut inner.strong
397 } else {
398 &mut inner.weak
399 };
400
401 // Update the count and determine whether we need to push work.
402 let need_push = if inc {
403 state.count += count;
404 // TODO: This method shouldn't be used for zero-to-one increments.
405 !is_dead && !state.has_count
406 } else {
407 if state.count < count {
408 pr_err!("Failure: refcount underflow!");
409 return None;
410 }
411 state.count -= count;
412 !is_dead && state.count == 0 && state.has_count
413 };
414
415 if need_push && inner.delivery_state.should_normal_push() {
416 let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();
417 inner.delivery_state.did_normal_push();
418 Some(list_arc)
419 } else {
420 None
421 }
422 }
423
424 pub(crate) fn incr_refcount_allow_zero2one(
425 self: &DArc<Self>,
426 strong: bool,
427 owner_inner: &mut ProcessInner,
428 ) -> Result<Option<DLArc<Node>>, CouldNotDeliverCriticalIncrement> {
429 let is_dead = owner_inner.is_dead;
430 let inner = self.inner.access_mut(owner_inner);
431
432 // Get a reference to the state we'll update.
433 let state = if strong {
434 &mut inner.strong
435 } else {
436 &mut inner.weak
437 };
438
439 // Update the count and determine whether we need to push work.
440 state.count += 1;
441 if is_dead || state.has_count {
442 return Ok(None);
443 }
444
445 // Userspace needs to be notified of this.
446 if !strong && inner.delivery_state.should_push_weak_zero2one() {
447 assert!(inner.delivery_state.can_push_weak_zero2one_normally());
448 let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();
449 inner.delivery_state.did_push_weak_zero2one();
450 Ok(Some(list_arc))
451 } else if strong && inner.delivery_state.should_push_strong_zero2one() {
452 if inner.delivery_state.can_push_strong_zero2one_normally() {
453 let list_arc = ListArc::try_from_arc(self.clone()).ok().unwrap();
454 inner.delivery_state.did_push_strong_zero2one();
455 Ok(Some(list_arc))
456 } else {
457 state.count -= 1;
458 Err(CouldNotDeliverCriticalIncrement)
459 }
460 } else {
461 // Work is already pushed, and we don't need to push again.
462 Ok(None)
463 }
464 }
465
466 pub(crate) fn incr_refcount_allow_zero2one_with_wrapper(
467 self: &DArc<Self>,
468 strong: bool,
469 wrapper: CritIncrWrapper,
470 owner_inner: &mut ProcessInner,
471 ) -> Option<DLArc<dyn DeliverToRead>> {
472 match self.incr_refcount_allow_zero2one(strong, owner_inner) {
473 Ok(Some(node)) => Some(node as DLArc<dyn DeliverToRead>),
474 Ok(None) => None,
475 Err(CouldNotDeliverCriticalIncrement) => {
476 assert!(strong);
477 let inner = self.inner.access_mut(owner_inner);
478 inner.strong.count += 1;
479 inner.delivery_state.did_push_strong_zero2one_wrapper();
480 Some(wrapper.init(self.clone()))
481 }
482 }
483 }
484
485 pub(crate) fn update_refcount(self: &DArc<Self>, inc: bool, count: usize, strong: bool) {
486 self.owner
487 .inner
488 .lock()
489 .update_node_refcount(self, inc, strong, count, None);
490 }
491
492 pub(crate) fn populate_counts(
493 &self,
494 out: &mut BinderNodeInfoForRef,
495 guard: &Guard<'_, ProcessInner, SpinLockBackend>,
496 ) {
497 let inner = self.inner.access(guard);
498 out.strong_count = inner.strong.count as u32;
499 out.weak_count = inner.weak.count as u32;
500 }
501
502 pub(crate) fn populate_debug_info(
503 &self,
504 out: &mut BinderNodeDebugInfo,
505 guard: &Guard<'_, ProcessInner, SpinLockBackend>,
506 ) {
507 out.ptr = self.ptr as uapi::binder_uintptr_t;
508 out.cookie = self.cookie as uapi::binder_uintptr_t;
509 let inner = self.inner.access(guard);
510 if inner.strong.has_count {
511 out.has_strong_ref = 1;
512 }
513 if inner.weak.has_count {
514 out.has_weak_ref = 1;
515 }
516 }
517
518 pub(crate) fn force_has_count(&self, guard: &mut Guard<'_, ProcessInner, SpinLockBackend>) {
519 let inner = self.inner.access_mut(guard);
520 inner.strong.has_count = true;
521 inner.weak.has_count = true;
522 }
523
524 fn write(&self, writer: &mut BinderReturnWriter<'_>, code: u32) -> Result {
525 writer.write_code(code)?;
526 writer.write_payload(&self.ptr)?;
527 writer.write_payload(&self.cookie)?;
528 Ok(())
529 }
530
531 pub(crate) fn submit_oneway(
532 &self,
533 transaction: DLArc<Transaction>,
534 guard: &mut Guard<'_, ProcessInner, SpinLockBackend>,
535 ) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> {
536 if guard.is_dead {
537 return Err((BinderError::new_dead(), transaction));
538 }
539
540 let inner = self.inner.access_mut(guard);
541 if inner.has_oneway_transaction {
542 inner.oneway_todo.push_back(transaction);
543 } else {
544 inner.has_oneway_transaction = true;
545 guard.push_work(&self.owner, transaction)?;
546 }
547 Ok(())
548 }
549
550 pub(crate) fn release(&self) {
551 let mut guard = self.owner.inner.lock();
552 while let Some(work) = self.inner.access_mut(&mut guard).oneway_todo.pop_front() {
553 drop(guard);
554 work.into_arc().cancel();
555 guard = self.owner.inner.lock();
556 }
557
558 while let Some(death) = self.inner.access_mut(&mut guard).death_list.pop_front() {
559 drop(guard);
560 death.into_arc().set_dead();
561 guard = self.owner.inner.lock();
562 }
563 }
564
565 pub(crate) fn pending_oneway_finished(&self) {
566 let mut guard = self.owner.inner.lock();
567 if guard.is_dead {
568 // Cleanup will happen in `Process::deferred_release`.
569 return;
570 }
571
572 let inner = self.inner.access_mut(&mut guard);
573
574 let transaction = inner.oneway_todo.pop_front();
575 inner.has_oneway_transaction = transaction.is_some();
576 if let Some(transaction) = transaction {
577 match guard.push_work(&self.owner, transaction) {
578 Ok(()) => {}
579 Err((_err, work)) => {
580 // Process is dead.
581 // This shouldn't happen due to the `is_dead` check, but if it does, just drop
582 // the transaction and return.
583 drop(guard);
584 drop(work);
585 }
586 }
587 }
588 }
589
590 /// Finds an outdated transaction that the given transaction can replace.
591 ///
592 /// If one is found, it is removed from the list and returned.
593 pub(crate) fn take_outdated_transaction(
594 &self,
595 new: &Transaction,
596 guard: &mut Guard<'_, ProcessInner, SpinLockBackend>,
597 ) -> Option<DLArc<Transaction>> {
598 let inner = self.inner.access_mut(guard);
599 let mut cursor = inner.oneway_todo.cursor_front();
600 while let Some(next) = cursor.peek_next() {
601 if new.can_replace(&next) {
602 return Some(next.remove());
603 }
604 cursor.move_next();
605 }
606 None
607 }
608
609 /// This is split into a separate function since it's called by both `Node::do_work` and
610 /// `NodeWrapper::do_work`.
611 fn do_work_locked(
612 &self,
613 writer: &mut BinderReturnWriter<'_>,
614 mut guard: Guard<'_, ProcessInner, SpinLockBackend>,
615 ) -> Result<bool> {
616 let inner = self.inner.access_mut(&mut guard);
617 let strong = inner.strong.count > 0;
618 let has_strong = inner.strong.has_count;
619 let weak = strong || inner.weak.count > 0;
620 let has_weak = inner.weak.has_count;
621
622 if weak && !has_weak {
623 inner.weak.has_count = true;
624 inner.active_inc_refs += 1;
625 }
626
627 if strong && !has_strong {
628 inner.strong.has_count = true;
629 inner.active_inc_refs += 1;
630 }
631
632 let no_active_inc_refs = inner.active_inc_refs == 0;
633 let should_drop_weak = no_active_inc_refs && (!weak && has_weak);
634 let should_drop_strong = no_active_inc_refs && (!strong && has_strong);
635 if should_drop_weak {
636 inner.weak.has_count = false;
637 }
638 if should_drop_strong {
639 inner.strong.has_count = false;
640 }
641 if no_active_inc_refs && !weak {
642 // Remove the node if there are no references to it.
643 guard.remove_node(self.ptr);
644 }
645 drop(guard);
646
647 if weak && !has_weak {
648 self.write(writer, BR_INCREFS)?;
649 }
650 if strong && !has_strong {
651 self.write(writer, BR_ACQUIRE)?;
652 }
653 if should_drop_strong {
654 self.write(writer, BR_RELEASE)?;
655 }
656 if should_drop_weak {
657 self.write(writer, BR_DECREFS)?;
658 }
659
660 Ok(true)
661 }
662
663 pub(crate) fn add_freeze_listener(
664 &self,
665 process: &Arc<Process>,
666 // If the vector needs to be resized, it's done via this argument.
667 vec_alloc: &mut KVVec<Arc<Process>>,
668 ) -> Result<Result<(), usize>> {
669 let mut guard = self.owner.inner.lock();
670 // Do not check for `guard.dead`. The `dead` flag that matters here is the owner of the
671 // listener, not the target.
672 let inner = self.inner.access_mut(&mut guard);
673 let len = inner.freeze_list.len();
674 if len == inner.freeze_list.capacity() {
675 if len >= vec_alloc.capacity() {
676 // Request the caller to reallocate.
677 return Ok(Err((1 + len).next_power_of_two()));
678 }
679 mem::swap(&mut inner.freeze_list, vec_alloc);
680 for elem in vec_alloc.drain_all() {
681 inner.freeze_list.push_within_capacity(elem)?;
682 }
683 }
684 inner.freeze_list.push_within_capacity(process.clone())?;
685 Ok(Ok(()))
686 }
687
688 pub(crate) fn remove_freeze_listener(&self, p: &Process) -> KVVec<Arc<Process>> {
689 let mut guard = self.owner.inner.lock();
690 let inner = self.inner.access_mut(&mut guard);
691 let len = inner.freeze_list.len();
692 inner
693 .freeze_list
694 .retain(|proc| !core::ptr::eq::<Process>(&**proc, p));
695 if len == inner.freeze_list.len() {
696 pr_warn!(
697 "Could not remove freeze listener for {}\n",
698 p.pid_in_current_ns()
699 );
700 }
701 // If the vector is empty it needs to be freed. However, we can't free it here because that
702 // might sleep, so return it to the caller.
703 if inner.freeze_list.is_empty() {
704 return mem::take(&mut inner.freeze_list);
705 }
706 KVVec::new()
707 }
708
709 pub(crate) fn freeze_list<'a>(&'a self, guard: &'a ProcessInner) -> &'a [Arc<Process>] {
710 &self.inner.access(guard).freeze_list
711 }
712}
713
714impl DeliverToRead for Node {
715 fn do_work(
716 self: DArc<Self>,
717 _thread: &Thread,
718 writer: &mut BinderReturnWriter<'_>,
719 ) -> Result<bool> {
720 let mut owner_inner = self.owner.inner.lock();
721 let inner = self.inner.access_mut(&mut owner_inner);
722
723 assert!(inner.delivery_state.has_pushed_node);
724 if inner.delivery_state.has_pushed_wrapper {
725 // If the wrapper is scheduled, then we are either a normal push or weak zero2one
726 // increment, and the wrapper is a strong zero2one increment, so the wrapper always
727 // takes precedence over us.
728 assert!(inner.delivery_state.has_strong_zero2one);
729 inner.delivery_state.has_pushed_node = false;
730 inner.delivery_state.has_weak_zero2one = false;
731 return Ok(true);
732 }
733
734 inner.delivery_state.has_pushed_node = false;
735 inner.delivery_state.has_weak_zero2one = false;
736 inner.delivery_state.has_strong_zero2one = false;
737
738 self.do_work_locked(writer, owner_inner)
739 }
740
741 fn cancel(self: DArc<Self>) {
742 let _drop_outside_lock;
743 let mut owner_inner = self.owner.inner.lock();
744
745 // We only do something on BINDER_THREAD_EXIT, not process exit.
746 if owner_inner.is_dead {
747 return;
748 }
749
750 // If BINDER_THREAD_EXIT is invoked on a thread with a pending node refcount update, we
751 // should move ourselves to ensure the refcount update is still delivered.
752 if let Some(node) = ListArc::try_from_arc_borrow(self.as_arc_borrow()) {
753 _drop_outside_lock = owner_inner.push_work(&self.owner, node);
754 }
755 }
756
757 fn should_sync_wakeup(&self) -> bool {
758 false
759 }
760
761 #[inline(never)]
762 fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
763 seq_print!(
764 m,
765 "{}node work {}: u{:016x} c{:016x}\n",
766 prefix,
767 self.debug_id,
768 self.ptr,
769 self.cookie,
770 );
771 Ok(())
772 }
773}
774
775/// Represents something that holds one or more ref-counts to a `Node`.
776///
777/// Whenever process A holds a refcount to a node owned by a different process B, then process A
778/// will store a `NodeRef` that refers to the `Node` in process B. When process A releases the
779/// refcount, we destroy the NodeRef, which decrements the ref-count in process A.
780///
781/// This type is also used for some other cases. For example, a transaction allocation holds a
782/// refcount on the target node, and this is implemented by storing a `NodeRef` in the allocation
783/// so that the destructor of the allocation will drop a refcount of the `Node`.
784pub(crate) struct NodeRef {
785 pub(crate) node: DArc<Node>,
786 /// How many times does this NodeRef hold a refcount on the Node?
787 strong_node_count: usize,
788 weak_node_count: usize,
789 /// How many times does userspace hold a refcount on this NodeRef?
790 strong_count: usize,
791 weak_count: usize,
792}
793
794impl NodeRef {
795 pub(crate) fn new(node: DArc<Node>, strong_count: usize, weak_count: usize) -> Self {
796 Self {
797 node,
798 strong_node_count: strong_count,
799 weak_node_count: weak_count,
800 strong_count,
801 weak_count,
802 }
803 }
804
805 pub(crate) fn absorb(&mut self, mut other: Self) {
806 assert!(
807 Arc::ptr_eq(&self.node, &other.node),
808 "absorb called with differing nodes"
809 );
810 self.strong_node_count += other.strong_node_count;
811 self.weak_node_count += other.weak_node_count;
812 self.strong_count += other.strong_count;
813 self.weak_count += other.weak_count;
814 other.strong_count = 0;
815 other.weak_count = 0;
816 other.strong_node_count = 0;
817 other.weak_node_count = 0;
818
819 if self.strong_node_count >= 2 || self.weak_node_count >= 2 {
820 let mut guard = self.node.owner.inner.lock();
821 let inner = self.node.inner.access_mut(&mut guard);
822
823 if self.strong_node_count >= 2 {
824 inner.strong.count -= self.strong_node_count - 1;
825 self.strong_node_count = 1;
826 assert_ne!(inner.strong.count, 0);
827 }
828 if self.weak_node_count >= 2 {
829 inner.weak.count -= self.weak_node_count - 1;
830 self.weak_node_count = 1;
831 assert_ne!(inner.weak.count, 0);
832 }
833 }
834 }
835
836 pub(crate) fn get_count(&self) -> (usize, usize) {
837 (self.strong_count, self.weak_count)
838 }
839
840 pub(crate) fn clone(&self, strong: bool) -> Result<NodeRef> {
841 if strong && self.strong_count == 0 {
842 binder_debug!(UserError, "tried to use weak ref as strong ref");
843 return Err(EINVAL);
844 }
845 Ok(self
846 .node
847 .owner
848 .inner
849 .lock()
850 .new_node_ref(self.node.clone(), strong, None))
851 }
852
853 /// Updates (increments or decrements) the number of references held against the node. If the
854 /// count being updated transitions from 0 to 1 or from 1 to 0, the node is notified by having
855 /// its `update_refcount` function called.
856 ///
857 /// Returns whether `self` should be removed (when both counts are zero).
858 pub(crate) fn update(&mut self, inc: bool, strong: bool) -> bool {
859 if strong && self.strong_count == 0 {
860 return false;
861 }
862 let (count, node_count, other_count) = if strong {
863 (
864 &mut self.strong_count,
865 &mut self.strong_node_count,
866 self.weak_count,
867 )
868 } else {
869 (
870 &mut self.weak_count,
871 &mut self.weak_node_count,
872 self.strong_count,
873 )
874 };
875 if inc {
876 if *count == 0 {
877 *node_count = 1;
878 self.node.update_refcount(true, 1, strong);
879 }
880 *count += 1;
881 } else {
882 if *count == 0 {
883 binder_debug!(
884 UserError,
885 "performed invalid {} decrement on ref",
886 if strong { "strong" } else { "weak" }
887 );
888 return false;
889 }
890 *count -= 1;
891 if *count == 0 {
892 self.node.update_refcount(false, *node_count, strong);
893 *node_count = 0;
894 return other_count == 0;
895 }
896 }
897 false
898 }
899}
900
901impl Drop for NodeRef {
902 // This destructor is called conditionally from `Allocation::drop`. That branch is often
903 // mispredicted. Inlining this method call reduces the cost of those branch mispredictions.
904 #[inline(always)]
905 fn drop(&mut self) {
906 if self.strong_node_count > 0 {
907 self.node
908 .update_refcount(false, self.strong_node_count, true);
909 }
910 if self.weak_node_count > 0 {
911 self.node
912 .update_refcount(false, self.weak_node_count, false);
913 }
914 }
915}
916
917struct NodeDeathInner {
918 dead: bool,
919 cleared: bool,
920 notification_done: bool,
921 /// Indicates whether the normal flow was interrupted by removing the handle. In this case, we
922 /// need behave as if the death notification didn't exist (i.e., we don't deliver anything to
923 /// the user.
924 aborted: bool,
925}
926
927/// Used to deliver notifications when a process dies.
928///
929/// A process can request to be notified when a process dies using `BC_REQUEST_DEATH_NOTIFICATION`.
930/// This will make the driver send a `BR_DEAD_BINDER` to userspace when the process dies (or
931/// immediately if it is already dead). Userspace is supposed to respond with `BC_DEAD_BINDER_DONE`
932/// once it has processed the notification.
933///
934/// Userspace can unregister from death notifications using the `BC_CLEAR_DEATH_NOTIFICATION`
935/// command. In this case, the kernel will respond with `BR_CLEAR_DEATH_NOTIFICATION_DONE` once the
936/// notification has been removed. Note that if the remote process dies before the kernel has
937/// responded with `BR_CLEAR_DEATH_NOTIFICATION_DONE`, then the kernel will still send a
938/// `BR_DEAD_BINDER`, which userspace must be able to process. In this case, the kernel will wait
939/// for the `BC_DEAD_BINDER_DONE` command before it sends `BR_CLEAR_DEATH_NOTIFICATION_DONE`.
940///
941/// Note that even if the kernel sends a `BR_DEAD_BINDER`, this does not remove the death
942/// notification. Userspace must still remove it manually using `BC_CLEAR_DEATH_NOTIFICATION`.
943///
944/// If a process uses `BC_RELEASE` to destroy its last refcount on a node that has an active death
945/// registration, then the death registration is immediately deleted (we implement this using the
946/// `aborted` field). However, userspace is not supposed to delete a `NodeRef` without first
947/// deregistering death notifications, so this codepath is not executed under normal circumstances.
948#[pin_data]
949pub(crate) struct NodeDeath {
950 node: DArc<Node>,
951 process: Arc<Process>,
952 pub(crate) cookie: u64,
953 #[pin]
954 links_track: AtomicTracker<0>,
955 /// Used by the owner `Node` to store a list of registered death notifications.
956 ///
957 /// # Invariants
958 ///
959 /// Only ever used with the `death_list` list of `self.node`.
960 #[pin]
961 death_links: ListLinks<1>,
962 /// Used by the process to keep track of the death notifications for which we have sent a
963 /// `BR_DEAD_BINDER` but not yet received a `BC_DEAD_BINDER_DONE`.
964 ///
965 /// # Invariants
966 ///
967 /// Only ever used with the `delivered_deaths` list of `self.process`.
968 #[pin]
969 delivered_links: ListLinks<2>,
970 #[pin]
971 delivered_links_track: AtomicTracker<2>,
972 #[pin]
973 inner: SpinLock<NodeDeathInner>,
974}
975
976impl NodeDeath {
977 /// Constructs a new node death notification object.
978 pub(crate) fn new(
979 node: DArc<Node>,
980 process: Arc<Process>,
981 cookie: u64,
982 ) -> impl PinInit<DTRWrap<Self>> {
983 DTRWrap::new(pin_init!(
984 Self {
985 node,
986 process,
987 cookie,
988 links_track <- AtomicTracker::new(),
989 death_links <- ListLinks::new(),
990 delivered_links <- ListLinks::new(),
991 delivered_links_track <- AtomicTracker::new(),
992 inner <- kernel::new_spinlock!(NodeDeathInner {
993 dead: false,
994 cleared: false,
995 notification_done: false,
996 aborted: false,
997 }, "NodeDeath::inner"),
998 }
999 ))
1000 }
1001
1002 /// Sets the cleared flag to `true`.
1003 ///
1004 /// It removes `self` from the node's death notification list if needed.
1005 ///
1006 /// Returns whether it needs to be queued.
1007 pub(crate) fn set_cleared(self: &DArc<Self>, abort: bool) -> bool {
1008 let (needs_removal, needs_queueing) = {
1009 // Update state and determine if we need to queue a work item. We only need to do it
1010 // when the node is not dead or if the user already completed the death notification.
1011 let mut inner = self.inner.lock();
1012 if abort {
1013 inner.aborted = true;
1014 }
1015 if inner.cleared {
1016 // Already cleared.
1017 return false;
1018 }
1019 inner.cleared = true;
1020 (!inner.dead, !inner.dead || inner.notification_done)
1021 };
1022
1023 // Remove death notification from node.
1024 if needs_removal {
1025 let mut owner_inner = self.node.owner.inner.lock();
1026 let node_inner = self.node.inner.access_mut(&mut owner_inner);
1027 // SAFETY: A `NodeDeath` is never inserted into the death list of any node other than
1028 // its owner, so it is either in this death list or in no death list.
1029 unsafe { node_inner.death_list.remove(self) };
1030 }
1031 needs_queueing
1032 }
1033
1034 /// Sets the 'notification done' flag to `true`.
1035 pub(crate) fn set_notification_done(self: DArc<Self>, thread: &Thread) {
1036 let needs_queueing = {
1037 let mut inner = self.inner.lock();
1038 inner.notification_done = true;
1039 inner.cleared
1040 };
1041 if needs_queueing {
1042 if let Some(death) = ListArc::try_from_arc_or_drop(self) {
1043 let _ = thread.push_work_if_looper(death);
1044 }
1045 }
1046 }
1047
1048 /// Sets the 'dead' flag to `true` and queues work item if needed.
1049 pub(crate) fn set_dead(self: DArc<Self>) {
1050 let needs_queueing = {
1051 let mut inner = self.inner.lock();
1052 if inner.cleared {
1053 false
1054 } else {
1055 inner.dead = true;
1056 true
1057 }
1058 };
1059 if needs_queueing {
1060 // Push the death notification to the target process. There is nothing else to do if
1061 // it's already dead.
1062 if let Some(death) = ListArc::try_from_arc_or_drop(self) {
1063 let process = death.process.clone();
1064 let _ = process.push_work(death);
1065 }
1066 }
1067 }
1068}
1069
1070kernel::list::impl_list_arc_safe! {
1071 impl ListArcSafe<0> for NodeDeath {
1072 tracked_by links_track: AtomicTracker;
1073 }
1074}
1075
1076kernel::list::impl_list_arc_safe! {
1077 impl ListArcSafe<1> for DTRWrap<NodeDeath> { untracked; }
1078}
1079kernel::list::impl_list_item! {
1080 impl ListItem<1> for DTRWrap<NodeDeath> {
1081 using ListLinks { self.wrapped.death_links };
1082 }
1083}
1084
1085kernel::list::impl_list_arc_safe! {
1086 impl ListArcSafe<2> for DTRWrap<NodeDeath> {
1087 tracked_by wrapped: NodeDeath;
1088 }
1089}
1090kernel::list::impl_list_arc_safe! {
1091 impl ListArcSafe<2> for NodeDeath {
1092 tracked_by delivered_links_track: AtomicTracker<2>;
1093 }
1094}
1095kernel::list::impl_list_item! {
1096 impl ListItem<2> for DTRWrap<NodeDeath> {
1097 using ListLinks { self.wrapped.delivered_links };
1098 }
1099}
1100
1101impl DeliverToRead for NodeDeath {
1102 fn do_work(
1103 self: DArc<Self>,
1104 _thread: &Thread,
1105 writer: &mut BinderReturnWriter<'_>,
1106 ) -> Result<bool> {
1107 let done = {
1108 let inner = self.inner.lock();
1109 if inner.aborted {
1110 return Ok(true);
1111 }
1112 inner.cleared && (!inner.dead || inner.notification_done)
1113 };
1114
1115 let cookie = self.cookie;
1116 let cmd = if done {
1117 BR_CLEAR_DEATH_NOTIFICATION_DONE
1118 } else {
1119 let process = self.process.clone();
1120 let mut process_inner = process.inner.lock();
1121 let inner = self.inner.lock();
1122 if inner.aborted {
1123 return Ok(true);
1124 }
1125 // We're still holding the inner lock, so it cannot be aborted while we insert it into
1126 // the delivered list.
1127 process_inner.death_delivered(self.clone());
1128 binder_debug!(
1129 DeathNotification,
1130 "sending death notification, cookie {:016x}",
1131 cookie
1132 );
1133 BR_DEAD_BINDER
1134 };
1135
1136 writer.write_code(cmd)?;
1137 writer.write_payload(&cookie)?;
1138 // DEAD_BINDER notifications can cause transactions, so stop processing work items when we
1139 // get to a death notification.
1140 Ok(cmd != BR_DEAD_BINDER)
1141 }
1142
1143 fn cancel(self: DArc<Self>) {
1144 binder_debug!(
1145 pid = self.process.task.pid(),
1146 DeadTransaction,
1147 "undelivered death notification, {:016x}",
1148 self.cookie
1149 );
1150 }
1151
1152 fn should_sync_wakeup(&self) -> bool {
1153 false
1154 }
1155
1156 #[inline(never)]
1157 fn debug_print(&self, m: &SeqFile, prefix: &str, _tprefix: &str) -> Result<()> {
1158 let inner = self.inner.lock();
1159
1160 let dead_binder = inner.dead && !inner.notification_done;
1161
1162 if dead_binder {
1163 if inner.cleared {
1164 seq_print!(m, "{}has cleared dead binder\n", prefix);
1165 } else {
1166 seq_print!(m, "{}has dead binder\n", prefix);
1167 }
1168 } else {
1169 seq_print!(m, "{}has cleared death notification\n", prefix);
1170 }
1171
1172 Ok(())
1173 }
1174}