process.rs

// SPDX-License-Identifier: GPL-2.0

// Copyright (C) 2025 Google LLC.

//! This module defines the `Process` type, which represents a process using a particular binder
//! context.
//!
//! The `Process` object keeps track of all of the resources that this process owns in the binder
//! context.
//!
//! There is one `Process` object for each binder fd that a process has opened, so processes using
//! several binder contexts have several `Process` objects. This ensures that the contexts are
//! fully separated.

use core::mem::take;

use kernel::{
    bindings,
    bits::bit_u8,
    cred::Credential,
    error::Error,
    fs::file::{self, File},
    id_pool::IdPool,
    list::{List, ListArc, ListArcField, ListLinks},
    mm,
    prelude::*,
    rbtree::{self, RBTree, RBTreeNode, RBTreeNodeReservation},
    seq_file::SeqFile,
    seq_print,
    sync::poll::PollTable,
    sync::{
        aref::ARef,
        lock::{spinlock::SpinLockBackend, Guard},
        poll::PollCondVarBox,
        Arc, ArcBorrow, CondVar, CondVarTimeoutResult, SetOnce, SpinLock, UniqueArc,
    },
    task::{Pid, Task},
    uaccess::{UserSlice, UserSliceReader},
    uapi,
    workqueue::{self, Work},
};

use crate::{
    allocation::{Allocation, AllocationInfo, NewAllocation},
    context::Context,
    defs::*,
    error::{BinderError, BinderResult},
    node::{CouldNotDeliverCriticalIncrement, CritIncrWrapper, Node, NodeDeath, NodeRef},
    page_range::ShrinkablePageRange,
    range_alloc::{RangeAllocator, ReserveNew, ReserveNewArgs},
    stats::BinderStats,
    thread::{PushWorkRes, Thread},
    transaction::TransactionInfo,
    BinderfsProcFile, DArc, DLArc, DTRWrap, DeliverToRead,
};

#[path = "freeze.rs"]
mod freeze;
use self::freeze::{FreezeCookie, FreezeListener};

struct Mapping {
    address: usize,
    alloc: RangeAllocator<AllocationInfo>,
}

impl Mapping {
    fn new(address: usize, size: usize) -> Self {
        Self {
            address,
            alloc: RangeAllocator::new(size),
        }
    }
}

kernel::impl_flags!(
    /// Represents multiple deferred work flags.
    #[derive(Debug, Clone, Default, Copy, PartialEq, Eq)]
    pub struct DeferWorks(u8);

    /// Represents a single deferred work category.
    #[derive(Debug, Clone, Copy, PartialEq, Eq)]
    pub enum DeferWork {
        Flush = bit_u8(0),
        Release = bit_u8(1),
    }
);

#[derive(Copy, Clone)]
pub(crate) enum IsFrozen {
    Yes,
    No,
    InProgress,
}

impl IsFrozen {
    /// Whether incoming transactions should be rejected due to freeze.
    pub(crate) fn is_frozen(self) -> bool {
        match self {
            IsFrozen::Yes => true,
            IsFrozen::No => false,
            IsFrozen::InProgress => true,
        }
    }

    /// Whether freeze notifications consider this process frozen.
    pub(crate) fn is_fully_frozen(self) -> bool {
        match self {
            IsFrozen::Yes => true,
            IsFrozen::No => false,
            IsFrozen::InProgress => false,
        }
    }
}

/// The fields of `Process` protected by the spinlock.
pub(crate) struct ProcessInner {
    is_manager: bool,
    pub(crate) is_dead: bool,
    threads: RBTree<i32, Arc<Thread>>,
    /// INVARIANT: Threads pushed to this list must be owned by this process.
    ready_threads: List<Thread>,
    nodes: RBTree<u64, DArc<Node>>,
    mapping: Option<Mapping>,
    work: List<DTRWrap<dyn DeliverToRead>>,
    delivered_deaths: List<DTRWrap<NodeDeath>, 2>,

    /// The number of requested threads that haven't registered yet.
    requested_thread_count: u32,
    /// The maximum number of threads used by the process thread pool.
    max_threads: u32,
    /// The number of threads the started and registered with the thread pool.
    started_thread_count: u32,

    /// Bitmap of deferred work to do.
    defer_work: DeferWorks,

    /// Number of transactions to be transmitted before processes in freeze_wait
    /// are woken up.
    outstanding_txns: u32,
    /// Process is frozen and unable to service binder transactions.
    pub(crate) is_frozen: IsFrozen,
    /// Process received sync transactions since last frozen.
    pub(crate) sync_recv: bool,
    /// Process received async transactions since last frozen.
    pub(crate) async_recv: bool,
    pub(crate) binderfs_file: Option<BinderfsProcFile>,
    /// Check for oneway spam
    oneway_spam_detection_enabled: bool,
}

impl ProcessInner {
    fn new() -> Self {
        Self {
            is_manager: false,
            is_dead: false,
            threads: RBTree::new(),
            ready_threads: List::new(),
            mapping: None,
            nodes: RBTree::new(),
            work: List::new(),
            delivered_deaths: List::new(),
            requested_thread_count: 0,
            max_threads: 0,
            started_thread_count: 0,
            defer_work: DeferWorks::default(),
            outstanding_txns: 0,
            is_frozen: IsFrozen::No,
            sync_recv: false,
            async_recv: false,
            binderfs_file: None,
            oneway_spam_detection_enabled: false,
        }
    }

    /// Schedule the work item for execution on this process.
    ///
    /// If any threads are ready for work, then the work item is given directly to that thread and
    /// it is woken up. Otherwise, it is pushed to the process work list.
    ///
    /// This call can fail only if the process is dead. In this case, the work item is returned to
    /// the caller so that the caller can drop it after releasing the inner process lock. This is
    /// necessary since the destructor of `Transaction` will take locks that can't necessarily be
    /// taken while holding the inner process lock.
    pub(crate) fn push_work(
        &mut self,
        proc: &Process,
        work: DLArc<dyn DeliverToRead>,
    ) -> Result<(), (BinderError, DLArc<dyn DeliverToRead>)> {
        let sync = work.should_sync_wakeup();

        // Try to find a ready thread to which to push the work.
        if let Some(thread) = self.ready_threads.pop_front() {
            // Push to thread while holding state lock. This prevents the thread from giving up
            // (for example, because of a signal) when we're about to deliver work.
            match thread.push_work_inner(work, sync) {
                PushWorkRes::Ok => Ok(()),
                PushWorkRes::OkNotifyPoll => {
                    proc.notify_poll(sync);
                    Ok(())
                }
                PushWorkRes::FailedDead(work) => Err((BinderError::new_dead(), work)),
            }
        } else if self.is_dead {
            Err((BinderError::new_dead(), work))
        } else {
            // Didn't find a thread waiting for proc work; this can happen
            // in two scenarios:
            // 1. All threads are busy handling transactions
            //    In that case, one of those threads should call back into
            //    the kernel driver soon and pick up this work.
            // 2. Threads are using the (e)poll interface, in which case
            //    they may be blocked on the waitqueue without having been
            //    added to waiting_threads. For this case, we wake it up
            //    directly.
            self.work.push_back(work);

            // Wake up polling threads, if any.
            proc.notify_poll(sync);

            Ok(())
        }
    }

    pub(crate) fn remove_node(&mut self, ptr: u64) {
        self.nodes.remove(&ptr);
    }

    /// Updates the reference count on the given node.
    pub(crate) fn update_node_refcount(
        &mut self,
        node: &DArc<Node>,
        inc: bool,
        strong: bool,
        count: usize,
        othread: Option<&Thread>,
    ) {
        let push = node.update_refcount_locked(inc, strong, count, self);

        // If we decided that we need to push work, push either to the process or to a thread if
        // one is specified.
        if let Some(pnode) = push {
            if let Some(thread) = othread {
                thread.push_work_deferred(pnode);
            } else {
                let _ = self.push_work(&node.owner, pnode);
                // Nothing to do: `push_work` may fail if the process is dead, but that's ok as in
                // that case, it doesn't care about the notification.
            }
        }
    }

    pub(crate) fn new_node_ref(
        &mut self,
        node: DArc<Node>,
        strong: bool,
        thread: Option<&Thread>,
    ) -> NodeRef {
        self.update_node_refcount(&node, true, strong, 1, thread);
        let strong_count = if strong { 1 } else { 0 };
        NodeRef::new(node, strong_count, 1 - strong_count)
    }

    pub(crate) fn new_node_ref_with_thread(
        &mut self,
        node: DArc<Node>,
        strong: bool,
        thread: &Thread,
        wrapper: Option<CritIncrWrapper>,
    ) -> Result<NodeRef, CouldNotDeliverCriticalIncrement> {
        let push = match wrapper {
            None => node
                .incr_refcount_allow_zero2one(strong, self)?
                .map(|node| node as DLArc<dyn DeliverToRead>),
            Some(wrapper) => node.incr_refcount_allow_zero2one_with_wrapper(strong, wrapper, self),
        };
        if let Some(node) = push {
            thread.push_work_deferred(node);
        }
        let strong_count = if strong { 1 } else { 0 };
        Ok(NodeRef::new(node, strong_count, 1 - strong_count))
    }

    /// Returns an existing node with the given pointer and cookie, if one exists.
    ///
    /// Returns an error if a node with the given pointer but a different cookie exists.
    fn get_existing_node(&self, ptr: u64, cookie: u64) -> Result<Option<DArc<Node>>> {
        match self.nodes.get(&ptr) {
            None => Ok(None),
            Some(node) => {
                let (_, node_cookie) = node.get_id();
                if node_cookie == cookie {
                    Ok(Some(node.clone()))
                } else {
                    Err(EINVAL)
                }
            }
        }
    }

    fn register_thread(&mut self) -> bool {
        if self.requested_thread_count == 0 {
            return false;
        }

        self.requested_thread_count -= 1;
        self.started_thread_count += 1;
        true
    }

    /// Finds a delivered death notification with the given cookie, removes it from the thread's
    /// delivered list, and returns it.
    fn pull_delivered_death(&mut self, cookie: u64) -> Option<DArc<NodeDeath>> {
        let mut cursor = self.delivered_deaths.cursor_front();
        while let Some(next) = cursor.peek_next() {
            if next.cookie == cookie {
                return Some(next.remove().into_arc());
            }
            cursor.move_next();
        }
        None
    }

    pub(crate) fn death_delivered(&mut self, death: DArc<NodeDeath>) {
        if let Some(death) = ListArc::try_from_arc_or_drop(death) {
            self.delivered_deaths.push_back(death);
        } else {
            pr_warn!("Notification added to `delivered_deaths` twice.");
        }
    }

    pub(crate) fn add_outstanding_txn(&mut self) {
        self.outstanding_txns += 1;
    }

    fn txns_pending_locked(&self) -> bool {
        if self.outstanding_txns > 0 {
            return true;
        }
        for thread in self.threads.values() {
            if thread.has_current_transaction() {
                return true;
            }
        }
        false
    }
}

/// Used to keep track of a node that this process has a handle to.
#[pin_data]
pub(crate) struct NodeRefInfo {
    debug_id: usize,
    /// The refcount that this process owns to the node.
    node_ref: ListArcField<NodeRef, { Self::LIST_PROC }>,
    death: ListArcField<Option<DArc<NodeDeath>>, { Self::LIST_PROC }>,
    /// Cookie of the active freeze listener for this node.
    freeze: ListArcField<Option<FreezeCookie>, { Self::LIST_PROC }>,
    /// Used to store this `NodeRefInfo` in the node's `refs` list.
    #[pin]
    links: ListLinks<{ Self::LIST_NODE }>,
    /// The handle for this `NodeRefInfo`.
    handle: u32,
    /// The process that has a handle to the node.
    pub(crate) process: Arc<Process>,
}

impl NodeRefInfo {
    /// The id used for the `Node::refs` list.
    pub(crate) const LIST_NODE: u64 = 0x2da16350fb724a10;
    /// The id used for the `ListArc` in `ProcessNodeRefs`.
    const LIST_PROC: u64 = 0xd703a5263dcc8650;

    fn new(node_ref: NodeRef, handle: u32, process: Arc<Process>) -> impl PinInit<Self> {
        pin_init!(Self {
            debug_id: super::next_debug_id(),
            node_ref: ListArcField::new(node_ref),
            death: ListArcField::new(None),
            freeze: ListArcField::new(None),
            links <- ListLinks::new(),
            handle,
            process,
        })
    }

    kernel::list::define_list_arc_field_getter! {
        pub(crate) fn death(&mut self<{Self::LIST_PROC}>) -> &mut Option<DArc<NodeDeath>> { death }
        pub(crate) fn freeze(&mut self<{Self::LIST_PROC}>) -> &mut Option<FreezeCookie> { freeze }
        pub(crate) fn node_ref(&mut self<{Self::LIST_PROC}>) -> &mut NodeRef { node_ref }
        pub(crate) fn node_ref2(&self<{Self::LIST_PROC}>) -> &NodeRef { node_ref }
    }
}

kernel::list::impl_list_arc_safe! {
    impl ListArcSafe<{Self::LIST_NODE}> for NodeRefInfo { untracked; }
    impl ListArcSafe<{Self::LIST_PROC}> for NodeRefInfo { untracked; }
}
kernel::list::impl_list_item! {
    impl ListItem<{Self::LIST_NODE}> for NodeRefInfo {
        using ListLinks { self.links };
    }
}

/// Keeps track of references this process has to nodes owned by other processes.
///
/// TODO: Currently, the rbtree requires two allocations per node reference, and two tree
/// traversals to look up a node by `Node::global_id`. Once the rbtree is more powerful, these
/// extra costs should be eliminated.
struct ProcessNodeRefs {
    /// Used to look up nodes using the 32-bit id that this process knows it by.
    by_handle: RBTree<u32, ListArc<NodeRefInfo, { NodeRefInfo::LIST_PROC }>>,
    /// Used to quickly find unused ids in `by_handle`.
    handle_is_present: IdPool,
    /// Used to look up nodes without knowing their local 32-bit id. The usize is the address of
    /// the underlying `Node` struct as returned by `Node::global_id`.
    by_node: RBTree<usize, u32>,
    /// Used to look up a `FreezeListener` by cookie.
    ///
    /// There might be multiple freeze listeners for the same node, but at most one of them is
    /// active.
    freeze_listeners: RBTree<FreezeCookie, FreezeListener>,
}

impl ProcessNodeRefs {
    fn new() -> Self {
        Self {
            by_handle: RBTree::new(),
            handle_is_present: IdPool::new(),
            by_node: RBTree::new(),
            freeze_listeners: RBTree::new(),
        }
    }
}

use core::mem::offset_of;
use kernel::bindings::rb_process_layout;
pub(crate) const PROCESS_LAYOUT: rb_process_layout = rb_process_layout {
    arc_offset: Arc::<Process>::DATA_OFFSET,
    task: offset_of!(Process, task),
};

/// A process using binder.
///
/// Strictly speaking, there can be multiple of these per process. There is one for each binder fd
/// that a process has opened, so processes using several binder contexts have several `Process`
/// objects. This ensures that the contexts are fully separated.
#[pin_data]
pub(crate) struct Process {
    pub(crate) ctx: Arc<Context>,

    // The task leader (process).
    pub(crate) task: ARef<Task>,

    // Credential associated with file when `Process` is created.
    pub(crate) cred: ARef<Credential>,

    #[pin]
    pub(crate) inner: SpinLock<ProcessInner>,

    #[pin]
    pub(crate) pages: ShrinkablePageRange,

    // Waitqueue of processes waiting for all outstanding transactions to be
    // processed.
    #[pin]
    freeze_wait: CondVar,

    // Node references are in a different lock to avoid recursive acquisition when
    // incrementing/decrementing a node in another process.
    #[pin]
    node_refs: SpinLock<ProcessNodeRefs>,

    // Synchronizes `register_wait` calls to the `PollCondVarBox`.
    //
    // The `PollCondVarBox` is not stored here because synchronization is
    // done for `register_wait` only. Wakeups do not take this lock.
    poll: SetOnce<PollCondVarBox>,

    // Work node for deferred work item.
    #[pin]
    defer_work: Work<Process>,

    // Links for process list in Context.
    #[pin]
    links: ListLinks,

    pub(crate) stats: BinderStats,
}

kernel::impl_has_work! {
    impl HasWork<Process> for Process { self.defer_work }
}

kernel::list::impl_list_arc_safe! {
    impl ListArcSafe<0> for Process { untracked; }
}
kernel::list::impl_list_item! {
    impl ListItem<0> for Process {
        using ListLinks { self.links };
    }
}

impl workqueue::WorkItem for Process {
    type Pointer = Arc<Process>;

    fn run(me: Arc<Self>) {
        let defer;
        {
            let mut inner = me.inner.lock();
            defer = inner.defer_work;
            inner.defer_work = DeferWorks::default();
        }

        if defer.contains(DeferWork::Flush) {
            me.deferred_flush();
        }
        if defer.contains(DeferWork::Release) {
            me.deferred_release();
        }
    }
}

impl Process {
    fn new(ctx: Arc<Context>, cred: ARef<Credential>) -> Result<Arc<Self>> {
        let current = kernel::current!();
        let process = Arc::pin_init::<Error>(
            try_pin_init!(Process {
                ctx,
                cred,
                inner <- kernel::new_spinlock!(ProcessInner::new(), "Process::inner"),
                pages <- ShrinkablePageRange::new(&super::BINDER_SHRINKER),
                node_refs <- kernel::new_spinlock!(ProcessNodeRefs::new(), "Process::node_refs"),
                freeze_wait <- kernel::new_condvar!("Process::freeze_wait"),
                task: current.group_leader().into(),
                defer_work <- kernel::new_work!("Process::defer_work"),
                links <- ListLinks::new(),
                stats: BinderStats::new(),
                poll: SetOnce::new(),
            }),
            GFP_KERNEL,
        )?;

        process.ctx.register_process(process.clone())?;

        Ok(process)
    }

    pub(crate) fn pid_in_current_ns(&self) -> kernel::task::Pid {
        self.task.tgid_nr_ns(None)
    }

    #[inline(never)]
    pub(crate) fn debug_print_stats(&self, m: &SeqFile, ctx: &Context) -> Result<()> {
        seq_print!(m, "proc {}\n", self.pid_in_current_ns());
        seq_print!(m, "context {}\n", &*ctx.name);

        let inner = self.inner.lock();
        seq_print!(m, "  threads: {}\n", inner.threads.iter().count());
        seq_print!(
            m,
            "  requested threads: {}+{}/{}\n",
            inner.requested_thread_count,
            inner.started_thread_count,
            inner.max_threads,
        );
        if let Some(mapping) = &inner.mapping {
            seq_print!(
                m,
                "  free oneway space: {}\n",
                mapping.alloc.free_oneway_space()
            );
            seq_print!(m, "  buffers: {}\n", mapping.alloc.count_buffers());
        }
        seq_print!(
            m,
            "  outstanding transactions: {}\n",
            inner.outstanding_txns
        );
        seq_print!(m, "  nodes: {}\n", inner.nodes.iter().count());
        drop(inner);

        {
            let mut refs = self.node_refs.lock();
            let (mut count, mut weak, mut strong) = (0, 0, 0);
            for r in refs.by_handle.values_mut() {
                let node_ref = r.node_ref();
                let (nstrong, nweak) = node_ref.get_count();
                count += 1;
                weak += nweak;
                strong += nstrong;
            }
            seq_print!(m, "  refs: {count} s {strong} w {weak}\n");
        }

        self.stats.debug_print("  ", m);

        Ok(())
    }

    #[inline(never)]
    pub(crate) fn debug_print(&self, m: &SeqFile, ctx: &Context, print_all: bool) -> Result<()> {
        seq_print!(m, "proc {}\n", self.pid_in_current_ns());
        seq_print!(m, "context {}\n", &*ctx.name);

        let mut all_threads = KVec::new();
        let mut all_nodes = KVec::new();
        loop {
            let inner = self.inner.lock();
            let num_threads = inner.threads.iter().count();
            let num_nodes = inner.nodes.iter().count();

            if all_threads.capacity() < num_threads || all_nodes.capacity() < num_nodes {
                drop(inner);
                all_threads.reserve(num_threads, GFP_KERNEL)?;
                all_nodes.reserve(num_nodes, GFP_KERNEL)?;
                continue;
            }

            for thread in inner.threads.values() {
                assert!(all_threads.len() < all_threads.capacity());
                let _ = all_threads.push(thread.clone(), GFP_ATOMIC);
            }

            for node in inner.nodes.values() {
                assert!(all_nodes.len() < all_nodes.capacity());
                let _ = all_nodes.push(node.clone(), GFP_ATOMIC);
            }

            break;
        }

        for thread in all_threads {
            thread.debug_print(m, print_all)?;
        }

        let mut inner = self.inner.lock();
        for node in all_nodes {
            if print_all || node.has_oneway_transaction(&mut inner) {
                node.full_debug_print(m, &mut inner)?;
            }
        }
        drop(inner);

        if print_all {
            let mut refs = self.node_refs.lock();
            for r in refs.by_handle.values_mut() {
                let node_ref = r.node_ref();
                let dead = node_ref.node.owner.inner.lock().is_dead;
                let (strong, weak) = node_ref.get_count();
                let debug_id = node_ref.node.debug_id;

                seq_print!(
                    m,
                    "  ref {}: desc {} {}node {debug_id} s {strong} w {weak}",
                    r.debug_id,
                    r.handle,
                    if dead { "dead " } else { "" }
                );
            }
        }

        let inner = self.inner.lock();
        for work in &inner.work {
            work.debug_print(m, "  ", "  pending transaction ")?;
        }
        for _death in &inner.delivered_deaths {
            seq_print!(m, "  has delivered dead binder\n");
        }
        if let Some(mapping) = &inner.mapping {
            mapping.alloc.debug_print(m)?;
        }
        drop(inner);

        Ok(())
    }

    /// Attempts to fetch a work item from the process queue.
    pub(crate) fn get_work(&self) -> Option<DLArc<dyn DeliverToRead>> {
        self.inner.lock().work.pop_front()
    }

    /// Attempts to fetch a work item from the process queue. If none is available, it registers the
    /// given thread as ready to receive work directly.
    ///
    /// This must only be called when the thread is not participating in a transaction chain; when
    /// it is, work will always be delivered directly to the thread (and not through the process
    /// queue).
    pub(crate) fn get_work_or_register<'a>(
        &'a self,
        thread: &'a Arc<Thread>,
    ) -> GetWorkOrRegister<'a> {
        let mut inner = self.inner.lock();
        // Try to get work from the process queue.
        if let Some(work) = inner.work.pop_front() {
            return GetWorkOrRegister::Work(work);
        }

        // Register the thread as ready.
        GetWorkOrRegister::Register(Registration::new(thread, &mut inner))
    }

    fn get_current_thread(self: ArcBorrow<'_, Self>) -> Result<Arc<Thread>> {
        let id = {
            let current = kernel::current!();
            if self.task != current.group_leader() {
                pr_err!("get_current_thread was called from the wrong process.");
                return Err(EINVAL);
            }
            current.pid()
        };

        {
            let inner = self.inner.lock();
            if let Some(thread) = inner.threads.get(&id) {
                return Ok(thread.clone());
            }
        }

        // Allocate a new `Thread` without holding any locks.
        let reservation = RBTreeNodeReservation::new(GFP_KERNEL)?;
        let ta: Arc<Thread> = Thread::new(id, self.into())?;

        let mut inner = self.inner.lock();
        match inner.threads.entry(id) {
            rbtree::Entry::Vacant(entry) => {
                entry.insert(ta.clone(), reservation);
                Ok(ta)
            }
            rbtree::Entry::Occupied(_entry) => {
                pr_err!("Cannot create two threads with the same id.");
                Err(EINVAL)
            }
        }
    }

    pub(crate) fn push_work(&self, work: DLArc<dyn DeliverToRead>) -> BinderResult {
        // If push_work fails, drop the work item outside the lock.
        let res = self.inner.lock().push_work(self, work);
        match res {
            Ok(()) => Ok(()),
            Err((err, work)) => {
                drop(work);
                Err(err)
            }
        }
    }

    fn set_as_manager(
        self: ArcBorrow<'_, Self>,
        info: Option<FlatBinderObject>,
        thread: &Thread,
    ) -> Result {
        let (ptr, cookie, flags) = if let Some(obj) = info {
            (
                // SAFETY: The object type for this ioctl is implicitly `BINDER_TYPE_BINDER`, so it
                // is safe to access the `binder` field.
                unsafe { obj.__bindgen_anon_1.binder },
                obj.cookie,
                obj.flags,
            )
        } else {
            (0, 0, 0)
        };
        let node_ref = self.get_node(ptr, cookie, flags, true, thread)?;
        let node = node_ref.node.clone();
        self.ctx.set_manager_node(node_ref)?;
        self.inner.lock().is_manager = true;

        // Force the state of the node to prevent the delivery of acquire/increfs.
        let mut owner_inner = node.owner.inner.lock();
        node.force_has_count(&mut owner_inner);
        Ok(())
    }

    fn get_node_inner(
        self: ArcBorrow<'_, Self>,
        ptr: u64,
        cookie: u64,
        flags: u32,
        strong: bool,
        thread: &Thread,
        wrapper: Option<CritIncrWrapper>,
    ) -> Result<Result<NodeRef, CouldNotDeliverCriticalIncrement>> {
        // Try to find an existing node.
        {
            let mut inner = self.inner.lock();
            if let Some(node) = inner.get_existing_node(ptr, cookie)? {
                return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper));
            }
        }

        // Allocate the node before reacquiring the lock.
        let node = DTRWrap::arc_pin_init(Node::new(ptr, cookie, flags, self.into()))?.into_arc();
        let rbnode = RBTreeNode::new(ptr, node.clone(), GFP_KERNEL)?;
        let mut inner = self.inner.lock();
        if let Some(node) = inner.get_existing_node(ptr, cookie)? {
            return Ok(inner.new_node_ref_with_thread(node, strong, thread, wrapper));
        }

        inner.nodes.insert(rbnode);
        // This can only fail if someone has already pushed the node to a list, but we just created
        // it and still hold the lock, so it can't fail right now.
        let node_ref = inner
            .new_node_ref_with_thread(node, strong, thread, wrapper)
            .unwrap();

        Ok(Ok(node_ref))
    }

    pub(crate) fn get_node(
        self: ArcBorrow<'_, Self>,
        ptr: u64,
        cookie: u64,
        flags: u32,
        strong: bool,
        thread: &Thread,
    ) -> Result<NodeRef> {
        let mut wrapper = None;
        for _ in 0..2 {
            match self.get_node_inner(ptr, cookie, flags, strong, thread, wrapper) {
                Err(err) => return Err(err),
                Ok(Ok(node_ref)) => return Ok(node_ref),
                Ok(Err(CouldNotDeliverCriticalIncrement)) => {
                    wrapper = Some(CritIncrWrapper::new()?);
                }
            }
        }
        // We only get a `CouldNotDeliverCriticalIncrement` error if `wrapper` is `None`, so the
        // loop should run at most twice.
        unreachable!()
    }

    pub(crate) fn insert_or_update_handle(
        self: ArcBorrow<'_, Process>,
        node_ref: NodeRef,
        is_manager: bool,
    ) -> Result<u32> {
        {
            let mut refs = self.node_refs.lock();

            // Do a lookup before inserting.
            if let Some(handle_ref) = refs.by_node.get(&node_ref.node.global_id()) {
                let handle = *handle_ref;
                let info = refs.by_handle.get_mut(&handle).unwrap();
                info.node_ref().absorb(node_ref);
                return Ok(handle);
            }
        }

        // Reserve memory for tree nodes.
        let reserve1 = RBTreeNodeReservation::new(GFP_KERNEL)?;
        let reserve2 = RBTreeNodeReservation::new(GFP_KERNEL)?;
        let info = UniqueArc::new_uninit(GFP_KERNEL)?;

        let mut refs_lock = self.node_refs.lock();
        let mut refs = &mut *refs_lock;

        let (unused_id, by_handle_slot) = loop {
            // ID 0 may only be used by the manager.
            let start = if is_manager { 0 } else { 1 };

            if let Some(res) = refs.handle_is_present.find_unused_id(start) {
                match refs.by_handle.entry(res.as_u32()) {
                    rbtree::Entry::Vacant(entry) => break (res, entry),
                    rbtree::Entry::Occupied(_) => {
                        pr_err!("Detected mismatch between handle_is_present and by_handle");
                        res.acquire();
                        kernel::warn_on!(true);
                        return Err(EINVAL);
                    }
                }
            }

            let grow_request = refs.handle_is_present.grow_request().ok_or(ENOMEM)?;
            drop(refs_lock);
            let resizer = grow_request.realloc(GFP_KERNEL)?;
            refs_lock = self.node_refs.lock();
            refs = &mut *refs_lock;
            refs.handle_is_present.grow(resizer);
        };
        let handle = unused_id.as_u32();

        // Do a lookup again as node may have been inserted before the lock was reacquired.
        let by_node_slot = match refs.by_node.entry(node_ref.node.global_id()) {
            rbtree::Entry::Vacant(by_node_slot) => by_node_slot,
            rbtree::Entry::Occupied(handle_ref) => {
                // The node was inserted by another thread while we didn't hold the lock.
                let handle = handle_ref.get();
                let info = refs.by_handle.get_mut(handle).unwrap();
                info.node_ref().absorb(node_ref);
                return Ok(*handle);
            }
        };

        let (info_proc, info_node) = {
            let info_init = NodeRefInfo::new(node_ref, handle, self.into());
            match info.pin_init_with(info_init) {
                Ok(info) => ListArc::pair_from_pin_unique(info),
                // error is infallible
                Err(err) => match err {},
            }
        };

        // Ensure the process is still alive while we insert a new reference.
        //
        // This releases the lock before inserting the nodes, but since `is_dead` is set as the
        // first thing in `deferred_release`, process cleanup will not miss the items inserted into
        // `refs` below.
        if self.inner.lock().is_dead {
            // Explicitly drop the lock so that `info_proc` and `info_node` are dropped outside of
            // the lock.
            drop(refs_lock);
            return Err(ESRCH);
        }

        // SAFETY: `info_proc` and `info_node` reference the same node, so we are inserting
        // `info_node` into the right node's `refs` list.
        unsafe { info_proc.node_ref2().node.insert_node_info(info_node) };

        by_node_slot.insert(handle, reserve1);
        by_handle_slot.insert(info_proc, reserve2);
        unused_id.acquire();
        Ok(handle)
    }

    pub(crate) fn get_transaction_node(&self, handle: u32) -> BinderResult<NodeRef> {
        // When handle is zero, try to get the context manager.
        if handle == 0 {
            let node_ref = self.ctx.get_manager_node(true)?;
            if core::ptr::eq(self, &*node_ref.node.owner) {
                return Err(EINVAL.into());
            }
            Ok(node_ref)
        } else {
            match self.get_node_from_handle(handle, true) {
                Ok(node_ref) => Ok(node_ref),
                Err(err) => {
                    binder_debug!(UserError, "got transaction to invalid handle {handle}");
                    Err(err.into())
                }
            }
        }
    }

    pub(crate) fn get_node_from_handle(&self, handle: u32, strong: bool) -> Result<NodeRef> {
        self.node_refs
            .lock()
            .by_handle
            .get_mut(&handle)
            .ok_or(ENOENT)?
            .node_ref()
            .clone(strong)
    }

    pub(crate) fn remove_from_delivered_deaths(&self, death: &DArc<NodeDeath>) {
        let mut inner = self.inner.lock();
        // SAFETY: By the invariant on the `delivered_links` field, this is the right linked list.
        let removed = unsafe { inner.delivered_deaths.remove(death) };
        drop(inner);
        drop(removed);
    }

    pub(crate) fn update_ref(
        self: ArcBorrow<'_, Process>,
        handle: u32,
        inc: bool,
        strong: bool,
    ) -> Result {
        if inc && handle == 0 {
            if let Ok(node_ref) = self.ctx.get_manager_node(strong) {
                if core::ptr::eq(&*self, &*node_ref.node.owner) {
                    return Err(EINVAL);
                }
                let _ = self.insert_or_update_handle(node_ref, true);
                return Ok(());
            }
        }

        // To preserve original binder behaviour, we only fail requests where the manager tries to
        // increment references on itself.
        let _to_free_by_handle;
        let _to_free_by_node;
        let _to_free_freeze_listener;
        let _to_free_freeze_listener_cleanup;
        let mut refs = self.node_refs.lock();
        if let Some(info) = refs.by_handle.get_mut(&handle) {
            if info.node_ref().update(inc, strong) {
                // Clean up death if there is one attached to this node reference.
                //
                // We remove the entire `info` below, so no need to remove `death` from `info`.
                if let Some(death) = info.death().as_ref() {
                    death.set_cleared(true);
                    self.remove_from_delivered_deaths(death);
                }

                // Remove reference from process tables, and from the node's `refs` list.

                // SAFETY: We are removing the `NodeRefInfo` from the right node.
                unsafe { info.node_ref2().node.remove_node_info(info) };

                let id = info.node_ref().node.global_id();