ax-task 0.8.1

OS-independent IRQ-safe SMP task scheduling core
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118

//! Linux-style per-lock PI waiter ownership and bounded chain propagation.

use super::*;
use crate::{
    runtime::{
        lock::PreemptScope,
        sync::{
            PiMutexClaimOutcome, PiMutexLockResult, PiMutexRef, PiWaitCancelOutcome,
            PiWaitStateError,
        },
    },
    thread::{
        PiMutexWaiters, lock_pi_mutex_waiters, lock_raw_pi_mutex_waiters,
        try_lock_raw_pi_mutex_waiters,
    },
};

#[derive(Clone, Copy)]
struct PiRqFollowup {
    reschedule: Option<RescheduleKind>,
    owner_work: bool,
}

struct PiWaiterRefresh {
    owner: Option<ThreadId>,
    owner_next_lock: Option<PiMutexRaw>,
    changed: bool,
    /// The new top waiter of an ownerless lock whose top changed. Linux wakes
    /// this waiter from `rt_mutex_adjust_prio_chain()` step [9]; otherwise no
    /// owner remains to provide the scheduling edge.
    ownerless_wake: Option<PiHandoffWake>,
}

/// A wake selected under wait_lock, retaining the exact PI registration.
struct PiHandoffWake {
    core: Arc<ThreadCore>,
    generation: u64,
    rt_lock_wait: bool,
}

impl PiHandoffWake {
    fn new(core: Arc<ThreadCore>, generation: u64) -> Self {
        let rt_lock_wait = core.in_rt_lock_wait();
        Self {
            core,
            generation,
            rt_lock_wait,
        }
    }

    fn deliver(self, system: &TaskSystem) {
        if self.rt_lock_wait {
            system.wake_rt_lock_thread(&self.core, self.generation);
        } else {
            system.wake_thread_from_current_cpu(&self.core, crate::thread::WakeIntent::Normal);
        }
    }
}

impl TaskSystem {
    /// Acquires a stable task reference without retaining the registry lock.
    ///
    /// This is the local `get_task_struct()` operation used by the PI chain
    /// walk. All PI graph state is protected by the returned task's scheduler
    /// lock, never by the registry lock used for this lookup.
    fn pi_thread_core(&self, thread: ThreadId) -> Result<Arc<ThreadCore>, TaskError> {
        let state = self.state.lock();
        Ok(Arc::clone(&state.thread_record(thread)?.core))
    }

    fn pi_donation(&self, core: &Arc<ThreadCore>) -> Result<PiDonation, TaskError> {
        let (policy, root) = {
            let sched = core.sched().lock();
            (
                core.effective_policy_snapshot(),
                sched.pi.donor.unwrap_or(core.id()),
            )
        };
        self.pi_donation_from_snapshot(core, policy, root)
    }

    fn pi_donation_from_snapshot(
        &self,
        core: &Arc<ThreadCore>,
        policy: SchedulePolicy,
        root: ThreadId,
    ) -> Result<PiDonation, TaskError> {
        let root_core = if root == core.id() {
            Arc::clone(core)
        } else {
            self.pi_thread_core(root)?
        };
        Ok(PiDonation::new(
            policy,
            root,
            core.effective_scheduling_urgency(),
            core,
            &root_core,
        ))
    }
}

mod graph;
mod operations;
mod schedule;
mod transition;

use transition::{
    PiOwnerRqAccountingPath, owner_rq_needs_current_settlement, publish_owner_after_waiter_detach,
};

#[cfg(axtest)]
mod axtest;
#[cfg(axtest)]
pub use {
    axtest::PiScheduleTestProbeSnapshot, axtest::begin_pi_schedule_test_probe,
    axtest::end_pi_schedule_test_probe, axtest::pi_schedule_test_probe_snapshot,
};