ax-task 0.7.0

OS-independent IRQ-safe SMP task scheduling core
Documentation 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106

//! Intrusive multi-producer, single-consumer coroutine inboxes.

use core::{ptr, sync::atomic::Ordering};

use super::CoroutineHeader;
use crate::runtime::delivery::epoch::EpochMpscQueue;

#[derive(Clone, Copy)]
pub(super) enum InboxKind {
    Ready,
}

pub(super) struct IntrusiveInbox {
    publication: EpochMpscQueue<CoroutineHeader>,
    kind: InboxKind,
}

impl IntrusiveInbox {
    pub(super) const fn new(kind: InboxKind) -> Self {
        Self {
            publication: EpochMpscQueue::new(),
            kind,
        }
    }

    pub(super) fn is_empty(&self) -> bool {
        self.publication.is_empty()
    }

    /// Publishes one node into this multi-producer inbox.
    ///
    /// # Safety
    ///
    /// `header` must be pinned, live through its queue reference, and absent from
    /// every list using this inbox's selected intrusive link.
    pub(super) unsafe fn push(&self, header: *mut CoroutineHeader) {
        let next = unsafe {
            // Caller guarantees exclusive membership for this node and inbox.
            (*header).next(self.kind)
        };
        unsafe {
            // RUN_QUEUED owns this pinned node and its ready link until the
            // epoch-graced publication transfers that queue membership.
            self.publication.publish(header, next);
        }
    }

    /// Detaches and FIFO-orders every node currently visible to the consumer.
    ///
    /// # Safety
    ///
    /// This function must be called only by the inbox's single owner consumer.
    pub(super) unsafe fn take_fifo(&self) -> *mut CoroutineHeader {
        let stack = unsafe {
            // The executor owner is the only consumer. A null result can also
            // mean that the retired head is waiting for an in-flight publisher.
            self.publication.take_graced_stack()
        };
        unsafe {
            // Completed epoch grace transfers every retained pointer provenance
            // with the detached stack to this single consumer.
            reverse(stack, self.kind)
        }
    }

    /// Removes the next link from a detached list node.
    ///
    /// # Safety
    ///
    /// The caller must exclusively own `header` in a detached list of `kind`.
    pub(super) unsafe fn take_next(
        header: *mut CoroutineHeader,
        kind: InboxKind,
    ) -> *mut CoroutineHeader {
        unsafe {
            // Caller owns a detached node from the selected inbox.
            (*header)
                .next(kind)
                .swap(ptr::null_mut(), Ordering::Relaxed)
        }
    }
}

/// Reverses an exclusively detached intrusive list.
///
/// # Safety
///
/// Every node reachable from `cursor` must be live and exclusively owned by the
/// single consumer through the intrusive link selected by `kind`.
unsafe fn reverse(mut cursor: *mut CoroutineHeader, kind: InboxKind) -> *mut CoroutineHeader {
    let mut reversed = ptr::null_mut();
    while !cursor.is_null() {
        let next = unsafe {
            // The detached list is exclusively owned by the consumer.
            (*cursor).next(kind).load(Ordering::Relaxed)
        };
        unsafe {
            // No producer can access this node until a future publication, which
            // requires it to leave the detached list first.
            (*cursor).next(kind).store(reversed, Ordering::Relaxed);
        }
        reversed = cursor;
        cursor = next;
    }
    reversed
}