dope_core/driver/schedule/ready/
mod.rs

1use std::{marker, mem, num};
2
3pub(crate) use arena::Arena;
4use arena::{
5    cells::{Drain, Entry, Renewal, Resident, Slots, Table},
6    layout, payload, pool, topology,
7};
8pub use key::retained::Retained;
9use key::{
10    Detached, DispatchKey, FreeIndex, Index, ReservedKey, RootIndex, Staged, TaskIndex, TaskKey,
11    TopologyKey,
12};
13pub(crate) use key::{DynamicIndex, QueueIndex, Stored};
14use kind::Kind;
15use layout::Layout;
16use o3::{
17    base::{marker::ThreadBound, num::Generation},
18    depot::queue::Removal,
19    mem::Lease,
20};
21use payload::{Link, Task};
22use pool::{Chain, Pool};
23use record::Record;
24pub use rounds::Rounds;
25use topology::Topology;
26
27use crate::{driver, driver::route, io::fd::identity};
28
29mod arena;
30mod record;
31mod rounds;
32
33pub(super) enum Lane {}
34
35/// Proof that a retained ready generation no longer occupies its slot.
36#[derive(Clone, Copy, Debug, PartialEq, Eq)]
37pub struct Stale;
38
39/// Readiness observed while taking one exact live slot.
40#[derive(Clone, Copy, Eq, PartialEq)]
41#[repr(u8)]
42pub enum Readiness {
43    Idle,
44    Ready,
45}
46
47/// Dynamic ready capacity was exhausted.
48#[derive(Clone, Copy, Debug, PartialEq, Eq)]
49#[repr(transparent)]
50pub struct Capacity {
51    available: u32,
52}
53
54impl Capacity {
55    const fn new(free: u32) -> Self {
56        Self { available: free }
57    }
58
59    #[must_use]
60    pub const fn available(self) -> usize {
61        self.available as usize
62    }
63}
64
65#[derive(Clone, Copy)]
66pub(in crate::driver) struct Access<'a, 'd> {
67    borrow: marker::PhantomData<fn(&'a ()) -> &'a ()>,
68    driver: driver::Invariant<'d>,
69}
70
71impl<'d> Access<'_, 'd> {
72    pub(in crate::driver) fn new<'a>(driver: &'a driver::Reference<'d>) -> Access<'a, 'd>
73    where
74        'd: 'a,
75    {
76        Access {
77            borrow: marker::PhantomData,
78            driver: driver.brand,
79        }
80    }
81
82    fn bind(self, stored: Stored) -> Key<'d> {
83        Key {
84            stored,
85            arena: self.driver,
86            thread: ThreadBound::NEW,
87        }
88    }
89}
90
91enum TaskParent<'d> {
92    Root,
93    Task(Key<'d>),
94    Stale,
95}
96
97#[doc(hidden)]
98pub mod task;
99pub mod wake;
100
101/// A generation-checked address in a driver's local ready queue.
102#[derive(Clone, Copy)]
103#[repr(transparent)]
104pub struct Key<'d> {
105    stored: Stored,
106    arena: driver::Invariant<'d>,
107    thread: ThreadBound,
108}
109
110impl PartialEq for Key<'_> {
111    fn eq(&self, other: &Self) -> bool {
112        self.stored == other.stored
113    }
114}
115
116impl Eq for Key<'_> {}
117
118impl<'d> Key<'d> {
119    fn from_index(index: num::NonZero<u16>, epoch: Generation) -> Self {
120        Self {
121            stored: Stored::from_parts(index, epoch),
122            arena: driver::Invariant(marker::PhantomData),
123            thread: ThreadBound::NEW,
124        }
125    }
126
127    pub(in crate::driver) const fn raw(self) -> Stored {
128        self.stored
129    }
130}
131
132#[derive(Clone, Copy)]
133struct Address<'d> {
134    driver: driver::Reference<'d>,
135    key: Retained<'d>,
136}
137
138impl PartialEq for Address<'_> {
139    fn eq(&self, other: &Self) -> bool {
140        self.key == other.key
141    }
142}
143
144impl Eq for Address<'_> {}
145
146impl<'d> Address<'d> {
147    fn new(driver: driver::Reference<'d>, key: Key<'d>) -> Self {
148        Self {
149            driver,
150            key: Retained(key),
151        }
152    }
153
154    fn wake(self) {
155        self.driver.ready().activate(self.key.0);
156    }
157
158    fn belongs_to(self, driver: driver::Reference<'_>) -> bool {
159        self.driver.is_same(driver)
160    }
161
162    fn key(self) -> Key<'d> {
163        self.key.0
164    }
165
166    fn retarget(self, key: Key<'d>) -> Self {
167        Self::new(self.driver, key)
168    }
169}
170
171/// A generation-checked driver readiness target that cannot name a task node.
172#[derive(Clone, Copy)]
173#[repr(transparent)]
174pub struct Target<'d> {
175    address: Address<'d>,
176}
177
178impl PartialEq for Target<'_> {
179    fn eq(&self, other: &Self) -> bool {
180        self.address == other.address
181    }
182}
183
184impl Eq for Target<'_> {}
185
186impl<'d> Target<'d> {
187    pub(crate) fn new(driver: driver::Reference<'d>, key: Key<'d>) -> Self {
188        Self {
189            address: Address::new(driver, key),
190        }
191    }
192
193    pub fn wake(self) {
194        self.address.wake();
195    }
196
197    #[doc(hidden)]
198    pub fn key(&self) -> Retained<'d> {
199        self.address.key
200    }
201
202    pub(in crate::driver) fn receive(self) -> (driver::Reference<'d>, Stored) {
203        (self.address.driver, self.address.key.raw())
204    }
205
206    fn address(self) -> Address<'d> {
207        self.address
208    }
209}
210
211#[derive(Clone, Copy)]
212pub struct Handle<'d> {
213    driver: driver::Reference<'d>,
214    id: identity::Identity<'d>,
215}
216
217impl<'d> Handle<'d> {
218    pub(in crate::driver) const fn new(
219        driver: driver::Reference<'d>,
220        id: identity::Identity<'d>,
221    ) -> Self {
222        Self { driver, id }
223    }
224
225    pub fn set_target<Tag: route::Tag>(self, target: route::Operation<'d, Tag>) {
226        self.driver
227            .descriptors()
228            .set_target(self.id, target.into_token());
229    }
230
231    pub fn activate(self) {
232        if self.driver.descriptors().target(self.id).is_some() {
233            self.driver.ready().arena().activate_fd(self.id);
234        }
235    }
236
237    pub fn key(self) -> Key<'d> {
238        let index = self.id.queue_index();
239        Key::from_index(index.0, self.id.generation())
240    }
241
242    pub fn target(self) -> Target<'d> {
243        Target::new(self.driver, self.key())
244    }
245}
246
247/// One ready slot branded by both driver domain and route tag.
248///
249pub struct Slot<'d, Tag: route::Tag> {
250    owner: Lease<DispatchKey<'d>>,
251    target: marker::PhantomData<*mut Tag>,
252}
253
254impl<'d, Tag: route::Tag> Slot<'d, Tag> {
255    fn new(driver: driver::Reference<'d>, key: DispatchKey<'d>) -> Self {
256        Self {
257            owner: Lease::new(key, driver),
258            target: marker::PhantomData,
259        }
260    }
261
262    pub fn activate(&self) {
263        self.owner
264            .item()
265            .ready()
266            .arena()
267            .activate_slot(self.dispatch_key());
268    }
269
270    /// Takes readiness for this exact live slot without draining other entries.
271    #[must_use]
272    pub fn take(&self) -> Readiness {
273        match self.owner.item().ready().arena().take(self.dispatch_key()) {
274            Removal::Removed => Readiness::Ready,
275            Removal::Absent => Readiness::Idle,
276        }
277    }
278
279    /// Returns the driver- and generation-branded root owned by this slot.
280    pub fn target(&self) -> Target<'d> {
281        Target::new(*self.owner.item(), self.dispatch_key().key())
282    }
283
284    fn dispatch_key(&self) -> &DispatchKey<'d> {
285        self.owner.sink()
286    }
287}
288
289const _: () = {
290    assert!(mem::size_of::<Readiness>() == mem::size_of::<u8>());
291    assert!(mem::size_of::<usize>() >= mem::size_of::<u32>());
292    assert!(mem::size_of::<Access<'static, 'static>>() == 0);
293};
294const _: () = {
295    assert!(mem::size_of::<Key<'static>>() == mem::size_of::<u64>());
296    assert!(mem::size_of::<Option<Key<'static>>>() == mem::size_of::<Key<'static>>());
297    assert!(mem::size_of::<Option<Retained<'static>>>() == mem::size_of::<Retained<'static>>());
298    assert!(mem::size_of::<DispatchKey<'static>>() == mem::size_of::<Key<'static>>());
299    assert!(mem::align_of::<DispatchKey<'static>>() == mem::align_of::<Key<'static>>());
300    assert!(mem::size_of::<Slot<'static, route::KeyTag<0>>>() == 2 * mem::size_of::<usize>());
301};
302const _: () = assert!(mem::size_of::<Target<'static>>() == 2 * mem::size_of::<usize>());
303const _: () = {
304    assert!(mem::size_of::<Handle<'static>>() == 2 * mem::size_of::<usize>());
305};
306
307mod key;
308mod kind;
309mod state;
310
311#[derive(Clone, Copy)]
312#[repr(transparent)]
313pub struct Context<'d>(driver::Reference<'d>);
314
315impl<'d> Context<'d> {
316    pub(in crate::driver) const fn new(driver: driver::Reference<'d>) -> Self {
317        Self(driver)
318    }
319
320    pub(crate) fn arena(self) -> &'d Arena {
321        self.0.shared.context().schedule.owner.context()
322    }
323
324    pub(crate) fn handle(self, id: identity::Identity<'d>) -> Handle<'d> {
325        Handle::new(self.0, id)
326    }
327
328    pub fn make_slot<Tag: route::Tag>(
329        self,
330        target: route::Operation<'d, Tag>,
331    ) -> Result<Slot<'d, Tag>, Capacity> {
332        self.arena().make_slot(self.0, target)
333    }
334
335    /// Activates a key branded by this exact driver domain.
336    ///
337    pub fn activate(self, key: Key<'d>) {
338        self.arena().activate(Access::new(&self.0), key);
339    }
340
341    /// Activates a retained key when its generation is still current.
342    pub fn try_activate(self, key: Retained<'d>) -> Result<(), Stale> {
343        self.arena().try_retained(key)
344    }
345
346    pub fn has_ready(self) -> bool {
347        self.arena().has_ready()
348    }
349}
350
351const _: () = assert!(
352    std::mem::size_of::<Context<'static>>() == std::mem::size_of::<driver::Reference<'static>>()
353);
dope_core/driver/schedule/ready/mod.rs

dope_core/driver/schedule/ready/
mod.rs
