dope_core/driver/schedule/ready/
mod.rs

1use std::{marker, mem, num};
2
3pub(crate) use arena::Arena;
4use arena::{
5    cells::{Entry, Renewal, Resident, Slots, Table},
6    layout, payload, pool, topology,
7};
8pub use key::retained::Retained;
9use key::{
10    Detached, DispatchKey, FreeIndex, Index, ReservedKey, RootIndex, Staged, TaskIndex, TaskKey,
11    TopologyKey,
12};
13pub(crate) use key::{DynamicIndex, QueueIndex, Stored};
14use kind::Kind;
15use layout::Layout;
16use o3::{
17    base::{marker::ThreadBound, num::Generation},
18    mem::Lease,
19};
20use payload::{Link, Task};
21use pool::{Chain, Pool};
22use record::Record;
23pub use rounds::Rounds;
24use topology::Topology;
25
26use crate::{driver, driver::route, io::fd::identity};
27
28mod arena;
29mod record;
30mod rounds;
31
32pub(super) enum Lane {}
33
34/// Proof that a retained ready generation no longer occupies its slot.
35#[derive(Clone, Copy, Debug, PartialEq, Eq)]
36pub struct Stale;
37
38/// Readiness observed while taking one exact live slot.
39#[derive(Clone, Copy, Eq, PartialEq)]
40#[repr(u8)]
41pub enum Readiness {
42    Idle,
43    Ready,
44}
45
46/// Dynamic ready capacity was exhausted.
47#[derive(Clone, Copy, Debug, PartialEq, Eq)]
48#[repr(transparent)]
49pub struct Capacity {
50    available: u32,
51}
52
53impl Capacity {
54    const fn new(free: u32) -> Self {
55        Self { available: free }
56    }
57
58    #[must_use]
59    pub const fn available(self) -> usize {
60        self.available as usize
61    }
62}
63
64#[derive(Clone, Copy)]
65pub(in crate::driver) struct Access<'a, 'd> {
66    borrow: marker::PhantomData<fn(&'a ()) -> &'a ()>,
67    driver: driver::Invariant<'d>,
68}
69
70impl<'d> Access<'_, 'd> {
71    pub(in crate::driver) fn new<'a>(driver: &'a driver::Reference<'d>) -> Access<'a, 'd>
72    where
73        'd: 'a,
74    {
75        Access {
76            borrow: marker::PhantomData,
77            driver: driver.brand,
78        }
79    }
80
81    fn bind(self, stored: Stored) -> Key<'d> {
82        Key {
83            stored,
84            arena: self.driver,
85            thread: ThreadBound::NEW,
86        }
87    }
88}
89
90enum TaskParent<'d> {
91    Root,
92    Task(Key<'d>),
93    Stale,
94}
95
96#[doc(hidden)]
97pub mod task;
98pub mod wake;
99
100/// A generation-checked address in a driver's local ready queue.
101#[derive(Clone, Copy)]
102#[repr(transparent)]
103pub struct Key<'d> {
104    stored: Stored,
105    arena: driver::Invariant<'d>,
106    thread: ThreadBound,
107}
108
109impl PartialEq for Key<'_> {
110    fn eq(&self, other: &Self) -> bool {
111        self.stored == other.stored
112    }
113}
114
115impl Eq for Key<'_> {}
116
117impl<'d> Key<'d> {
118    fn from_index(index: num::NonZero<u16>, epoch: Generation) -> Self {
119        Self {
120            stored: Stored::from_parts(index, epoch),
121            arena: driver::Invariant(marker::PhantomData),
122            thread: ThreadBound::NEW,
123        }
124    }
125
126    pub(in crate::driver) const fn raw(self) -> Stored {
127        self.stored
128    }
129}
130
131#[derive(Clone, Copy)]
132struct Address<'d> {
133    driver: driver::Reference<'d>,
134    key: Retained<'d>,
135}
136
137impl PartialEq for Address<'_> {
138    fn eq(&self, other: &Self) -> bool {
139        self.key == other.key
140    }
141}
142
143impl Eq for Address<'_> {}
144
145impl<'d> Address<'d> {
146    fn new(driver: driver::Reference<'d>, key: Key<'d>) -> Self {
147        Self {
148            driver,
149            key: Retained(key),
150        }
151    }
152
153    fn wake(self) {
154        self.driver.ready().activate(self.key.0);
155    }
156
157    fn belongs_to(self, driver: driver::Reference<'_>) -> bool {
158        self.driver.is_same(driver)
159    }
160
161    fn key(self) -> Key<'d> {
162        self.key.0
163    }
164
165    fn retarget(self, key: Key<'d>) -> Self {
166        Self::new(self.driver, key)
167    }
168}
169
170/// A generation-checked driver readiness target that cannot name a task node.
171#[derive(Clone, Copy)]
172#[repr(transparent)]
173pub struct Target<'d> {
174    address: Address<'d>,
175}
176
177impl PartialEq for Target<'_> {
178    fn eq(&self, other: &Self) -> bool {
179        self.address == other.address
180    }
181}
182
183impl Eq for Target<'_> {}
184
185impl<'d> Target<'d> {
186    pub(crate) fn new(driver: driver::Reference<'d>, key: Key<'d>) -> Self {
187        Self {
188            address: Address::new(driver, key),
189        }
190    }
191
192    pub fn wake(self) {
193        self.address.wake();
194    }
195
196    #[doc(hidden)]
197    pub fn key(&self) -> Retained<'d> {
198        self.address.key
199    }
200
201    pub(in crate::driver) fn receive(self) -> (driver::Reference<'d>, Stored) {
202        (self.address.driver, self.address.key.raw())
203    }
204
205    fn address(self) -> Address<'d> {
206        self.address
207    }
208}
209
210#[derive(Clone, Copy)]
211pub struct Handle<'d> {
212    driver: driver::Reference<'d>,
213    id: identity::Identity<'d>,
214}
215
216impl<'d> Handle<'d> {
217    pub(in crate::driver) const fn new(
218        driver: driver::Reference<'d>,
219        id: identity::Identity<'d>,
220    ) -> Self {
221        Self { driver, id }
222    }
223
224    pub fn set_target<Tag: route::Tag>(self, target: route::Operation<'d, Tag>) {
225        self.driver
226            .descriptors()
227            .set_target(self.id, target.into_token());
228    }
229
230    pub fn activate(self) {
231        if self.driver.descriptors().target(self.id).is_some() {
232            self.driver.ready().arena().activate_fd(self.id);
233        }
234    }
235
236    pub fn key(self) -> Key<'d> {
237        let index = self.id.queue_index();
238        Key::from_index(index.0, self.id.generation())
239    }
240
241    pub fn target(self) -> Target<'d> {
242        Target::new(self.driver, self.key())
243    }
244}
245
246/// One ready slot branded by both driver domain and route tag.
247///
248pub struct Slot<'d, Tag: route::Tag> {
249    owner: Lease<DispatchKey<'d>>,
250    target: marker::PhantomData<*mut Tag>,
251}
252
253impl<'d, Tag: route::Tag> Slot<'d, Tag> {
254    fn new(driver: driver::Reference<'d>, key: DispatchKey<'d>) -> Self {
255        Self {
256            owner: Lease::new(key, driver),
257            target: marker::PhantomData,
258        }
259    }
260
261    pub fn activate(&self) {
262        self.owner
263            .item()
264            .ready()
265            .arena()
266            .activate_slot(self.dispatch_key());
267    }
268
269    /// Takes readiness for this exact live slot without draining other entries.
270    #[must_use]
271    pub fn take(&self) -> Readiness {
272        self.owner.item().ready().arena().take(self.dispatch_key())
273    }
274
275    /// Returns the driver- and generation-branded root owned by this slot.
276    pub fn target(&self) -> Target<'d> {
277        Target::new(*self.owner.item(), self.dispatch_key().key())
278    }
279
280    fn dispatch_key(&self) -> &DispatchKey<'d> {
281        self.owner.sink()
282    }
283}
284
285const _: () = {
286    assert!(mem::size_of::<Readiness>() == mem::size_of::<u8>());
287    assert!(mem::size_of::<usize>() >= mem::size_of::<u32>());
288    assert!(mem::size_of::<Access<'static, 'static>>() == 0);
289};
290const _: () = {
291    assert!(mem::size_of::<Key<'static>>() == mem::size_of::<u64>());
292    assert!(mem::size_of::<Option<Key<'static>>>() == mem::size_of::<Key<'static>>());
293    assert!(mem::size_of::<Option<Retained<'static>>>() == mem::size_of::<Retained<'static>>());
294    assert!(mem::size_of::<DispatchKey<'static>>() == mem::size_of::<Key<'static>>());
295    assert!(mem::align_of::<DispatchKey<'static>>() == mem::align_of::<Key<'static>>());
296    assert!(mem::size_of::<Slot<'static, route::KeyTag<0>>>() == 2 * mem::size_of::<usize>());
297};
298const _: () = assert!(mem::size_of::<Target<'static>>() == 2 * mem::size_of::<usize>());
299const _: () = {
300    assert!(mem::size_of::<Handle<'static>>() == 2 * mem::size_of::<usize>());
301};
302
303mod key;
304mod kind;
305mod state;
306
307#[derive(Clone, Copy)]
308#[repr(transparent)]
309pub struct Context<'d>(driver::Reference<'d>);
310
311impl<'d> Context<'d> {
312    pub(in crate::driver) const fn new(driver: driver::Reference<'d>) -> Self {
313        Self(driver)
314    }
315
316    pub(crate) fn arena(self) -> &'d Arena {
317        self.0.shared.context().schedule.owner.context()
318    }
319
320    pub(crate) fn handle(self, id: identity::Identity<'d>) -> Handle<'d> {
321        Handle::new(self.0, id)
322    }
323
324    pub fn make_slot<Tag: route::Tag>(
325        self,
326        target: route::Operation<'d, Tag>,
327    ) -> Result<Slot<'d, Tag>, Capacity> {
328        self.arena().make_slot(self.0, target)
329    }
330
331    /// Activates a key branded by this exact driver domain.
332    ///
333    pub fn activate(self, key: Key<'d>) {
334        self.arena().activate(Access::new(&self.0), key);
335    }
336
337    /// Activates a retained key when its generation is still current.
338    pub fn try_activate(self, key: Retained<'d>) -> Result<(), Stale> {
339        self.arena().try_retained(key)
340    }
341
342    pub fn has_ready(self) -> bool {
343        self.arena().has_ready()
344    }
345}
346
347const _: () = assert!(
348    std::mem::size_of::<Context<'static>>() == std::mem::size_of::<driver::Reference<'static>>()
349);
dope_core/driver/schedule/ready/mod.rs

dope_core/driver/schedule/ready/
mod.rs
