dope-core 0.15.0

The manifold runtime
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
use std::{num, process};

use crate::{
    driver::route::table,
    io::{datagram, socket::option, transfer},
};

mod config;

pub use config::Config;

const MIN_SUBMISSION_CAPACITY: u32 = (option::MAX_STREAM_OPTIONS + 1) as u32;
const MAX_SUBMISSION_CAPACITY: u32 = 32_768;
const MAX_COMPLETION_CAPACITY: u32 = 65_536;

/// Backend-neutral intent for making completed work visible to the driver.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(u8)]
pub enum CompletionProgress {
    /// Avoid voluntarily deferring backend completion work.
    Prompt,
    /// Permit a backend to coalesce completion work until the next driver entry.
    BatchedWhenSupported,
}

const _: () = {
    assert!(std::mem::size_of::<CompletionProgress>() == std::mem::size_of::<bool>());
    assert!(std::mem::align_of::<CompletionProgress>() == std::mem::align_of::<bool>());
};

/// Static driver policy with a validated receive-buffer layout.
///
/// Invalid layouts cannot become profile constants:
///
/// ```compile_fail
/// use dope_core::driver::settings::{Profile, QueueLayout, Receive};
///
/// struct Invalid;
/// impl Profile for Invalid {
///     const QUEUES: QueueLayout = QueueLayout::fixed::<64, 128>();
///     const RECEIVE: Receive = Receive::fixed::<3, 4096>();
/// }
///
/// let _ = Invalid::RECEIVE;
/// ```
pub trait Profile: 'static {
    const QUEUES: QueueLayout;
    const SCHEDULER: SchedulerLayout = SchedulerLayout::DEFAULT;
    const COMPLETION_PROGRESS: CompletionProgress = CompletionProgress::BatchedWhenSupported;

    /// Maximum listener-owned accept slots realized by `for_tcp_profile`.
    const MAX_ACCEPT_SLOTS: u32 = 65_279;
    /// Descriptor slots available to outbound pools and standalone files.
    const OUTBOUND_SLOTS: u32 = 256;
    const RECEIVE: Receive = Receive::DEFAULT;
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(transparent)]
struct SubmissionCapacity(u32);

impl SubmissionCapacity {
    const fn new(entries: u32) -> Option<Self> {
        if entries == 0 {
            return None;
        }
        if entries < MIN_SUBMISSION_CAPACITY
            || !entries.is_power_of_two()
            || entries > MAX_SUBMISSION_CAPACITY
        {
            return None;
        }
        Some(Self(entries))
    }

    const fn get(self) -> u32 {
        self.0
    }
}

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct QueueLayout {
    submissions: SubmissionCapacity,
    completions: u32,
}

struct FixedQueueLayout<const SUBMISSIONS: u32, const COMPLETIONS: u32>;

impl<const SUBMISSIONS: u32, const COMPLETIONS: u32> FixedQueueLayout<SUBMISSIONS, COMPLETIONS> {
    const VALUE: QueueLayout = {
        assert!(SUBMISSIONS >= MIN_SUBMISSION_CAPACITY);
        assert!(SUBMISSIONS.is_power_of_two());
        assert!(SUBMISSIONS <= MAX_SUBMISSION_CAPACITY);
        assert!(COMPLETIONS.is_power_of_two());
        assert!(COMPLETIONS <= MAX_COMPLETION_CAPACITY);
        assert!(COMPLETIONS >= SUBMISSIONS);
        QueueLayout {
            submissions: SubmissionCapacity(SUBMISSIONS),
            completions: COMPLETIONS,
        }
    };
}

impl QueueLayout {
    pub const MAX: Self = Self::fixed::<MAX_SUBMISSION_CAPACITY, MAX_COMPLETION_CAPACITY>();

    /// Creates a queue layout validated during compilation.
    ///
    /// ```compile_fail
    /// use dope_core::driver::settings::QueueLayout;
    ///
    /// let _ = QueueLayout::fixed::<64, 32>();
    /// ```
    pub const fn fixed<const SUBMISSIONS: u32, const COMPLETIONS: u32>() -> Self {
        FixedQueueLayout::<SUBMISSIONS, COMPLETIONS>::VALUE
    }

    /// Creates power-of-two backend queues. Submissions cover atomic stream
    /// setup and cap at 32,768; completions cover submissions and cap at 65,536.
    #[must_use]
    pub const fn new(submissions: u32, completions: u32) -> Option<Self> {
        let Some(submissions) = SubmissionCapacity::new(submissions) else {
            return None;
        };
        if completions == 0 {
            return None;
        }
        if !completions.is_power_of_two()
            || completions > MAX_COMPLETION_CAPACITY
            || completions < submissions.get()
        {
            return None;
        }
        Some(Self {
            submissions,
            completions,
        })
    }

    #[must_use]
    pub const fn submissions(self) -> u32 {
        self.submissions.get()
    }

    #[must_use]
    pub const fn completions(self) -> u32 {
        self.completions
    }
}

const _: () = {
    assert!(QueueLayout::new(0, 8).is_none());
    assert!(QueueLayout::new(MIN_SUBMISSION_CAPACITY / 2, 8).is_none());
    assert!(QueueLayout::new(64, 32).is_none());
    assert!(QueueLayout::MAX.submissions() == MAX_SUBMISSION_CAPACITY);
    assert!(QueueLayout::MAX.completions() == MAX_COMPLETION_CAPACITY);
    assert!(std::mem::size_of::<QueueLayout>() == std::mem::size_of::<[u32; 2]>());
};

/// Scheduler-owned index capacity matching ready and timer slot indices.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
#[repr(transparent)]
pub struct ScheduleCapacity(u32);

impl ScheduleCapacity {
    pub const ZERO: Self = Self(0);
    pub const MAX: Self = Self(u32::MAX);

    /// Creates a scheduler capacity represented directly by its internal index.
    pub const fn fixed<const SLOTS: u32>() -> Self {
        Self(SLOTS)
    }

    #[must_use]
    pub const fn new(slots: usize) -> Option<Self> {
        if slots <= u32::MAX as usize {
            Some(Self(slots as u32))
        } else {
            None
        }
    }

    #[must_use]
    pub const fn get(self) -> usize {
        self.0 as usize
    }
}

/// Independent ready-slot and timer-cache bounds. Timer registrations beyond
/// the realized cache remain live in the intrusive overflow.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct SchedulerLayout {
    ready: ScheduleCapacity,
    timer_cache_limit: ScheduleCapacity,
}

impl SchedulerLayout {
    pub const DEFAULT: Self = Self::fixed::<1024, 1024>();

    pub const fn fixed<const READY: u32, const TIMER_CACHE: u32>() -> Self {
        Self {
            ready: ScheduleCapacity::fixed::<READY>(),
            timer_cache_limit: ScheduleCapacity::fixed::<TIMER_CACHE>(),
        }
    }

    #[must_use]
    pub const fn new(ready: usize, timer_cache_limit: usize) -> Option<Self> {
        let Some(ready) = ScheduleCapacity::new(ready) else {
            return None;
        };
        let Some(timer_cache_limit) = ScheduleCapacity::new(timer_cache_limit) else {
            return None;
        };
        Some(Self {
            ready,
            timer_cache_limit,
        })
    }

    #[must_use]
    pub const fn ready(self) -> ScheduleCapacity {
        self.ready
    }

    #[must_use]
    pub const fn timer_cache_limit(self) -> ScheduleCapacity {
        self.timer_cache_limit
    }

    #[must_use]
    pub const fn with_ready(mut self, ready: ScheduleCapacity) -> Self {
        self.ready = ready;
        self
    }

    #[must_use]
    pub const fn with_timer_cache_limit(mut self, timer_cache_limit: ScheduleCapacity) -> Self {
        self.timer_cache_limit = timer_cache_limit;
        self
    }
}

const _: () = {
    assert!(ScheduleCapacity::new(0).is_some());
    assert!(ScheduleCapacity::MAX.get() == u32::MAX as usize);
    if let Some(overflow) = (u32::MAX as usize).checked_add(1) {
        assert!(ScheduleCapacity::new(overflow).is_none());
    }
    assert!(std::mem::size_of::<ScheduleCapacity>() == std::mem::size_of::<u32>());
    assert!(std::mem::size_of::<SchedulerLayout>() == std::mem::size_of::<[u32; 2]>());
};

#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct Receive {
    entries: u16,
    len: u32,
}

struct FixedReceive<const ENTRIES: u16, const LEN: u32>;

impl<const ENTRIES: u16, const LEN: u32> FixedReceive<ENTRIES, LEN> {
    const VALUE: Receive = {
        assert!(ENTRIES >= MIN_RECEIVE_ENTRIES);
        assert!(ENTRIES.is_power_of_two());
        assert!(LEN != 0);
        assert!(LEN <= transfer::MAX_BYTES as u32);
        assert!((ENTRIES as usize) <= (isize::MAX as usize) / (LEN as usize));
        Receive {
            entries: ENTRIES,
            len: LEN,
        }
    };
}

const MIN_RECEIVE_ENTRIES: u16 = 2;

const _: () = {
    assert!(std::mem::size_of::<Receive>() == std::mem::size_of::<u64>());
};

impl Receive {
    pub const DEFAULT: Self = Self::fixed::<4096, 4096>();
    pub const MIN_DATAGRAM_BUFFER_LEN: u32 = datagram::SlotLen::MIN_BYTES;

    /// Creates a receive-buffer layout validated during compilation.
    ///
    /// ```compile_fail
    /// use dope_core::driver::settings::Receive;
    ///
    /// let _ = Receive::fixed::<1, 4096>();
    /// ```
    pub const fn fixed<const ENTRIES: u16, const LEN: u32>() -> Self {
        FixedReceive::<ENTRIES, LEN>::VALUE
    }

    /// Creates a cross-backend layout with at least two power-of-two slots.
    /// Slot length must fit transfer fields and total allocation in `isize`.
    #[must_use]
    pub const fn new(entries: u16, len: u32) -> Option<Self> {
        if len == 0 {
            return None;
        }
        if len > transfer::MAX_BYTES as u32 {
            return None;
        }
        Self::with_len(entries, len)
    }

    /// Creates a layout with the requested cross-backend datagram payload capacity.
    #[must_use]
    pub const fn for_datagram_payload(entries: u16, payload_len: u32) -> Option<Self> {
        let Some(len) = datagram::SlotLen::for_payload(payload_len) else {
            return None;
        };
        Self::with_len(entries, len.nonzero().get())
    }

    const fn with_len(entries: u16, len: u32) -> Option<Self> {
        if entries < MIN_RECEIVE_ENTRIES || !entries.is_power_of_two() {
            return None;
        }
        let Some(bytes) = (entries as usize).checked_mul(len as usize) else {
            return None;
        };
        if bytes > isize::MAX as usize {
            return None;
        }
        Some(Self { entries, len })
    }

    #[must_use]
    pub const fn entries(self) -> u16 {
        self.entries
    }

    #[must_use]
    pub const fn buffer_len(self) -> usize {
        self.len as usize
    }

    pub(crate) fn nonzero_buffer_len(self) -> num::NonZeroU32 {
        let Some(len) = num::NonZeroU32::new(self.len) else {
            process::abort();
        };
        len
    }

    pub(crate) const fn backing_bytes(self) -> usize {
        self.entries as usize * self.len as usize
    }
}

/// A fixed-file layout representable by every backend target table.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct FileSlots {
    accept_end: table::Capacity,
    table_end: table::Capacity,
}

struct FixedFileSlots<const ACCEPT: u32, const OUTBOUND: u32>;

impl<const ACCEPT: u32, const OUTBOUND: u32> FixedFileSlots<ACCEPT, OUTBOUND> {
    const VALUE: FileSlots = {
        assert!(ACCEPT <= u32::MAX - OUTBOUND);
        let total = ACCEPT + OUTBOUND;
        assert!(total != 0);
        FileSlots {
            accept_end: table::Capacity::fixed(ACCEPT),
            table_end: table::Capacity::fixed(total),
        }
    };
}

const _: () =
    assert!(std::mem::size_of::<FileSlots>() == std::mem::size_of::<[table::Capacity; 2]>());

impl FileSlots {
    /// Creates a fixed-file layout validated during compilation.
    ///
    /// ```compile_fail
    /// use dope_core::driver::settings::FileSlots;
    ///
    /// let _ = FileSlots::fixed::<16, { u32::MAX }>();
    /// ```
    pub const fn fixed<const ACCEPT: u32, const OUTBOUND: u32>() -> Self {
        FixedFileSlots::<ACCEPT, OUTBOUND>::VALUE
    }

    /// Creates independent accept and outbound domains in one fixed-file table.
    #[must_use]
    pub const fn new(accept: u32, outbound: u32) -> Option<Self> {
        let Some(total) = accept.checked_add(outbound) else {
            return None;
        };
        if total == 0 {
            return None;
        }
        let Some(accept_end) = table::Capacity::new(accept as usize) else {
            return None;
        };
        let Some(table_end) = table::Capacity::new(total as usize) else {
            return None;
        };
        Some(Self {
            accept_end,
            table_end,
        })
    }

    #[must_use]
    pub const fn capacity(self) -> u32 {
        self.table_end.raw()
    }

    #[must_use]
    pub const fn accept(self) -> u32 {
        self.accept_end.raw()
    }

    #[must_use]
    pub const fn outbound(self) -> u32 {
        self.table_end.raw() - self.accept_end.raw()
    }

    pub(crate) const fn table_capacity(self) -> table::Capacity {
        self.table_end
    }

    pub(crate) const fn accept_capacity(self) -> table::Capacity {
        self.accept_end
    }
}