flynnel 0.6.0

K-aware NUMA-aware work-stealing scheduler with extended-Flynn-taxonomy dispatch (MIMD / SIMC / MIMC / MISD / SIMT / MIMT): a rayon-shaped pool reshaped around per-call execution-class plans with first-class CPU+GPU hybrid (join_hybrid, hybrid_pipeline) and per-call dispatch profiles.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
//! Runtime switches for the noisy-host mechanisms.
//!
//! Every mechanism here has to earn its place against the same code
//! without it: never slower on a quiet host, faster on a contended one.
//! A switch is what makes that measurable. One binary runs both arms, so
//! the comparison carries no difference in commit, harness, build or
//! machine - only the mechanism.
//!
//! # Why a default is off
//!
//! Off is the behavior that shipped. A mechanism defaults on only when a
//! measurement says it should, and until then the crate behaves exactly
//! as it did, so carrying the code costs nothing but the branch it is on.
//! A switch whose default flips on the strength of an argument rather
//! than a reading is how the thing this campaign is fixing got in.
//!
//! Four have earned it: [`calibration_refusal`], [`oncore_spread`],
//! [`allowed_width`] and [`join_park_oversubscribed`] default on, and the
//! reading is on each function.
//! The rest are off and say what would have to be measured for that to
//! change.
//!
//! # Why environment variables rather than features
//!
//! A cargo feature is chosen at build time, so an A/B across one needs
//! two binaries, and two binaries differ in more than the feature. These
//! are read once per process and cached, so a switch costs one relaxed
//! load at a call site and the two arms are the same bytes on disk.
//!
//! # What a switch must not do
//!
//! Change anything but the mechanism it names. A switch that also moves
//! a threshold, or that turns two mechanisms on together, makes its own
//! A/B unreadable and there is no way to tell from the result.

use std::sync::OnceLock;

/// Read a switch once. Absent, empty, `0`, `off` and `false` are off;
/// anything else is on.
///
/// Anything-else-is-on rather than only `1`: a caller who writes `yes`
/// or `true` meant on, and a switch that silently ignored them would run
/// the arm they did not ask for while printing nothing. The values that
/// mean off are the ones a script produces when it means off.
fn read(name: &str) -> bool {
    match std::env::var(name) {
        Ok(text) => {
            let v = text.trim().to_ascii_lowercase();
            !(v.is_empty() || v == "0" || v == "off" || v == "false")
        }
        Err(std::env::VarError::NotPresent) => false,
        Err(std::env::VarError::NotUnicode(raw)) => {
            // Set to something unreadable. Reporting it matters because
            // the alternative is an arm that was asked for, did not run,
            // and produced a row indistinguishable from the other arm.
            notice!(
                "flynnel: {name} is set to something that is not UTF-8 ({raw:?}); \
                 treating it as off, which may not be the arm you asked for"
            );
            false
        }
    }
}

/// What a stored calibration has to satisfy before it is served.
///
/// Four arms rather than one, because which of them is right has not
/// been measured and the campaign has twice adopted a statistic on an
/// argument that later data refuted.
///
/// The shape they are judged on has two halves. A process that serves a
/// record skips a draw, measured at about 15 ms on a 12-core host. It
/// also routes on that record's thresholds rather than on a fresh
/// draw's, so an arm can win on start cost and lose on routing.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
pub enum ServePolicy {
    /// The shipped arm: whatever
    /// `calibration_store::CpuCalibration::is_trustworthy` admits,
    /// which is a record some other draw has agreed with. Named in
    /// backticks rather than linked, because the item is behind
    /// `persisted-calibration` and a link to it is broken in a build
    /// without that feature.
    ///
    /// A fresh stamp therefore serves from its third start: the first
    /// stores a provisional record, the second agrees with it and
    /// confirms, and the rest read.
    Spread,
    /// Anything that was actually measured. The cheapest-cost ordering
    /// decides which record stands, and load only adds time, so a draw
    /// taken on a busy host cannot win it.
    Any,
    /// The share of its interval the measuring thread held a core,
    /// against [`occupancy_floor_per_mille`]. Reads 998 to 999 on a
    /// quiet draw here and 664 to 799 on a loaded one.
    Occupancy,
    /// The spread again, against a bound the caller supplies rather
    /// than the shipped one.
    SpreadAt(u32),
}

/// Carried only where the store is, because the record it judges is
/// `calibration_store::CpuCalibration` and that module sits behind
/// `persisted-calibration`. The one caller, `stored_record_serves` in
/// `par_iter`, is gated on the same feature, so a build without it has
/// nothing to ask and nothing to answer with.
///
/// The enum itself stays ungated: it names no gated type, and
/// `serve_policy()` reads the same switch in every configuration.
#[cfg(feature = "persisted-calibration")]
impl ServePolicy {
    /// Whether this record may be served.
    ///
    /// A record with no samples is refused by every arm: a table nobody
    /// has published reads back zeroed, and zero is not a measurement.
    pub fn admits(self, cpu: &crate::sched::calibration_store::CpuCalibration) -> bool {
        if cpu.samples == 0 {
            return false;
        }
        match self {
            Self::Spread => cpu.is_trustworthy(),
            Self::Any => true,
            Self::Occupancy => match cpu.occupancy() {
                // No thread clock means no reading, and a platform that
                // cannot measure occupancy must not be frozen out of
                // its own store by an arm that needs one.
                None => true,
                Some(share) => share >= occupancy_floor_per_mille(),
            },
            Self::SpreadAt(bound) => cpu.spread_per_mille <= bound,
        }
    }
}

/// The raw text of a variable, or `None` where it is genuinely unset.
///
/// A value that is not UTF-8 is reported and read as unset, rather than
/// dropped: an arm that failed to take effect produces rows that look
/// exactly like the arm that did.
fn raw(name: &str) -> Option<String> {
    match std::env::var(name) {
        Ok(text) => Some(text),
        Err(std::env::VarError::NotPresent) => None,
        Err(std::env::VarError::NotUnicode(value)) => {
            notice!(
                "flynnel: {name} is set to something that is not UTF-8 ({value:?}); \
                 treating it as unset, which may not be the arm you asked for"
            );
            None
        }
    }
}

/// Which arm [`stored_record_serves`] applies, from
/// `FLYNNEL_SERVE_POLICY`: `spread` (the shipped behavior and the
/// default), `any`, `occupancy`, or `spread:<per-mille>`.
///
/// An unreadable value reports itself and falls back to the shipped
/// arm, because a rotation that silently ran the default while its log
/// claimed another arm is the failure this campaign exists to stop.
///
/// [`stored_record_serves`]: crate::sched::par_iter
pub fn serve_policy() -> ServePolicy {
    static V: OnceLock<ServePolicy> = OnceLock::new();
    *V.get_or_init(|| {
        let Some(text) = raw("FLYNNEL_SERVE_POLICY") else {
            return ServePolicy::Spread;
        };
        let v = text.trim().to_ascii_lowercase();
        if let Some(rest) = v.strip_prefix("spread:") {
            return match rest.parse::<u32>() {
                Ok(bound) => ServePolicy::SpreadAt(bound),
                Err(e) => {
                    notice!(
                        "flynnel: FLYNNEL_SERVE_POLICY spread: wants a whole number of \
                         parts per mille and got {rest:?} ({e}); running the shipped arm"
                    );
                    ServePolicy::Spread
                }
            };
        }
        match v.as_str() {
            "" | "spread" => ServePolicy::Spread,
            "any" => ServePolicy::Any,
            "occupancy" => ServePolicy::Occupancy,
            other => {
                notice!(
                    "flynnel: FLYNNEL_SERVE_POLICY is {other:?}, which is not spread, any, \
                     occupancy or spread:<per-mille>; running the shipped arm"
                );
                ServePolicy::Spread
            }
        }
    })
}

/// The floor [`ServePolicy::Occupancy`] admits at, from
/// `FLYNNEL_OCCUPANCY_FLOOR_PER_MILLE`.
///
/// 900 by default, which sits between the 998 to 999 a quiet draw reads
/// on the host measured and the 664 to 799 a loaded one reads. A
/// starting point for a rotation rather than a derived value, and the
/// rotation is what it exists for.
pub fn occupancy_floor_per_mille() -> u32 {
    static V: OnceLock<u32> = OnceLock::new();
    *V.get_or_init(|| {
        let Some(text) = raw("FLYNNEL_OCCUPANCY_FLOOR_PER_MILLE") else {
            return 900;
        };
        match text.trim().parse::<u32>() {
            Ok(v) => v,
            Err(e) => {
                notice!(
                    "flynnel: FLYNNEL_OCCUPANCY_FLOOR_PER_MILLE wants a whole number and \
                     got {text:?} ({e}); using 900"
                );
                900
            }
        }
    })
}

/// Read a switch once, absent meaning on.
///
/// The same spellings mean off as in [`read`], so a caller who turns
/// something off gets it off by any of the words a script produces. The
/// difference is only what an unset variable means, which for a
/// mechanism a measurement has already settled is that it runs.
fn read_defaulting_on(name: &str) -> bool {
    match std::env::var_os(name) {
        None => true,
        Some(_) => read(name),
    }
}

/// Take the classifier's per-item spread from the thread's own clock
/// rather than from wall time.
///
/// Wall time rises both because the work is irregular and because the
/// thread lost its core, and preemption lands on some leaves and not
/// others, so it reaches a wall-time spread as variance indistinguishable
/// from the work's own. Costs two thread-clock reads on each sampled
/// leaf, which is why it is a switch rather than simply the behavior.
///
/// Read by two recorders, which between them cover the dispatch
/// entries. `record_leaf_sampled` serves the plain steal-driven bisect,
/// where one leaf in the stride is both wall-timed and bracketed.
/// `record_leaf_bracket_sampled` serves the indexed and triple entries,
/// which time every leaf and bracket one in the same stride. The two
/// cadences differ because those entries need a reading from every leaf
/// to converge, and that requirement is about the wall clock alone.
///
/// [`crate::sched::call_site::CallSiteState::oncore_items`] stays at
/// zero when no leaf was bracketed, so it reports whether this switch
/// reached the dispatch under measurement rather than leaving a switch
/// that never engaged to look like one that did not help.
/// On unless the variable turns it off. The bracket costs nothing
/// measurable: on a 24-thread host, three workload shapes each over
/// forty paired trials read 1.0033, 1.0000 and 1.0019 against the same
/// code with it off, at bounds of 0.64, 0.22 and 0.31 percent with
/// controls resolving to 0.19 and tighter.
/// Measured again on three hosts, paired by trial, at 4096 reps of
/// uniform work: 0.9978 at a 0.22 percent bound on a 24-thread Windows
/// bare-metal box, 32 clean pairs of 40, control 0.9988 at 0.38;
/// 0.9922 at 4.79 on a 16-core Linux guest, 0.9917 at 11.15 on a
/// 16-core FreeBSD guest. Not slower on any, no speed-up claimed on any.
///
/// The bare-metal cell was read twice. An earlier rotation gave 0.9964
/// at 1.90 percent over 7 pairs of 40, taken while an unrelated process
/// held a core continuously: the box's idle floor was 1.81 cores against
/// the harness's 1.4-core gate, so most trials were dropped for a
/// condition none of them caused. The figures above are the repeat on a
/// quiet box, and the difference between the two bounds is the floor
/// rather than the lever.
///
/// The bracket is taken in all three - oncore_items reads zero on the
/// arm without the lever and millions on the arm with it - and on the
/// bare-metal run no class moves, so 1.90 percent is what the
/// instrumentation costs when nothing downstream changes.
///
/// A correctness lever, judged on the class it produces rather than on
/// throughput. On-core timing is the thread clock and excludes
/// descheduled time, so what it changes in the classifier's input is
/// the descheduling there is: the learned class differs between the
/// arms in 29 trials of 40 on a Linux guest, 6 of 40 on a FreeBSD
/// guest, and 1 of 40 on bare metal. A host that does not deschedule
/// has nothing here to correct.
///
/// Its price on a guest, at an 8 second window over 40 trials: 0.9814
/// at a 2.52 percent bound where the class moves, 0.9872 at 4.34 where
/// it holds. About the same either way. On quiet bare metal, where no
/// class moves at this size, 0.9978 at 0.22 over 32 pairs.
pub fn oncore_spread() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read_defaulting_on("FLYNNEL_LEVER_ONCORE_SPREAD"))
}

/// Weight a leaf batch by the share of its interval the pool spent on a
/// core, so a contended batch counts for less.
pub fn batch_weight() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read("FLYNNEL_LEVER_BATCH_WEIGHT"))
}

/// Give each thread its own wasmtime store per kernel, instantiated on
/// that thread's first dispatch of it, instead of sharing one store
/// behind a lock.
///
/// Off until measured. A wasmtime store is not `Sync` and its API
/// takes a mutable store, so two dispatches of one kernel on the
/// shared store take turns; that lock is the last one the crate has
/// that an external type forces. A store per thread removes the
/// sharing rather than the waiting, which is why it can remove the
/// lock at all, and it moves instantiation out of registration and
/// into each thread's first dispatch. An arm that measures only the
/// steady state reports a win a short-lived dispatch never sees, so
/// the instantiation has to be inside the measured span.
pub fn wasm_local_store() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read("FLYNNEL_LEVER_WASM_LOCAL_STORE"))
}

/// Wait for a foreign caller's latch through a spin floor, then bounded
/// monitor waits on the latch flag's own line, then the park, instead
/// of spinning the caller's whole window on `PAUSE`.
///
/// Off until measured. This is the one placement the monitor wait is
/// built for and the pool's idle search is not: the event waited on is
/// a single store to a known line, the waiter has nothing else it
/// could be doing, and the alternative it replaces is a spin rather
/// than a yield or a park. A monitor wait ends on the same store a
/// spin would, issues nothing into the pipeline while it waits, and
/// holds the logical processor no longer than the spin it replaces.
pub fn latch_monitor() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read("FLYNNEL_LEVER_LATCH_MONITOR"))
}

/// Wait for a shared-memory backend's result through a timed floor of
/// polls, then bounded monitor waits on the result cell's own line, in
/// place of the rest of the caller's `PAUSE` budget, before the yields
/// the budget has always ended in.
///
/// Off until measured. The originator of a cross-process dispatch polls
/// one latch cell, a single 64-byte line whose state byte the publishing
/// process stores to, and has nothing else to do while it waits: the
/// same three conditions [`latch_monitor`] meets, with the store coming
/// from another process instead of another thread. The rung spends the
/// wall time the polls it replaces would have spent, so both arms reach
/// the yields at the same moment and differ only in what the waiting
/// thread did before them.
pub fn backend_spin_monitor() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read("FLYNNEL_LEVER_BACKEND_SPIN_MONITOR"))
}

/// Learn each phase of a declared kernel run through the PowerShell
/// module at a site of its own, keyed by the kernel, the phase and the
/// base-2 logarithm of the phase's block size in items, instead of at
/// one site a kernel.
///
/// Off until measured. A site's class, leaf statistics and policy-arm
/// averages are site-wide, and only hybrid placement buckets by size,
/// while a kernel's phases cost differently per block and a block's
/// cost grows with its size, so one site a kernel learns a mixture of
/// them. The finer key learns each apart at the price of more sites to
/// warm, at most a few dozen a kernel, all in the keyed table.
pub fn kernel_phase_sites() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read("FLYNNEL_LEVER_KERNEL_PHASE_SITES"))
}

/// Give the outside caller's slot-wait parker no yield rounds, so a
/// caller whose spin budget is spent goes from the sleep handshake
/// straight to the park.
///
/// A yield while the process holds more runnable threads than cores
/// hands the core to a ready thread for the rest of that thread's time
/// slice, so a latch set during the yield rounds is seen late. On pc2 at
/// 13be54b, with 18 spinners beside 24 workers, 97 of 104 slow calls
/// ended their wait inside those rounds, a median 12.6 ms after their
/// job ended, and the callers that had reached the park woke in 2 to
/// 7 us.
///
/// Off: measured to change nothing. In four arms of the oversubscribed
/// caller, on pc2 at faa0d05 and on a 16-vCPU guest at 517dba9, this arm
/// sat with the arm that had no switch on, quiet and loaded, and adding
/// it to [`join_park`] moved nothing that arm had not. The pc2 run routed
/// on a profile drawn on a pool worker, dispatch 100 ns and collapse
/// 20561 ns, where the host's idle profile is about 1400 and 11500.
pub fn slot_park_now() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read("FLYNNEL_LEVER_SLOT_PARK_NOW"))
}

/// Park a worker whose stolen right half is still running, once its
/// spin budget is spent and it finds nothing to steal, in a kernel wait
/// that the thief ends by setting the half's latch, instead of yielding
/// each round.
///
/// The same cause as [`slot_park_now`], inside the join: on pc2 at
/// 13be54b every slow call whose join ran on for a millisecond or more
/// after its last leaf had a waiter whose last yield covered most of
/// that time after its latch was set.
///
/// Off, and measured not to be never-slower. On pc2 at 32b92da, with 18
/// spinners beside 24 workers and the host's idle profile pinned, three
/// rounds against the same code with every switch off: the loaded
/// median stayed level, 1.17 ms against 1.13, and the tail fell, p99
/// 15.2 ms against 32.2 and calls of 5 ms or more 23 against 128 in the
/// median round; the quiet median rose 5.8 percent beside a serial
/// control that rose 3.7. On a 16-vCPU guest at 517dba9 it raised the
/// quiet median from about 1.3 to 3.1 ms in each of three rounds, with
/// the serial control level across the arms: a parked thread halts its
/// vCPU, and the wake through the hypervisor costs far more than the
/// yield it replaced.
/// [`join_park_oversubscribed`] parks only where a yield would lose the
/// core.
pub fn join_park() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read("FLYNNEL_LEVER_JOIN_PARK"))
}

/// Park a join waiter whose spin budget is spent in the kernel, as
/// [`join_park`] does, but only while a yield somewhere in the process
/// has given its core away within the last
/// [`crate::sched::oversubscription::WINDOW`]; otherwise yield as the
/// shipped code does.
///
/// A yield loses the core only when another thread is ready to take it,
/// and when it does, it comes back a time slice later rather than in a
/// microsecond. So the pool's idle rounds, which yield with no latch
/// pending, time their yields while this is on, and a long one is the
/// reading. A quiet process parks a join waiter only on the rare yield
/// that a descheduled vCPU stretched past the line, and a guest pays no
/// hypervisor wake for a yield that would have cost nothing. What it
/// adds while on is two clock reads and one counter add around each idle
/// yield.
///
/// On, measured in the oversubscribed caller against the same code with
/// it off, rounds paired, quiet and loaded. On a 16-vCPU Linux guest, in
/// two runs of 15 rounds each on fresh calibrations, the quiet median was
/// 0.994 and 0.991 of the off arm's, slower in 6 rounds of 15 in each run
/// (0.977 to 1.017 and 0.944 to 1.034 across the middle nine), and 1.002
/// and 0.992 against each process's own serial calls; under load, calls
/// of 5 ms or more fell to 0.56 and 0.60 of the off arm's, fewer in 13 and
/// 12 rounds, while the loaded median resolved in neither. On a 24-thread
/// Windows host with its profile pinned, the median round's loaded p99
/// was 25.0 ms against 32.2 and its calls of 5 ms or more 104 against
/// 128, with the quiet median at 0.469 ms against 0.452 beside a serial
/// control at 3.710 against 3.576. On an 8-core Windows desktop the loaded
/// median was 2.76 ms against 5.31 and the calls of 5 ms or more 120
/// against 414. `FLYNNEL_LEVER_JOIN_PARK_OVERSUBSCRIBED=0` turns it off.
pub fn join_park_oversubscribed() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read_defaulting_on("FLYNNEL_LEVER_JOIN_PARK_OVERSUBSCRIBED"))
}

/// Spend an idle pool worker's spin rounds in a bounded monitor wait on
/// the sleep coordinator's counters word rather than in `yield_now`, on
/// a host with MONITORX or WAITPKG, so a producer's store wakes the
/// worker inside the round instead of at the scheduler's next pick. The
/// park that follows the window stays the kernel park.
///
/// Off until measured. A monitor wait is a running thread to the OS
/// for as long as it lasts, so the window bounds what it can hold: one
/// round is the host's calibrated dispatch cost, and the round count
/// is the spin window the coordinator already spends. Measured as the
/// pool park it must never be, a monitor wait held every hardware
/// thread the pool had and cost sixteen processor seconds per wall
/// second on a 24-thread host; the loaded arm of a cold-dispatch
/// measurement and the process's processor time beside it decide this
/// one.
pub fn spin_monitor() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read("FLYNNEL_LEVER_SPIN_MONITOR"))
}

/// Decide SMT from the window the site's classifier last read rather
/// than from its lifetime cv^2, which never decays.
///
/// Engaged and not slower on three hosts, paired by trial against the
/// same code with it off: a 24-thread Windows bare-metal box reads
/// 1.0012 at a 0.61 percent bound over ten pairs, a 16-core Linux
/// guest sits inside its null at all three loads, and a 16-core FreeBSD
/// guest reads 1.0639 at 9.37 percent over forty. The arms differ on
/// every host - off resolves SMT true, on resolves it false - and no
/// speed-up is claimed anywhere.
///
/// The bound tracks the host rather than the method. The guest controls
/// span 0.33 to 1.32 where bare metal spans 0.99 to 1.02, because a
/// guest cannot see its own vCPU being descheduled, so neither its
/// clocks nor its controls subtract it.
pub fn smt_from_window() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read("FLYNNEL_LEVER_SMT_WINDOW"))
}

/// Cap a plan's worker count by the CPUs the process may currently use,
/// re-read on a cadence, so a narrowed affinity mask or cgroup quota
/// reaches the sizing.
///
/// On unless the variable turns it off. What it buys is a host that
/// narrows after the pool is spawned: `tests/affinity_follows_process_mask`
/// holds that the plan's width follows the mask down and back on all
/// three platforms. Its price is the re-read, one affinity query every
/// 250 ms, paid for nothing on a host whose mask never changes. Measured
/// twice, paired by trial on a 24-thread Windows bare-metal box at 4096
/// reps of uniform work, 40 trials each, no decision moving between the
/// arms: 1.0006 at a 0.13 percent bound over 40 clean pairs, retained
/// 1.0005 at 0.17, control 1.0000 at 0.10, on the code that ships; and
/// 1.0000 at 0.15 over 23 clean pairs, retained 1.0014 at 0.21, control
/// 1.0000 at 0.30, on a tree whose Windows probe was still
/// `available_parallelism`.
pub fn allowed_width() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read_defaulting_on("FLYNNEL_LEVER_ALLOWED_WIDTH"))
}

/// Decline to displace a stored calibration whose dispatch cost is
/// cheaper than the one being offered.
///
/// One of the four switches here that default on, with `oncore_spread`,
/// `allowed_width` and `join_park_oversubscribed`. The rest default off
/// because off is the behavior that shipped and a measurement has to
/// earn the flip; this one has the measurement.
/// Across 16 draws at each of three
/// load levels on a 12-core host, the dispatch cost read 1300 to 1500 ns
/// idle and 3.2 to 7.0 million saturated, with no overlap, so the
/// cheaper record is the quieter draw and the ordering needs no
/// threshold.
///
/// Off restores the previous behavior, where the most recent draw wins
/// and a calibration taken while the host was busy stands until
/// something displaces it.
pub fn calibration_refusal() -> bool {
    static V: OnceLock<bool> = OnceLock::new();
    *V.get_or_init(|| read_defaulting_on("FLYNNEL_LEVER_CALIBRATION_REFUSAL"))
}

/// Every switch and its state, for a harness to print beside its result.
///
/// A row that does not say which arm produced it is a row that cannot be
/// checked, and an arm that failed to engage looks exactly like one that
/// engaged and did nothing.
///
/// The spin pair lives in [`crate::sched::jec_sleep`] rather than here,
/// and is reported anyway: it is a runtime switch that changes how the
/// pool parks, so a row without it does not say which arm produced it.
/// `spin_window` is reported beside `spin_adaptive` because the window
/// alone cannot say whether the controller ran - a rescue-dominated
/// workload grows it and is clamped back to the tuned default it
/// started from.
pub fn describe() -> String {
    format!(
        "oncore_spread={} batch_weight={} smt_window={} allowed_width={} \
         calibration_refusal={} latch_monitor={} backend_spin_monitor={} \
         kernel_phase_sites={} spin_monitor={} join_park={} join_park_oversubscribed={} \
         slot_park_now={} spin_adaptive={} spin_window={} serve_policy={:?} occupancy_floor={}",
        oncore_spread(),
        batch_weight(),
        smt_from_window(),
        allowed_width(),
        calibration_refusal(),
        latch_monitor(),
        backend_spin_monitor(),
        kernel_phase_sites(),
        spin_monitor(),
        join_park(),
        join_park_oversubscribed(),
        slot_park_now(),
        crate::sched::spin_adaptive(),
        crate::sched::spin_window(),
        serve_policy(),
        occupancy_floor_per_mille(),
    )
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn the_values_that_mean_off_are_the_ones_a_script_writes() {
        // Read directly rather than through the cached accessors, which
        // would fix whatever the first test in the process observed.
        for off in ["", "0", "off", "false", "OFF", "False", " 0 "] {
            unsafe { std::env::set_var("FLYNNEL_LEVER_TEST", off) };
            assert!(!read("FLYNNEL_LEVER_TEST"), "{off:?} should read as off");
        }
        for on in ["1", "on", "true", "yes", "ON"] {
            unsafe { std::env::set_var("FLYNNEL_LEVER_TEST", on) };
            assert!(read("FLYNNEL_LEVER_TEST"), "{on:?} should read as on");
        }
        unsafe { std::env::remove_var("FLYNNEL_LEVER_TEST") };
        assert!(!read("FLYNNEL_LEVER_TEST"), "an unset switch is off");
    }

    #[test]
    fn an_unset_switch_takes_the_default_of_the_reader_it_uses() {
        // Both readers, not the accessors: each accessor caches in a
        // OnceLock, so calling one here fixes its value for every later
        // test in this binary.
        //
        // Named by reader rather than by lever because the levers do not
        // share a default. oncore_spread, calibration_refusal,
        // allowed_width and join_park_oversubscribed take
        // read_defaulting_on; batch_weight and smt_from_window take read.
        unsafe { std::env::remove_var("FLYNNEL_LEVER_DEFAULT_TEST") };
        assert!(
            !read("FLYNNEL_LEVER_DEFAULT_TEST"),
            "read must answer off for a switch nobody set"
        );
        assert!(
            read_defaulting_on("FLYNNEL_LEVER_DEFAULT_TEST"),
            "read_defaulting_on must answer on for a switch nobody set"
        );

        // And that each reader still honors an explicit off, so a
        // default-on switch can be turned off by a caller.
        unsafe { std::env::set_var("FLYNNEL_LEVER_DEFAULT_TEST", "0") };
        assert!(!read("FLYNNEL_LEVER_DEFAULT_TEST"));
        assert!(!read_defaulting_on("FLYNNEL_LEVER_DEFAULT_TEST"));
        unsafe { std::env::remove_var("FLYNNEL_LEVER_DEFAULT_TEST") };
    }
}