flynnel 0.6.0

K-aware NUMA-aware work-stealing scheduler with extended-Flynn-taxonomy dispatch (MIMD / SIMC / MIMC / MISD / SIMT / MIMT): a rayon-shaped pool reshaped around per-call execution-class plans with first-class CPU+GPU hybrid (join_hybrid, hybrid_pipeline) and per-call dispatch profiles.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
//! Adaptive DispatchProfile + WorkloadClass migration.
//!
//! The K_gating adaptive pattern proven in
//! [`crate::sched::adaptive_worker`] generalizes to any routing
//! decision that's consulted once per dispatch (not per push/pop).
//! DispatchProfile is the canonical example: it drives `use_smt`,
//! `oversubscription_log2`, `estimated_per_item_ns`,
//! `use_mailbox_routing`, and `deque_tier_hint` - all of which are
//! read at `JobPlan::set_profile` construction time, not at the
//! deque hot path.
//!
//! This module exposes:
//! - [`WorkloadClass`]: the user-facing Light / Compute / Heavy /
//!   Memory vocabulary that maps to a [`DispatchProfile`]
//! - [`active_dispatch_profile`] / [`migrate_dispatch_profile`]:
//!   global active-profile read / swap via a single AtomicU8 tag
//! - [`migrate_workload_class`]: high-level swap via WorkloadClass
//!
//! ## Cost analysis
//!
//! - Per-op cost on the deque hot path: **zero** (profile is
//!   consulted at plan-construction time, never per push/pop)
//! - Migration cost: **one AtomicU8 Release-store** (~1 ns)
//! - Subsequent dispatch reads the new profile via one AtomicU8
//!   Acquire-load at `JobPlan::set_profile` time (~1 ns)
//!
//! Applications observe per-call elapsed-vs-expected ratio and
//! call [`migrate_workload_class`] when the observed pattern
//! doesn't match the active class. The flip is FREE on the per-op
//! path; only the application's classification cost matters.

#![allow(clippy::missing_errors_doc)]

use core::sync::atomic::{AtomicU8, Ordering};

use crate::dispatch_profile::DispatchProfile;

/// User-facing workload-class vocabulary. Maps to a
/// [`DispatchProfile`] via [`Self::to_dispatch_profile`].
///
/// The application is expected to classify its workload (via
/// observed per-item elapsed time or a-priori knowledge) and
/// migrate the active class when the workload shifts. Cost: one
/// atomic store per migration; zero on the per-op path.
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum WorkloadClass {
    /// Tiny per-item cost (~< 50 ns/item); dispatch overhead
    /// dominates. Maps to [`DispatchProfile::PortBound`] because the
    /// inline-collapse decision is the dominant scheduler concern
    /// at this scale and the leaf body is typically port-bound when
    /// it does parallelize.
    FineGrain,
    /// Medium per-item cost (~50-500 ns/item); port-saturated
    /// pipeline (typically integer multiply or FMA on a single
    /// execution port). SMT siblings parked because they would
    /// contest the same port. Maps to [`DispatchProfile::PortBound`].
    PortBound,
    /// Large per-item cost (~> 500 ns/item); long FP dependency
    /// chains stall the pipeline. SMT siblings active to fill the
    /// stall bubbles. Maps to [`DispatchProfile::LatencyBound`].
    LatencyBound,
    /// Memory work with irregular access (pointer-chase, gather/
    /// scatter, hash probes, graph traversal). SMT siblings active
    /// to interleave cache misses. Maps to
    /// [`DispatchProfile::MemoryBound`].
    MemoryBound,
    /// Sequential streaming where per-core memory bandwidth is the
    /// bottleneck (byte scan, image kernels, prefix-sum block sums,
    /// histogram). SMT siblings parked - both threads on the same
    /// physical core would compete for the same L2/L3 bandwidth
    /// instead of helping. Maps to [`DispatchProfile::Streaming`].
    Streaming,
}

impl WorkloadClass {
    /// Map the user-facing class to the scheduler's
    /// [`DispatchProfile`].
    #[inline]
    pub fn to_dispatch_profile(self) -> DispatchProfile {
        match self {
            WorkloadClass::FineGrain => DispatchProfile::PortBound,
            WorkloadClass::PortBound => DispatchProfile::PortBound,
            WorkloadClass::LatencyBound => DispatchProfile::LatencyBound,
            WorkloadClass::MemoryBound => DispatchProfile::MemoryBound,
            WorkloadClass::Streaming => DispatchProfile::Streaming,
        }
    }
}

/// Caller-supplied hint about the structural shape of a workload.
/// When set on a [`crate::JobPlan`] via
/// [`crate::JobPlan::with_workload_shape`], the static initial
/// classifier consults it directly instead of inferring from
/// (k_outer, batch_size, estimated_per_item_ns). Use this when you
/// know your workload's pattern at compile time -- it lets the
/// scheduler get the right routing on call 1 with no observer
/// refinement needed.
///
/// Map to [`WorkloadClass`] via [`Self::to_workload_class`].
#[derive(Copy, Clone, Debug, PartialEq, Eq, Hash)]
pub enum LeafShape {
    /// Compute-bound work whose bottleneck is the issue port
    /// (IMUL, FMA, integer compare). SMT siblings on the same
    /// physical core would contest the same port and produce
    /// no net throughput.
    ///
    /// Examples: branchy compare-and-swap sort, RNG IMUL loops,
    /// kmeans FMA-bound distance computation, Conway stencil
    /// add+match.
    PortCompute,
    /// Compute-bound work whose bottleneck is a long FP
    /// dependency chain (sqrt, div, chained Newton iterations).
    /// SMT siblings active to fill the dispatch bubbles between
    /// latency stalls.
    ///
    /// Examples: n-body sqrt chain per pair, recursive Newton
    /// iteration, branchy adaptive integrators, FFT butterflies
    /// with register dependencies.
    LatencyCompute,
    /// Sequential streaming where per-core memory bandwidth is
    /// the bottleneck. SMT siblings parked because both threads
    /// would compete for the same L2/L3 bandwidth.
    ///
    /// Examples: byte scans (grep), image kernels (rgb->gray,
    /// box blur), CSV parsing, sequential prefix-sum stages,
    /// histogram on large arrays.
    Streaming,
    /// Irregular memory access (gather, scatter, pointer chase).
    /// SMT siblings active to interleave cache-miss loads.
    ///
    /// Examples: sparse matrix-vector multiply (col_idx gather),
    /// PageRank graph adjacency gather, hash table probes,
    /// merkle tree pair lookups, strided column reads.
    Gather,
    /// Mixed / unknown shape. Fall back to (k_outer, batch_size,
    /// estimated_per_item_ns) heuristics in
    /// [`infer_class_static`].
    Unknown,
}

impl LeafShape {
    /// Map the shape directly to a [`WorkloadClass`] when the
    /// shape is known. Returns `None` for `Unknown` -- the caller
    /// (typically the static classifier) falls back to ns + K + N
    /// heuristics.
    #[inline]
    pub fn to_workload_class(self) -> Option<WorkloadClass> {
        match self {
            Self::PortCompute => Some(WorkloadClass::PortBound),
            Self::LatencyCompute => Some(WorkloadClass::LatencyBound),
            Self::Streaming => Some(WorkloadClass::Streaming),
            Self::Gather => Some(WorkloadClass::MemoryBound),
            Self::Unknown => None,
        }
    }
}

/// Encoded active-profile tag stored in [`ACTIVE_PROFILE_TAG`].
const TAG_LATENCY_BOUND: u8 = 0;
const TAG_PORT_BOUND: u8 = 1;
const TAG_MEMORY_BOUND: u8 = 2;
const TAG_UNSPECIFIED: u8 = 3;
const TAG_STREAMING: u8 = 4;

/// Global active-profile tag. Read by the scheduler when
/// constructing default JobPlans via [`active_dispatch_profile`];
/// flipped by [`migrate_dispatch_profile`] /
/// [`migrate_workload_class`]. Initial value: PortBound (the
/// flynnel calibration default for Zen+ R7 2700 - see the
/// realistic_bench results in the WorkerCtx swap commit).
static ACTIVE_PROFILE_TAG: AtomicU8 = AtomicU8::new(ACTIVE_PROFILE_TAG_DEFAULT);

/// The tag [`ACTIVE_PROFILE_TAG`] holds before anything migrates it.
const ACTIVE_PROFILE_TAG_DEFAULT: u8 = TAG_PORT_BOUND;

/// Linkage confirmation marker. When the binary links this
/// module, `nm <bin> | grep __flynnel_marker` returns this
/// symbol, confirming the adaptive profile dispatch path is
/// present in the build.
#[unsafe(no_mangle)]
pub static __flynnel_marker_adaptive_profile: u8 = 0;

/// The profile a stored tag names. Any value that is not a profile's
/// tag reads as [`DispatchProfile::Unspecified`].
#[inline]
const fn profile_of_tag(tag: u8) -> DispatchProfile {
    match tag {
        TAG_LATENCY_BOUND => DispatchProfile::LatencyBound,
        TAG_PORT_BOUND => DispatchProfile::PortBound,
        TAG_MEMORY_BOUND => DispatchProfile::MemoryBound,
        TAG_STREAMING => DispatchProfile::Streaming,
        _ => DispatchProfile::Unspecified,
    }
}

/// The tag a profile is stored as.
#[inline]
const fn tag_of_profile(profile: DispatchProfile) -> u8 {
    match profile {
        DispatchProfile::LatencyBound => TAG_LATENCY_BOUND,
        DispatchProfile::PortBound => TAG_PORT_BOUND,
        DispatchProfile::MemoryBound => TAG_MEMORY_BOUND,
        DispatchProfile::Streaming => TAG_STREAMING,
        DispatchProfile::Unspecified => TAG_UNSPECIFIED,
    }
}

/// Read the active DispatchProfile via one AtomicU8 Acquire-load.
/// Used by the scheduler to pick default plan knobs when the
/// caller doesn't specify a profile explicitly.
#[inline]
pub fn active_dispatch_profile() -> DispatchProfile {
    profile_of_tag(ACTIVE_PROFILE_TAG.load(Ordering::Acquire))
}

/// Migrate the global active DispatchProfile via one AtomicU8
/// Release-store. Subsequent dispatches that consult
/// [`active_dispatch_profile`] see the new value; per-op cost on
/// the deque hot path is unchanged.
///
/// [`tick_auto_classify`] calls this from whichever thread ran a
/// dispatch, so the value is the whole process's and any reader can
/// find it changed between two of its own reads.
#[inline]
pub fn migrate_dispatch_profile(profile: DispatchProfile) {
    ACTIVE_PROFILE_TAG.store(tag_of_profile(profile), Ordering::Release);
}

/// High-level WorkloadClass migration. Maps to the underlying
/// [`DispatchProfile`] then calls [`migrate_dispatch_profile`].
#[inline]
pub fn migrate_workload_class(class: WorkloadClass) {
    migrate_dispatch_profile(class.to_dispatch_profile());
}

/// Active WorkloadClass derived from [`active_dispatch_profile`].
/// The [`DispatchProfile`] enum has one fewer variant than
/// [`WorkloadClass`] (there is no PortBound-vs-FineGrain
/// distinction on the profile side because the inline-collapse
/// decision is a downstream policy question, not a per-op tuning
/// input). This function collapses:
/// - `PortBound` and `Unspecified` -> [`WorkloadClass::PortBound`]
///   (the calibration default; `FineGrain` is only reachable via
///   [`classify_observed`] or [`infer_class_static`] when a real
///   per-leaf cost signal is available).
/// - other profiles map straight through to their same-named
///   [`WorkloadClass`] variant.
#[inline]
pub fn active_workload_class() -> WorkloadClass {
    class_of_profile(active_dispatch_profile())
}

/// The class a profile stands for, with the collapse
/// [`active_workload_class`] documents.
#[inline]
const fn class_of_profile(profile: DispatchProfile) -> WorkloadClass {
    match profile {
        DispatchProfile::LatencyBound => WorkloadClass::LatencyBound,
        DispatchProfile::PortBound => WorkloadClass::PortBound,
        DispatchProfile::MemoryBound => WorkloadClass::MemoryBound,
        DispatchProfile::Streaming => WorkloadClass::Streaming,
        DispatchProfile::Unspecified => WorkloadClass::PortBound,
    }
}

/// Host-calibratable classifier boundaries. Every field is an
/// atomic so [`calibrate_class_thresholds`] can update the live
/// values with plain Release stores while dispatch paths read them
/// with Relaxed loads (one MOV each on x86).
///
/// Defaults are empirically-tuned Zen-family values; calibration
/// replaces them with quantities measured on the running host in
/// the running binary, clamped to sane bands so a noisy calibration
/// run cannot produce nonsense.
pub struct ClassThresholds {
    /// Per-item ns below which work is FineGrain (dispatch overhead
    /// dominates). Default 50.
    pub fine_grain_ns: AtomicU64,
    /// Per-item ns boundary between PortBound and the heavy classes.
    /// Default 500.
    pub port_heavy_ns: AtomicU64,
    /// Per-item ns boundary between MemoryBound and LatencyBound in
    /// the hint-driven static classifier. Default 2000.
    pub memory_latency_ns: AtomicU64,
    /// cv^2 (parts-per-1000) below which leaves count as uniform.
    /// Default 50.
    pub cv2_low_per_mille: AtomicU64,
    /// cv^2 (parts-per-1000) at or above which leaves count as
    /// high-variance. Default 500.
    pub cv2_high_per_mille: AtomicU64,
    /// Reduce-merge cycle count at or below which a reducer counts
    /// as trivial for the flat-fanout gate in
    /// [`crate::sched::par_iter::reduce_chunks`]. Default 30_000.
    pub trivial_reduce_cycles: AtomicU64,
}

impl ClassThresholds {
    /// A fresh threshold set holding the documented defaults.
    /// `const` so it can back both the process-global static and
    /// caller-owned instances for isolated calibration.
    pub const fn new_defaults() -> Self {
        Self {
            fine_grain_ns: AtomicU64::new(50),
            port_heavy_ns: AtomicU64::new(500),
            memory_latency_ns: AtomicU64::new(2000),
            cv2_low_per_mille: AtomicU64::new(50),
            cv2_high_per_mille: AtomicU64::new(500),
            trivial_reduce_cycles: AtomicU64::new(30_000),
        }
    }
}

static CLASS_THRESHOLDS: ClassThresholds = ClassThresholds::new_defaults();

use core::sync::atomic::AtomicU64;

/// Live classifier thresholds. Reads are single Relaxed loads.
#[inline]
pub fn class_thresholds() -> &'static ClassThresholds {
    &CLASS_THRESHOLDS
}

/// Report returned by [`calibrate_class_thresholds`] so callers can
/// log what the probes measured and what was installed.
#[derive(Debug, Clone, Copy)]
pub struct ThresholdCalibration {
    /// Measured in-pool empty-join round-trip in nanoseconds.
    pub join_ns: u64,
    /// Installed FineGrain boundary derived from `join_ns`.
    pub fine_grain_ns: u64,
    /// Measured reference trivial-reduce merge in TSC cycles.
    pub trivial_reduce_measured_cycles: u64,
    /// Installed trivial-reduce ceiling (8x measured, clamped).
    pub trivial_reduce_cycles: u64,
    /// SMT-on / SMT-off wall-time ratio (per-mille) of the sqrt-chain
    /// probe: below 950 means SMT helped, above 1050 means it hurt.
    pub smt_ratio_per_mille: u64,
    /// Installed MemoryBound/LatencyBound boundary after the SMT
    /// nudge.
    pub memory_latency_ns: u64,
}

/// Measure this host's classifier boundaries and install them.
///
/// Three probes, all executed inside the running binary so the
/// installed values are always in the running build profile's own
/// cost units (cycle counts and dispatch floors differ between
/// debug and release builds; measuring in-binary keeps the
/// thresholds consistent with whichever profile is live):
///
/// 1. **Dispatch floor**: median of 64 in-pool empty `join` calls.
///    `fine_grain_ns = clamp(join_ns / MIN_LEAF_ITEMS, 25..=200)`;
///    per-item work below that is dominated by scheduling cost even
///    at full leaf granularity.
/// 2. **Trivial-reduce reference**: a 256-lane u64 element-wise add
///    (the histogram-merge shape) timed in TSC cycles.
///    `trivial_reduce_cycles = clamp(8 * measured, 5_000..=200_000)`.
/// 3. **SMT-benefit probe**: a latency-bound sqrt-chain batch run
///    with SMT siblings parked vs active. When SMT helps by >= 5%
///    the MemoryBound/LatencyBound boundary drops toward 1500 (SMT
///    pays off earlier on this host); when it hurts by >= 5% the
///    boundary rises toward 3000. Clamped to 1_000..=4_000.
///
/// Wall-clock cost: a few milliseconds. Runs synchronously on the
/// calling thread; see [`spawn_class_threshold_calibration`] for the
/// IoPool variant.
pub fn calibrate_class_thresholds() -> ThresholdCalibration {
    calibrate_class_thresholds_into(class_thresholds())
}

/// [`calibrate_class_thresholds`] with the install target supplied
/// by the caller. The measured, clamped values land in `target`
/// instead of the process-global thresholds, so a caller can
/// calibrate into an isolated [`ClassThresholds`] (inspection,
/// comparison against the live set) without changing routing for
/// the rest of the process.
pub fn calibrate_class_thresholds_into(target: &ClassThresholds) -> ThresholdCalibration {
    use crate::sched::plan::JobPlan;

    // Probe 1: in-pool empty-join floor. Warm the pool first so the
    // measurement sees steady state, not spawn cost.
    let plan = JobPlan::bare(6, 2);
    for _ in 0..16 {
        crate::sched::arena::join(&plan, || (), || ());
    }
    let mut samples = [0u64; 64];
    for s in samples.iter_mut() {
        let t0 = std::time::Instant::now();
        crate::sched::arena::join(&plan, || (), || ());
        *s = t0.elapsed().as_nanos() as u64;
    }
    samples.sort_unstable();
    let join_ns = samples[samples.len() / 2].max(1);
    let fine_grain_ns = (join_ns / 256).clamp(25, 200);
    target.fine_grain_ns.store(fine_grain_ns, Ordering::Release);

    // Probe 2: trivial-reduce reference in cycles. Uses the same
    // 256-bin element-wise-add shape the reduce_chunks gate was
    // tuned against.
    let mut a = [1u64; 256];
    let b = [2u64; 256];
    let mut best_cycles = u64::MAX;
    for _ in 0..16 {
        let t0 = read_tsc_cal();
        for i in 0..256 {
            a[i] = a[i].wrapping_add(b[i]);
        }
        std::hint::black_box(&a);
        let dt = read_tsc_cal().wrapping_sub(t0);
        best_cycles = best_cycles.min(dt.max(1));
    }
    let trivial_reduce_cycles = (best_cycles.saturating_mul(8)).clamp(5_000, 200_000);
    target
        .trivial_reduce_cycles
        .store(trivial_reduce_cycles, Ordering::Release);

    // Probe 3: SMT benefit on a latency-bound sqrt chain. Runs the
    // same fixed workload twice through the pool, once with the SMT
    // extension parked (default) and once with the siblings raised
    // via acquire_smt. On hosts without an SMT extension the two
    // runs are identical and the nudge is a no-op.
    let sqrt_probe = |use_smt: bool| -> u64 {
        let arena = crate::sched::arena::global_local_arena();
        let _guards = if use_smt { arena.acquire_smt() } else { Vec::new() };
        let mut items: Vec<f64> = (1..=2048u32).map(f64::from).collect();
        let probe_plan = JobPlan::bare(6, items.len() as u32);
        let t0 = std::time::Instant::now();
        crate::sched::par_iter::for_each_chunk(&probe_plan, &mut items, |chunk| {
            for x in chunk.iter_mut() {
                let mut v = *x;
                for _ in 0..64 {
                    v = v.sqrt() + 1.0;
                }
                *x = v;
            }
        });
        std::hint::black_box(&items);
        t0.elapsed().as_nanos() as u64
    };
    let off_ns = sqrt_probe(false).max(1);
    let on_ns = sqrt_probe(true).max(1);
    let smt_ratio_per_mille = on_ns.saturating_mul(1000) / off_ns;
    let current = target.memory_latency_ns.load(Ordering::Relaxed);
    let nudged = if smt_ratio_per_mille <= 950 {
        // SMT helped: latency-bound behavior starts earlier here.
        current.saturating_sub(250)
    } else if smt_ratio_per_mille >= 1050 {
        // SMT hurt: demand heavier work before classifying latency.
        current.saturating_add(250)
    } else {
        current
    };
    let memory_latency_ns = nudged.clamp(1_000, 4_000);
    target
        .memory_latency_ns
        .store(memory_latency_ns, Ordering::Release);

    ThresholdCalibration {
        join_ns,
        fine_grain_ns,
        trivial_reduce_measured_cycles: best_cycles,
        trivial_reduce_cycles,
        smt_ratio_per_mille,
        memory_latency_ns,
    }
}

/// TSC read for the calibration probes. Mirrors the par_iter
/// read_tsc shape (which is private to that module).
#[inline]
fn read_tsc_cal() -> u64 {
    #[cfg(target_arch = "x86_64")]
    // SAFETY: `_rdtsc` is part of the base x86_64 ISA with no
    // CPU-feature or operand-state preconditions.
    unsafe {
        std::arch::x86_64::_rdtsc()
    }
    #[cfg(not(target_arch = "x86_64"))]
    {
        std::time::Instant::now().elapsed().as_nanos() as u64
    }
}

/// Run [`calibrate_class_thresholds`] on the SMT-sibling IoPool.
/// No-op when the pool is disabled (`FLYNNEL_SCHED_SMT_AS_IO`
/// unset): the defaults stay live and the caller can invoke the
/// synchronous variant explicitly instead.
pub fn spawn_class_threshold_calibration() {
    if let Some(pool) = crate::sched::io_pool::global_io_pool() {
        pool.submit(|| {
            let report = calibrate_class_thresholds();
            std::hint::black_box(report);
        });
    }
}

/// Classify an observed leaf-time distribution into a
/// [`WorkloadClass`]. Used by the closing-loop observer to turn
/// raw `(mean_ns, cv2_per_mille)` samples into a routing decision.
///
/// Threshold rationale (matches the doc-comments on each
/// [`WorkloadClass`] variant):
///
/// - `mean_ns < 50`: per-leaf cost is in the dispatch-overhead
///   range; treat as FineGrain (routes to PortBound, no SMT, no
///   extra split budget).
/// - `mean_ns >= 50 and cv2 < 50`: low variance. Either:
///   * `mean_ns < 500` -> PortBound (regular compute, SMT off
///     because siblings contest the issue port).
///   * `mean_ns >= 500` -> Streaming (steady high-bandwidth
///     sequential work; SMT off because siblings compete for
///     L2 bandwidth).
/// - `mean_ns >= 50 and cv2 >= 500`: high variance. Either:
///   * `mean_ns < 500` -> still PortBound (short bursts with
///     scheduling jitter; SMT off).
///   * `mean_ns >= 500` -> LatencyBound (long irregular
///     dependency chains; SMT on to fill stalls).
/// - `mean_ns >= 500 and 50 <= cv2 < 500`: moderate variance,
///   long-ish leaves -> MemoryBound (cache-hierarchy effects
///   producing some leaves slower than others; SMT on to
///   interleave misses).
#[inline]
pub fn classify_observed(mean_ns: u64, cv2_per_mille: u64) -> WorkloadClass {
    let t = class_thresholds();
    match class_from_mean(t, mean_ns) {
        Some(class) => class,
        None => class_from_spread(t, cv2_per_mille),
    }
}

/// The class a window's mean decides by itself: FineGrain and PortBound
/// below [`ClassThresholds::port_heavy_ns`], where no spread is read, and
/// `None` at or above it, where the spread picks the class.
#[inline]
fn class_from_mean(t: &ClassThresholds, mean_ns: u64) -> Option<WorkloadClass> {
    if mean_ns < t.fine_grain_ns.load(Ordering::Relaxed) {
        Some(WorkloadClass::FineGrain)
    } else if mean_ns < t.port_heavy_ns.load(Ordering::Relaxed) {
        Some(WorkloadClass::PortBound)
    } else {
        None
    }
}

/// Heavy per-leaf work: the spread picks between streaming,
/// memory-gather and latency.
#[inline]
fn class_from_spread(t: &ClassThresholds, cv2_per_mille: u64) -> WorkloadClass {
    if cv2_per_mille < t.cv2_low_per_mille.load(Ordering::Relaxed) {
        WorkloadClass::Streaming
    } else if cv2_per_mille < t.cv2_high_per_mille.load(Ordering::Relaxed) {
        WorkloadClass::MemoryBound
    } else {
        WorkloadClass::LatencyBound
    }
}

/// The class a classified window decides: from its mean, and from its
/// spread where the mean leaves the choice to it. `None` when the spread
/// is needed and could not be read, because the window's leaves were
/// shorter than the unit it is kept in; a window the classifier cannot
/// see makes no decision, and the active class stands.
#[inline]
pub(crate) fn classify_window(mean_ns: u64, cv2_per_mille: Option<u64>) -> Option<WorkloadClass> {
    let t = class_thresholds();
    match (class_from_mean(t, mean_ns), cv2_per_mille) {
        (Some(class), _) => Some(class),
        (None, Some(cv2)) => Some(class_from_spread(t, cv2)),
        (None, None) => None,
    }
}

/// Static initial classifier: pick a [`WorkloadClass`] from JobPlan
/// parameters ALONE, with no leaf-time observation. Called from
/// [`crate::JobPlan::new`] so the FIRST dispatch already uses the
/// right routing instead of waiting for the auto-classifier
/// observer to converge over several iterations.
///
/// Inputs:
/// - `k_outer`: precision tier (0..12). Low tier = small per-op
///   cost; high tier = long FP dependency chains.
/// - `batch_size`: total work-unit count for the dispatch.
/// - `estimated_per_item_ns`: caller's hint, if any. When present
///   it dominates the classification (caller knows their workload
///   better than the heuristic).
///
/// Heuristic with ns_hint (caller's signal is authoritative):
/// - `ns < 50` -> FineGrain (per-item below dispatch breakeven)
/// - `50 <= ns < 500` -> PortBound (regular compute)
/// - `ns >= 500` -> MemoryBound (heavy per-item; conservative
///   middle pick because we lack cv2 at construction time. The
///   observer refines to Streaming or LatencyBound after a few
///   iterations if the observed cv2 differs from the assumption.)
///
/// Heuristic without ns_hint (fall back to K_outer + batch_size):
/// - `k_outer >= 7`: high-precision tier -> LatencyBound (long
///   FP dependency chains stall the pipeline; SMT-on helps)
/// - `k_outer <= 3 and batch_size >= 16384`: low-tier with large
///   data -> Streaming (bandwidth-bound bulk processing)
/// - everything else: PortBound (the safe middle pick that the
///   observer can shift via classification migration)
///
/// The observer remains the authority: if leaf-time observation
/// contradicts the static guess over a few iterations, it migrates
/// the global active class. Static gives the right routing on
/// call 1; the observer corrects a wrong call-1 guess.
#[inline]
pub fn infer_class_static(
    k_outer: u8,
    batch_size: u32,
    estimated_per_item_ns: Option<u32>,
) -> WorkloadClass {
    infer_class_static_with_shape(
        k_outer,
        batch_size,
        estimated_per_item_ns,
        LeafShape::Unknown,
    )
}

/// Same as [`infer_class_static`] but with an explicit caller
/// shape hint. The shape is the STRONGEST static signal: when
/// known it maps directly to a [`WorkloadClass`] and the ns + K
/// + N heuristics are skipped.
#[inline]
pub fn infer_class_static_with_shape(
    k_outer: u8,
    batch_size: u32,
    estimated_per_item_ns: Option<u32>,
    shape: LeafShape,
) -> WorkloadClass {
    // Strongest signal: caller's structural shape hint maps
    // directly to a class. The ns and K + N heuristics below are
    // refinement scaffolding for when the caller doesn't know
    // (or hasn't told us).
    if let Some(class) = shape.to_workload_class() {
        // Handle the FineGrain corner case: even when the caller
        // says "PortCompute" or "Streaming", a tiny total batch
        // should inline-collapse instead of dispatching.
        if let Some(ns) = estimated_per_item_ns {
            let total_ns = (ns as u64).saturating_mul(batch_size as u64);
            if total_ns < 50_000 {
                return WorkloadClass::FineGrain;
            }
        }
        return class;
    }

    if let Some(ns) = estimated_per_item_ns {
        let total_ns = (ns as u64).saturating_mul(batch_size as u64);
        // Below dispatch breakeven (~50 us total): no point
        // parallelizing; treat as fine-grain so the scheduler
        // takes the inline-collapse fast path if available.
        if total_ns < 50_000 {
            return WorkloadClass::FineGrain;
        }
        // Very heavy per-item (>= 2000 ns) without a shape hint
        // is more often a long FP dependency chain (Newton
        // iteration, sqrt chain, branchy adaptive integrator)
        // than a single cache-miss gather; default to
        // LatencyBound. Callers with a gather pattern should pass
        // LeafShape::Gather.
        //
        // NOTE: tried routing batch>=16384+ns>=2000 to PortBound
        // (theory: FMA-bound streaming would benefit from SMT-off)
        // but the empirical regression was 2.76x on cold_workloads
        // 16k_10us (8.13ms LatencyBound vs 19.02ms PortBound).
        // PortBound's default oversubscription_log2 is lower so
        // fewer leaves and worse work-stealing balance on big
        // batches. LatencyBound stays the best class for heavy
        // per-item even on big batches.
        let t = class_thresholds();
        if (ns as u64) >= t.memory_latency_ns.load(Ordering::Relaxed) {
            return WorkloadClass::LatencyBound;
        }
        // Moderate-heavy per-item (between the port_heavy and
        // memory_latency boundaries) is the cache-miss-gather
        // signature (spmv, pagerank, hash probes). Pick
        // MemoryBound; observer can refine to LatencyBound if
        // observed cv^2 says otherwise.
        if (ns as u64) >= t.port_heavy_ns.load(Ordering::Relaxed) {
            return WorkloadClass::MemoryBound;
        }
        // Light per-item but a large total batch: bulk bandwidth-bound
        // streaming work over a >= 64 KiB-equivalent batch is the
        // signature pattern of byte scans, image kernels, and
        // histogram-style accumulators.
        if (ns as u64) < t.fine_grain_ns.load(Ordering::Relaxed) && batch_size >= 65536 {
            return WorkloadClass::Streaming;
        }
        // Medium per-item: integer pipeline saturating compute.
        return WorkloadClass::PortBound;
    }
    // No caller hint: fall back to K_outer + batch_size as proxies
    // for workload shape.
    if k_outer >= 7 {
        return WorkloadClass::LatencyBound;
    }
    if k_outer <= 3 && batch_size >= 16384 {
        return WorkloadClass::Streaming;
    }
    // Small batch (<=32) without a ns hint is most often a small
    // collection of heavy items: NMFD batches, ML inference passes,
    // image-tile transforms, recursive algorithm fan-outs. The
    // failure mode of routing these to PortBound (SMT-off, primaries
    // only) is that the worker pool is half-width and the user sees
    // ~2x slower than rayon on cold-cache one-off dispatches.
    // LatencyBound enables SMT, doubling effective throughput on
    // hyperthreaded hosts. The observer remains the safety net: if
    // leaf-time observation reveals the items are actually light,
    // it migrates atomically to PortBound within a few iterations.
    // Cross-host cold-workload bench shows this lift takes
    // flynnel_def from ~2x slower to TIE on NMFD-like cells.
    if batch_size <= 32 {
        return WorkloadClass::LatencyBound;
    }
    WorkloadClass::PortBound
}

/// How many consecutive auto-classifier ticks must agree to
/// migrate when the observed class is adjacent to the active class
/// (bucket distance == 1). Larger bucket distances bypass this
/// via the fast-adapt path in [`tick_auto_classify`]. Set to 2
/// so adjacent-bucket migrations happen within ~8 leaves of
/// observed evidence, fast enough for criterion warm-up to
/// converge before the measurement phase.
pub const AUTO_MIGRATION_HYSTERESIS: u32 = 2;

/// "Bucket distance" between two WorkloadClass values. Adjacent
/// buckets in the FineGrain -> PortBound -> Streaming/MemoryBound
/// -> LatencyBound progression have distance 1; the extremes have
/// distance 3. Used by the fast-adapt path: when the observer sees
/// the workload sitting at bucket-distance >= 2 from the active
/// class, it migrates IMMEDIATELY (single tick, no hysteresis)
/// because the static initial guess was clearly wrong.
#[inline]
fn bucket_index(class: WorkloadClass) -> u8 {
    match class {
        WorkloadClass::FineGrain => 0,
        WorkloadClass::PortBound => 1,
        WorkloadClass::Streaming => 2,
        WorkloadClass::MemoryBound => 2,
        WorkloadClass::LatencyBound => 3,
    }
}

#[inline]
fn bucket_distance(a: WorkloadClass, b: WorkloadClass) -> u8 {
    bucket_index(a).abs_diff(bucket_index(b))
}

/// Encoded class tag stored in a [`PendingRun`]. Same
/// encoding as [`ACTIVE_PROFILE_TAG`] for the migrated portion
/// (LatencyBound / PortBound / MemoryBound / Streaming /
/// Unspecified) plus a TAG_FINE_GRAIN value because the
/// classifier can produce WorkloadClass::FineGrain which the
/// underlying profile encoding rolls into PortBound.
const TAG_FINE_GRAIN: u8 = 5;

#[inline]
fn workload_class_to_tag(class: WorkloadClass) -> u8 {
    match class {
        WorkloadClass::FineGrain => TAG_FINE_GRAIN,
        WorkloadClass::PortBound => TAG_PORT_BOUND,
        WorkloadClass::LatencyBound => TAG_LATENCY_BOUND,
        WorkloadClass::MemoryBound => TAG_MEMORY_BOUND,
        WorkloadClass::Streaming => TAG_STREAMING,
    }
}

/// Encode a [`WorkloadClass`] into the shared u8 tag space. Used by
/// [`crate::sched::call_site::CallSiteState`] so per-site classifier
/// state stores the same encoding the process-global tag uses.
#[inline]
pub(crate) fn class_tag_encode(class: WorkloadClass) -> u8 {
    workload_class_to_tag(class)
}

/// Decode a u8 tag back to a [`WorkloadClass`]. Returns `None` for
/// any value outside the encoded range (notably the per-site
/// "uninitialized" sentinel 0xFF).
#[inline]
pub(crate) fn class_tag_decode(tag: u8) -> Option<WorkloadClass> {
    match tag {
        TAG_LATENCY_BOUND => Some(WorkloadClass::LatencyBound),
        TAG_PORT_BOUND => Some(WorkloadClass::PortBound),
        TAG_MEMORY_BOUND => Some(WorkloadClass::MemoryBound),
        TAG_STREAMING => Some(WorkloadClass::Streaming),
        TAG_FINE_GRAIN => Some(WorkloadClass::FineGrain),
        _ => None,
    }
}

/// Bucket distance between two classes on the FineGrain ->
/// PortBound -> Streaming/MemoryBound -> LatencyBound progression.
/// Shared with the per-site classifier so its fast-adapt rule uses
/// exactly the same geometry as the process-global observer.
#[inline]
pub(crate) fn class_bucket_distance(a: WorkloadClass, b: WorkloadClass) -> u8 {
    bucket_distance(a, b)
}

/// The hysteresis state the closing-loop observer carries between
/// ticks: the class the latest observations agreed on and how many of
/// them in a row. Packed into one word so a tick reads it and replaces
/// it as a unit.
#[derive(Copy, Clone, Debug, PartialEq, Eq)]
struct PendingRun {
    tag: u8,
    run: u32,
}

impl PendingRun {
    const fn fresh() -> Self {
        Self {
            tag: TAG_PORT_BOUND,
            run: 0,
        }
    }

    const fn packed(self) -> u64 {
        ((self.tag as u64) << 32) | self.run as u64
    }

    const fn unpacked(word: u64) -> Self {
        Self {
            tag: (word >> 32) as u8,
            run: word as u32,
        }
    }

    /// One observation of `observed` while `active` is the active class
    /// and the window behind it held `dcount` leaves. Returns the class
    /// to migrate to, when this observation decides one, and the state
    /// to carry forward.
    ///
    /// A class two or more buckets from the active one, backed by at
    /// least 64 leaves, migrates at once: the static guess was far off
    /// and one wide window is enough to say so. An adjacent class
    /// migrates after [`AUTO_MIGRATION_HYSTERESIS`] agreeing
    /// observations in a row. The run keeps counting past that, so a
    /// later disagreement starts it over rather than resuming it.
    fn observe(
        self,
        active: WorkloadClass,
        observed: WorkloadClass,
        dcount: u64,
    ) -> (Option<WorkloadClass>, Self) {
        let tag = workload_class_to_tag(observed);
        if active != observed && bucket_distance(active, observed) >= 2 && dcount >= 64 {
            return (Some(observed), Self { tag, run: 0 });
        }
        if self.tag != tag {
            return (None, Self { tag, run: 1 });
        }
        let run = self.run.saturating_add(1);
        let migrate = run >= AUTO_MIGRATION_HYSTERESIS && active != observed;
        (migrate.then_some(observed), Self { tag, run })
    }
}

/// The observer's [`PendingRun`], packed. Replaced by a compare-exchange
/// at every tick, so two ticks landing together each count once.
static AUTO_PENDING: core::sync::atomic::AtomicU64 =
    core::sync::atomic::AtomicU64::new(PendingRun::fresh().packed());

/// Last-seen cumulative LEAF_COUNT / LEAF_TIME_SUM_NS /
/// LEAF_TIME_SUMSQ snapshot, captured at the prior tick. The
/// next tick computes its observation window as `current - last`
/// without RESETTING the global counters -- that way tests and
/// other consumers (e.g. cv2 reporting) see the full cumulative
/// stats, while the classifier sees only the incremental window.
static AUTO_LAST_COUNT: core::sync::atomic::AtomicU64 =
    core::sync::atomic::AtomicU64::new(0);
static AUTO_LAST_SUM_NS: core::sync::atomic::AtomicU64 =
    core::sync::atomic::AtomicU64::new(0);
static AUTO_LAST_SUMSQ: core::sync::atomic::AtomicU64 =
    core::sync::atomic::AtomicU64::new(0);
/// The same snapshot for the items those leaves covered and their
/// summed per-item squared time, which is what the window is classified
/// on when the samples carried item counts.
static AUTO_LAST_ITEMS: core::sync::atomic::AtomicU64 =
    core::sync::atomic::AtomicU64::new(0);
static AUTO_LAST_SUMSQ_PER_ITEM: core::sync::atomic::AtomicU64 =
    core::sync::atomic::AtomicU64::new(0);

/// One tick of the closing-loop observer. Reads the global leaf
/// stats, computes the delta since the last tick, classifies the
/// delta window as [`classify_observed`] does, and migrates the
/// active workload class once the same observation has been
/// confirmed [`AUTO_MIGRATION_HYSTERESIS`] times in a row. A window
/// whose class turns on a spread its leaves were too short to show
/// decides nothing and leaves the pending run where it was.
///
/// Does not reset the global stats -- tests and cv^2 readers see
/// the full cumulative counters. The "fresh window per tick"
/// behavior comes from comparing the current snapshot against the
/// AUTO_LAST_* snapshot stored on the prior tick.
#[inline]
pub fn tick_auto_classify() {
    use crate::sched::split_observer::snapshot_leaf_stats;
    let stats = snapshot_leaf_stats();
    let prev_count = AUTO_LAST_COUNT.load(Ordering::Relaxed);
    let prev_sum = AUTO_LAST_SUM_NS.load(Ordering::Relaxed);
    let prev_sumsq = AUTO_LAST_SUMSQ.load(Ordering::Relaxed);

    // Window delta. If stats wrapped or were reset externally,
    // delta computations may saturate to 0; in that case skip.
    let dcount = stats.count.saturating_sub(prev_count);
    if dcount < 4 {
        return;
    }
    let dsum = stats.sum_ns.saturating_sub(prev_sum);
    let dsumsq = stats.sumsq_scaled.saturating_sub(prev_sumsq);
    let ditems = stats
        .items
        .saturating_sub(AUTO_LAST_ITEMS.load(Ordering::Relaxed));
    let dsumsq_per_item = stats
        .sumsq_per_item
        .saturating_sub(AUTO_LAST_SUMSQ_PER_ITEM.load(Ordering::Relaxed));

    AUTO_LAST_COUNT.store(stats.count, Ordering::Relaxed);
    AUTO_LAST_SUM_NS.store(stats.sum_ns, Ordering::Relaxed);
    AUTO_LAST_SUMSQ.store(stats.sumsq_scaled, Ordering::Relaxed);
    AUTO_LAST_ITEMS.store(stats.items, Ordering::Relaxed);
    AUTO_LAST_SUMSQ_PER_ITEM.store(stats.sumsq_per_item, Ordering::Relaxed);

    let (mean_ns, cv2) = window_reading(dcount, dsum, dsumsq, ditems, dsumsq_per_item);
    let Some(observed) = classify_window(mean_ns, cv2) else {
        return;
    };
    let active = active_workload_class();

    let mut word = AUTO_PENDING.load(Ordering::Relaxed);
    loop {
        let (migrate, next) = PendingRun::unpacked(word).observe(active, observed, dcount);
        match AUTO_PENDING.compare_exchange_weak(
            word,
            next.packed(),
            Ordering::Relaxed,
            Ordering::Relaxed,
        ) {
            Ok(_) => {
                if let Some(class) = migrate {
                    migrate_workload_class(class);
                }
                return;
            }
            Err(current) => word = current,
        }
    }
}

/// The mean and spread a delta window is classified on. Per item when
/// the samples carried item counts, so the reading describes the work
/// rather than the split that produced it: leaf times scale with the
/// items in a leaf, and the split follows from the class this decides.
/// A window whose samples carried no item count is read per leaf,
/// which is all such a sample can say. The spread is cv^2 in parts per
/// mille, and `None` when the mean is under the 256 ns unit its squares
/// are kept in, since no spread about a mean that small can be read.
fn window_reading(
    dcount: u64,
    dsum: u64,
    dsumsq: u64,
    ditems: u64,
    dsumsq_per_item: u64,
) -> (u64, Option<u64>) {
    if let Some(mean) = dsum.checked_div(ditems) {
        let mean_sq = ((mean as u128).saturating_mul(mean as u128) >> 16) as u64;
        let spread = (mean_sq > 0).then(|| {
            let var = dsumsq_per_item.saturating_sub(mean_sq.saturating_mul(ditems)) / ditems;
            var.saturating_mul(1000) / mean_sq
        });
        (mean, spread)
    } else {
        let mean = dsum / dcount;
        let scaled_mean = (dsum >> 8) / dcount;
        let spread = (scaled_mean > 0).then(|| {
            let sumsq_per_n = dsumsq / dcount;
            let mean_sq = scaled_mean.saturating_mul(scaled_mean);
            let var = sumsq_per_n.saturating_sub(mean_sq);
            var.saturating_mul(1000) / mean_sq
        });
        (mean, spread)
    }
}

/// Reset the auto-classifier hysteresis state: the pending run and
/// the AUTO_LAST_* snapshot counters that [`tick_auto_classify`]
/// reads to compute the delta window. After reset, the next tick
/// treats the entire current global leaf-stats counter as one fresh
/// delta window.
///
/// Public for observer-driven tests and for callers that need to
/// reset the closing-loop state at a workload-phase boundary
/// (e.g., the application starts a new bench cell with a different
/// expected workload shape and wants the observer to converge from
/// scratch rather than smooth across the old phase). The function
/// is idempotent and cheap (6 Relaxed atomic stores).
pub fn reset_auto_classify_state() {
    AUTO_PENDING.store(PendingRun::fresh().packed(), Ordering::Relaxed);
    AUTO_LAST_COUNT.store(0, Ordering::Relaxed);
    AUTO_LAST_SUM_NS.store(0, Ordering::Relaxed);
    AUTO_LAST_SUMSQ.store(0, Ordering::Relaxed);
    AUTO_LAST_ITEMS.store(0, Ordering::Relaxed);
    AUTO_LAST_SUMSQ_PER_ITEM.store(0, Ordering::Relaxed);
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn threshold_calibration_installs_clamped_values() {
        // Calibrate into a LOCAL threshold set: the probes and the
        // clamp/install logic are the code under test, and routing
        // for concurrently-running tests must not shift mid-suite.
        let local = ClassThresholds::new_defaults();
        let report = calibrate_class_thresholds_into(&local);
        assert!(
            (25..=200).contains(&report.fine_grain_ns),
            "fine_grain_ns {} outside clamp band",
            report.fine_grain_ns
        );
        assert!(
            (5_000..=200_000).contains(&report.trivial_reduce_cycles),
            "trivial_reduce_cycles {} outside clamp band",
            report.trivial_reduce_cycles
        );
        assert!(
            (1_000..=4_000).contains(&report.memory_latency_ns),
            "memory_latency_ns {} outside clamp band",
            report.memory_latency_ns
        );
        // The install target must reflect the reported values.
        assert_eq!(
            local.fine_grain_ns.load(Ordering::Relaxed),
            report.fine_grain_ns
        );
        assert_eq!(
            local.trivial_reduce_cycles.load(Ordering::Relaxed),
            report.trivial_reduce_cycles
        );
        assert_eq!(
            local.memory_latency_ns.load(Ordering::Relaxed),
            report.memory_latency_ns
        );
    }

    #[test]
    fn workload_class_maps_to_dispatch_profile() {
        assert_eq!(WorkloadClass::FineGrain.to_dispatch_profile(), DispatchProfile::PortBound);
        assert_eq!(WorkloadClass::PortBound.to_dispatch_profile(), DispatchProfile::PortBound);
        assert_eq!(WorkloadClass::LatencyBound.to_dispatch_profile(), DispatchProfile::LatencyBound);
        assert_eq!(WorkloadClass::MemoryBound.to_dispatch_profile(), DispatchProfile::MemoryBound);
        assert_eq!(WorkloadClass::Streaming.to_dispatch_profile(), DispatchProfile::Streaming);
    }

    /// Every profile survives the round trip the global cell performs,
    /// and the cell starts at PortBound.
    ///
    /// Over the tag rather than the process-wide cell: `tick_auto_classify`
    /// migrates that cell from whichever thread ran a dispatch, so any
    /// test that stored a profile and read it back could be answered by
    /// a neighboring test's classifier instead of by its own store.
    #[test]
    fn every_profile_survives_its_tag_and_the_cell_starts_port_bound() {
        for profile in [
            DispatchProfile::LatencyBound,
            DispatchProfile::PortBound,
            DispatchProfile::MemoryBound,
            DispatchProfile::Streaming,
            DispatchProfile::Unspecified,
        ] {
            assert_eq!(profile_of_tag(tag_of_profile(profile)), profile);
        }
        assert_eq!(profile_of_tag(TAG_PORT_BOUND), DispatchProfile::PortBound);
        assert_eq!(
            profile_of_tag(ACTIVE_PROFILE_TAG_DEFAULT),
            DispatchProfile::PortBound,
            "the cell's initial value is the calibration default"
        );
        assert_eq!(
            profile_of_tag(200),
            DispatchProfile::Unspecified,
            "a tag no profile is stored as reads as unspecified"
        );
    }

    #[test]
    fn classifier_picks_fine_grain_for_tiny_leaves() {
        assert_eq!(classify_observed(10, 0), WorkloadClass::FineGrain);
        assert_eq!(classify_observed(49, 1000), WorkloadClass::FineGrain);
    }

    #[test]
    fn classifier_picks_port_bound_for_medium_leaves() {
        assert_eq!(classify_observed(100, 10), WorkloadClass::PortBound);
        assert_eq!(classify_observed(499, 5000), WorkloadClass::PortBound);
    }

    #[test]
    fn classifier_picks_streaming_for_heavy_uniform_leaves() {
        assert_eq!(classify_observed(500, 0), WorkloadClass::Streaming);
        assert_eq!(classify_observed(10_000, 49), WorkloadClass::Streaming);
    }

    #[test]
    fn classifier_picks_memory_bound_for_heavy_moderate_variance() {
        assert_eq!(classify_observed(500, 100), WorkloadClass::MemoryBound);
        assert_eq!(classify_observed(2000, 499), WorkloadClass::MemoryBound);
    }

    #[test]
    fn classifier_picks_latency_bound_for_heavy_high_variance() {
        assert_eq!(classify_observed(500, 500), WorkloadClass::LatencyBound);
        assert_eq!(classify_observed(5000, 10_000), WorkloadClass::LatencyBound);
    }

    // -- Static initial classifier (infer_class_static) -----------

    #[test]
    fn static_classifier_picks_streaming_for_big_byte_scan() {
        // grep-style: ns_hint=1, batch_size=16 MiB
        assert_eq!(
            infer_class_static(6, 16 * 1024 * 1024, Some(1)),
            WorkloadClass::Streaming,
        );
    }

    #[test]
    fn static_classifier_picks_streaming_for_big_image() {
        // rgb_to_gray: ns_hint=10, batch_size=4 M pixels
        assert_eq!(
            infer_class_static(6, 2048 * 2048, Some(10)),
            WorkloadClass::Streaming,
        );
    }

    #[test]
    fn static_classifier_picks_port_bound_for_medium_per_item() {
        // kmeans: ns_hint=100, batch_size=100K
        assert_eq!(
            infer_class_static(6, 100 * 1024, Some(100)),
            WorkloadClass::PortBound,
        );
    }

    #[test]
    fn static_classifier_picks_memory_bound_for_heavy_per_item() {
        // nbody-style: ns_hint=600 (per-particle force computation)
        assert_eq!(
            infer_class_static(6, 1024, Some(600)),
            WorkloadClass::MemoryBound,
        );
    }

    #[test]
    fn static_classifier_picks_fine_grain_for_tiny_total() {
        // Tiny workload below dispatch breakeven
        assert_eq!(
            infer_class_static(6, 100, Some(10)),
            WorkloadClass::FineGrain,
        );
    }

    #[test]
    fn static_classifier_picks_latency_for_high_precision_tier() {
        // No hint, K_outer=12 (high precision)
        assert_eq!(
            infer_class_static(12, 1024, None),
            WorkloadClass::LatencyBound,
        );
    }

    #[test]
    fn static_classifier_picks_streaming_for_low_tier_large_batch() {
        // No hint, K=2 (low tier), big batch
        assert_eq!(
            infer_class_static(2, 100_000, None),
            WorkloadClass::Streaming,
        );
    }

    #[test]
    fn static_classifier_falls_back_to_port_bound_when_unsure() {
        // No hint, medium tier, medium batch -> safe middle pick
        assert_eq!(
            infer_class_static(6, 1024, None),
            WorkloadClass::PortBound,
        );
    }

    // The observer decides over `PendingRun::observe` and
    // `window_reading`, and these drive those. Driving the
    // process-global counters instead reads every other test's leaves
    // as observations, since every dispatch in the suite feeds them.

    #[test]
    fn auto_classify_migrates_after_hysteresis() {
        // Streaming sits one bucket from PortBound, so the far path does
        // not apply and the run has to reach the hysteresis.
        let mut state = PendingRun::fresh();
        for seen in 1..AUTO_MIGRATION_HYSTERESIS {
            let (migrate, next) =
                state.observe(WorkloadClass::PortBound, WorkloadClass::Streaming, 16);
            assert_eq!(
                migrate, None,
                "observation {seen} of {AUTO_MIGRATION_HYSTERESIS} migrated early"
            );
            state = next;
        }
        let (migrate, next) = state.observe(WorkloadClass::PortBound, WorkloadClass::Streaming, 16);
        assert_eq!(
            migrate,
            Some(WorkloadClass::Streaming),
            "auto-classifier did not migrate after {AUTO_MIGRATION_HYSTERESIS} consecutive \
             Streaming observations"
        );
        assert_eq!(next.run, AUTO_MIGRATION_HYSTERESIS);
    }

    #[test]
    fn auto_classify_does_not_migrate_on_a_single_outlier() {
        let (migrate, next) =
            PendingRun::fresh().observe(WorkloadClass::PortBound, WorkloadClass::Streaming, 16);
        assert_eq!(
            migrate, None,
            "auto-classifier migrated on a single observation; hysteresis broken"
        );
        assert_eq!(
            next,
            PendingRun {
                tag: workload_class_to_tag(WorkloadClass::Streaming),
                run: 1,
            }
        );
        // A disagreeing observation starts the run over.
        let (migrate, next) =
            next.observe(WorkloadClass::PortBound, WorkloadClass::MemoryBound, 16);
        assert_eq!(migrate, None);
        assert_eq!(next.run, 1);
    }

    #[test]
    fn auto_classify_migrates_at_once_when_the_observation_is_far_and_wide() {
        // LatencyBound is two buckets from PortBound. Sixty-four leaves
        // behind the window migrate on the first observation; fewer take
        // the hysteresis path.
        let (migrate, next) =
            PendingRun::fresh().observe(WorkloadClass::PortBound, WorkloadClass::LatencyBound, 64);
        assert_eq!(migrate, Some(WorkloadClass::LatencyBound));
        assert_eq!(next.run, 0);
        let (migrate, next) =
            PendingRun::fresh().observe(WorkloadClass::PortBound, WorkloadClass::LatencyBound, 16);
        assert_eq!(migrate, None);
        assert_eq!(next.run, 1);
    }

    #[test]
    fn a_window_reads_per_item_when_its_samples_carried_items() {
        // Sixty-four leaves of a millisecond each, formed the way the
        // recorder forms them: the scaled square per leaf, and the
        // squared time over the items a leaf covered.
        let leaves = 64u64;
        let leaf_ns = 1_000_000u64;
        let scaled = leaf_ns >> 8;
        let dsum = leaf_ns * leaves;
        let dsumsq = scaled * scaled * leaves;

        // No items: read per leaf, a flat window.
        assert_eq!(
            window_reading(leaves, dsum, dsumsq, 0, 0),
            (leaf_ns, Some(0))
        );

        // Sixteen items a leaf: read per item, still flat.
        let items = 16u64;
        let per_item_sq = ((leaf_ns as u128 * leaf_ns as u128) / ((items as u128) << 16)) as u64;
        let (mean, cv2) =
            window_reading(leaves, dsum, dsumsq, leaves * items, per_item_sq * leaves);
        assert_eq!(mean, leaf_ns / items);
        let cv2 = cv2.expect("a 62,500 ns item is well above the unit");
        assert!(
            cv2 < 50,
            "identical leaves read as a flat window; got {cv2}"
        );
    }

    #[test]
    fn a_window_under_the_unit_reads_its_mean_and_no_spread() {
        // Sixty-four leaves of 200 ns with no items, and the same leaves
        // as four items each: both means are measured, and neither has a
        // spread, because 200 ns and 50 ns both scale to nothing.
        let leaves = 64u64;
        let leaf_ns = 200u64;
        let dsum = leaf_ns * leaves;
        assert_eq!(window_reading(leaves, dsum, 0, 0, 0), (leaf_ns, None));
        assert_eq!(
            window_reading(leaves, dsum, 0, leaves * 4, 0),
            (leaf_ns / 4, None)
        );
    }

    #[test]
    fn a_window_decides_by_its_mean_where_the_mean_decides() {
        // Below the heavy boundary the mean alone places the window, so
        // a spread that could not be read changes nothing.
        let port_heavy = class_thresholds().port_heavy_ns.load(Ordering::Relaxed);
        assert!(port_heavy > 1, "the heavy boundary is above one nanosecond");
        let light = port_heavy - 1;
        assert_eq!(
            classify_window(light, None),
            Some(classify_observed(light, 0))
        );
        assert_eq!(
            classify_window(light, Some(900)),
            Some(classify_observed(light, 900))
        );
    }

    #[test]
    fn a_heavy_window_with_no_readable_spread_decides_nothing() {
        let heavy = class_thresholds().port_heavy_ns.load(Ordering::Relaxed) + 1_000;
        assert_eq!(classify_window(heavy, None), None);
        assert_eq!(
            classify_window(heavy, Some(0)),
            Some(WorkloadClass::Streaming)
        );
        assert_eq!(
            classify_window(heavy, Some(10_000)),
            Some(WorkloadClass::LatencyBound)
        );
    }

    /// A class reaches the cell as its profile and comes back as a
    /// class, over the whole round trip a migration performs, and
    /// FineGrain comes back as PortBound because the profile side has
    /// no such variant.
    #[test]
    fn a_class_survives_the_round_trip_a_migration_performs() {
        for class in [
            WorkloadClass::LatencyBound,
            WorkloadClass::MemoryBound,
            WorkloadClass::Streaming,
            WorkloadClass::PortBound,
        ] {
            let profile = class.to_dispatch_profile();
            assert_eq!(
                class_of_profile(profile_of_tag(tag_of_profile(profile))),
                class
            );
        }
        assert_eq!(
            class_of_profile(profile_of_tag(tag_of_profile(
                WorkloadClass::FineGrain.to_dispatch_profile()
            ))),
            WorkloadClass::PortBound,
            "the profile side has no FineGrain, so it collapses to PortBound"
        );
    }

}