flynnel 0.6.0

K-aware NUMA-aware work-stealing scheduler with extended-Flynn-taxonomy dispatch (MIMD / SIMC / MIMC / MISD / SIMT / MIMT): a rayon-shaped pool reshaped around per-call execution-class plans with first-class CPU+GPU hybrid (join_hybrid, hybrid_pipeline) and per-call dispatch profiles.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
993
994
995
996
997
998
999
1000
1001
1002
1003
1004
1005
1006
1007
1008
1009
1010
1011
1012
1013
1014
1015
1016
1017
1018
1019
1020
1021
1022
1023
1024
1025
1026
1027
1028
1029
1030
1031
1032
1033
1034
1035
1036
1037
1038
1039
1040
1041
1042
1043
1044
1045
1046
1047
1048
1049
1050
1051
1052
1053
1054
1055
1056
1057
1058
1059
1060
1061
1062
1063
1064
1065
1066
1067
1068
1069
1070
1071
1072
1073
1074
1075
1076
1077
1078
1079
1080
1081
1082
1083
1084
1085
1086
1087
1088
1089
1090
1091
1092
1093
1094
1095
1096
1097
1098
1099
1100
1101
1102
1103
1104
1105
1106
1107
1108
1109
1110
1111
1112
1113
1114
1115
1116
1117
1118
1119
1120
1121
1122
1123
1124
1125
1126
1127
1128
1129
1130
1131
1132
1133
1134
1135
1136
1137
1138
1139
1140
1141
1142
1143
1144
1145
1146
1147
1148
1149
1150
1151
1152
1153
1154
1155
1156
1157
1158
1159
1160
1161
1162
1163
1164
1165
1166
1167
1168
1169
1170
1171
1172
1173
1174
1175
1176
1177
1178
1179
1180
1181
1182
1183
1184
1185
1186
1187
1188
1189
1190
1191
1192
1193
1194
1195
1196
1197
1198
1199
1200
1201
1202
1203
1204
1205
1206
1207
1208
1209
1210
1211
1212
1213
1214
1215
1216
1217
1218
1219
1220
1221
1222
1223
1224
1225
1226
1227
1228
1229
1230
1231
1232
1233
1234
1235
1236
1237
1238
1239
1240
1241
1242
1243
1244
1245
1246
1247
1248
1249
1250
1251
1252
1253
1254
1255
1256
1257
1258
1259
1260
1261
1262
1263
1264
1265
1266
1267
1268
1269
1270
1271
1272
1273
1274
1275
1276
1277
1278
1279
1280
1281
1282
1283
1284
1285
1286
1287
1288
1289
1290
1291
1292
1293
1294
1295
1296
1297
1298
1299
1300
1301
1302
1303
1304
1305
1306
1307
1308
1309
1310
1311
1312
1313
1314
1315
1316
1317
1318
1319
1320
1321
1322
1323
1324
1325
1326
1327
1328
1329
1330
1331
1332
1333
1334
1335
1336
1337
1338
1339
1340
1341
1342
1343
1344
1345
1346
1347
1348
1349
1350
1351
1352
1353
1354
1355
1356
1357
1358
1359
1360
1361
1362
1363
1364
1365
1366
1367
1368
1369
1370
1371
1372
1373
1374
1375
1376
1377
1378
1379
1380
1381
1382
1383
1384
1385
1386
1387
1388
1389
1390
1391
1392
1393
1394
1395
1396
1397
1398
1399
1400
1401
1402
1403
1404
1405
1406
1407
1408
1409
1410
1411
1412
1413
1414
1415
1416
1417
1418
1419
1420
1421
1422
1423
1424
1425
1426
1427
1428
1429
1430
1431
1432
1433
1434
1435
1436
1437
1438
1439
1440
1441
1442
1443
1444
1445
1446
1447
1448
1449
1450
1451
1452
1453
1454
1455
1456
1457
1458
1459
1460
1461
1462
1463
1464
1465
1466
1467
1468
1469
1470
1471
1472
1473
1474
1475
1476
1477
1478
1479
1480
1481
1482
1483
1484
1485
1486
1487
1488
1489
1490
1491
1492
1493
1494
1495
1496
1497
1498
1499
1500
1501
1502
1503
1504
1505
1506
1507
1508
1509
1510
1511
1512
1513
1514
1515
1516
1517
1518
1519
1520
1521
1522
1523
1524
1525
1526
1527
1528
1529
1530
1531
1532
1533
1534
1535
1536
1537
1538
1539
1540
1541
1542
1543
1544
1545
1546
1547
1548
1549
1550
1551
1552
1553
1554
1555
1556
1557
1558
1559
1560
1561
1562
1563
1564
1565
1566
1567
1568
1569
1570
1571
1572
1573
1574
1575
1576
1577
1578
1579
1580
1581
1582
1583
1584
1585
1586
1587
1588
1589
1590
1591
1592
1593
1594
1595
1596
1597
1598
1599
1600
1601
1602
1603
1604
1605
1606
1607
1608
1609
1610
1611
1612
1613
1614
1615
1616
1617
1618
1619
1620
1621
1622
1623
1624
1625
1626
1627
1628
1629
1630
1631
1632
1633
1634
1635
1636
1637
1638
1639
1640
1641
1642
1643
1644
1645
1646
1647
1648
1649
1650
1651
1652
1653
1654
1655
1656
1657
1658
1659
1660
1661
1662
1663
1664
1665
1666
1667
1668
1669
1670
1671
1672
1673
1674
1675
1676
1677
1678
1679
1680
1681
1682
1683
1684
1685
1686
1687
1688
1689
1690
1691
1692
1693
1694
1695
1696
1697
1698
1699
1700
1701
1702
1703
1704
1705
1706
1707
1708
1709
1710
1711
1712
1713
1714
1715
1716
1717
1718
1719
1720
1721
1722
1723
1724
1725
1726
1727
1728
1729
1730
1731
1732
1733
1734
1735
1736
1737
1738
1739
1740
1741
1742
1743
1744
1745
1746
1747
1748
1749
1750
1751
1752
1753
1754
1755
1756
1757
1758
1759
1760
1761
1762
1763
1764
1765
1766
1767
1768
1769
1770
1771
1772
1773
1774
1775
1776
1777
1778
1779
1780
1781
1782
1783
1784
1785
1786
1787
1788
1789
1790
1791
1792
1793
1794
1795
1796
1797
1798
1799
1800
1801
1802
1803
1804
1805
1806
1807
1808
1809
1810
1811
1812
1813
1814
1815
1816
1817
1818
1819
1820
1821
1822
1823
1824
1825
1826
1827
1828
1829
1830
1831
1832
1833
1834
1835
1836
1837
1838
1839
1840
1841
1842
1843
1844
1845
1846
1847
1848
1849
1850
1851
1852
1853
1854
1855
1856
1857
1858
1859
1860
1861
1862
1863
1864
1865
1866
1867
1868
1869
1870
1871
1872
1873
1874
1875
1876
1877
1878
1879
1880
1881
1882
1883
1884
1885
1886
1887
1888
1889
1890
1891
1892
1893
1894
1895
1896
1897
1898
1899
1900
1901
1902
1903
1904
1905
1906
1907
1908
1909
1910
1911
1912
1913
1914
1915
1916
1917
1918
1919
1920
1921
1922
1923
1924
1925
1926
1927
1928
1929
1930
1931
1932
1933
1934
1935
1936
1937
1938
1939
1940
1941
1942
1943
1944
1945
1946
1947
1948
1949
1950
1951
1952
1953
1954
1955
1956
1957
1958
1959
1960
1961
1962
1963
1964
1965
1966
1967
1968
1969
1970
1971
1972
1973
1974
1975
1976
1977
1978
1979
1980
1981
1982
1983
1984
1985
1986
1987
1988
1989
1990
1991
1992
1993
1994
1995
1996
1997
1998
1999
2000
2001
2002
2003
2004
2005
2006
2007
2008
2009
2010
2011
2012
2013
2014
2015
2016
2017
2018
2019
2020
2021
2022
2023
2024
2025
2026
2027
2028
2029
2030
2031
2032
2033
2034
2035
2036
2037
2038
2039
2040
2041
2042
2043
2044
2045
2046
2047
2048
2049
2050
2051
2052
2053
2054
2055
2056
2057
2058
2059
2060
2061
2062
2063
2064
2065
2066
2067
2068
2069
2070
2071
2072
2073
2074
2075
2076
2077
2078
2079
2080
2081
2082
2083
2084
2085
2086
2087
2088
2089
2090
2091
2092
2093
2094
2095
2096
2097
2098
2099
2100
2101
2102
2103
2104
2105
2106
2107
2108
2109
2110
2111
2112
2113
2114
2115
2116
2117
2118
2119
2120
2121
2122
2123
2124
2125
2126
2127
2128
2129
2130
2131
2132
2133
2134
2135
2136
2137
2138
2139
2140
2141
2142
2143
2144
2145
2146
2147
2148
2149
2150
2151
2152
2153
2154
2155
2156
2157
2158
2159
2160
2161
2162
2163
2164
2165
2166
2167
2168
2169
2170
2171
2172
2173
2174
2175
2176
2177
2178
2179
2180
2181
2182
2183
2184
2185
2186
2187
2188
2189
2190
2191
2192
2193
2194
2195
2196
2197
2198
2199
2200
2201
2202
2203
2204
2205
2206
2207
2208
2209
2210
2211
2212
2213
2214
2215
2216
2217
2218
2219
2220
2221
2222
2223
2224
2225
2226
2227
2228
2229
2230
2231
2232
2233
2234
2235
2236
2237
2238
2239
2240
2241
2242
2243
2244
2245
2246
2247
2248
2249
2250
2251
2252
2253
2254
2255
2256
2257
2258
2259
2260
2261
2262
2263
2264
2265
2266
2267
2268
2269
2270
2271
2272
2273
2274
2275
2276
2277
2278
2279
2280
2281
2282
2283
2284
2285
2286
2287
2288
2289
2290
2291
2292
2293
2294
2295
2296
2297
2298
2299
2300
2301
2302
2303
2304
2305
2306
2307
2308
2309
2310
2311
2312
2313
2314
2315
2316
2317
2318
2319
2320
2321
2322
2323
2324
2325
2326
2327
2328
2329
2330
2331
2332
2333
2334
2335
2336
2337
2338
2339
2340
2341
2342
2343
2344
2345
2346
2347
2348
2349
2350
2351
2352
2353
2354
2355
2356
2357
2358
2359
2360
2361
2362
2363
2364
2365
2366
2367
2368
2369
2370
2371
2372
2373
2374
2375
2376
2377
2378
2379
2380
2381
2382
2383
2384
2385
2386
2387
2388
2389
2390
2391
2392
2393
2394
2395
2396
2397
2398
2399
2400
2401
2402
2403
2404
2405
2406
2407
2408
2409
2410
2411
2412
2413
2414
2415
2416
2417
2418
2419
2420
2421
2422
2423
2424
2425
2426
2427
2428
2429
2430
2431
2432
2433
2434
2435
2436
2437
2438
2439
2440
2441
2442
2443
2444
2445
2446
2447
2448
2449
2450
2451
2452
2453
2454
2455
2456
2457
2458
2459
2460
2461
2462
2463
2464
2465
2466
2467
2468
2469
2470
2471
2472
2473
2474
2475
2476
2477
2478
2479
2480
2481
2482
2483
2484
2485
2486
2487
2488
2489
2490
2491
2492
2493
2494
2495
2496
2497
2498
2499
2500
2501
2502
2503
2504
2505
2506
2507
2508
2509
2510
2511
2512
2513
2514
2515
2516
2517
2518
2519
2520
2521
2522
2523
2524
2525
2526
2527
2528
2529
2530
2531
2532
2533
2534
2535
2536
2537
2538
2539
2540
2541
2542
2543
2544
2545
2546
2547
2548
2549
2550
2551
2552
2553
2554
2555
2556
2557
2558
2559
2560
2561
2562
2563
2564
2565
2566
2567
2568
2569
2570
2571
2572
2573
2574
2575
2576
2577
2578
2579
2580
2581
2582
2583
2584
2585
2586
2587
2588
2589
2590
2591
2592
2593
2594
2595
2596
2597
2598
2599
2600
2601
2602
2603
2604
2605
2606
2607
2608
2609
2610
2611
2612
2613
2614
2615
2616
2617
2618
2619
2620
2621
2622
2623
2624
2625
2626
2627
2628
2629
2630
2631
2632
2633
2634
2635
2636
2637
2638
2639
2640
2641
2642
2643
2644
2645
2646
2647
2648
2649
2650
2651
2652
2653
2654
2655
2656
2657
2658
2659
2660
2661
2662
2663
2664
2665
2666
2667
2668
2669
2670
2671
2672
2673
2674
2675
2676
2677
2678
2679
2680
2681
2682
2683
2684
2685
2686
2687
2688
2689
2690
2691
2692
2693
2694
2695
2696
2697
2698
2699
2700
2701
2702
2703
2704
2705
2706
2707
2708
2709
2710
2711
2712
2713
2714
2715
2716
2717
2718
2719
2720
2721
2722
2723
2724
2725
2726
2727
2728
2729
2730
2731
2732
2733
2734
2735
2736
2737
2738
2739
2740
2741
2742
2743
2744
2745
2746
2747
2748
2749
2750
2751
2752
2753
2754
2755
2756
2757
2758
2759
2760
2761
2762
2763
2764
2765
2766
2767
2768
2769
2770
2771
2772
2773
2774
2775
2776
2777
2778
2779
2780
2781
2782
2783
2784
2785
2786
2787
2788
2789
2790
2791
2792
2793
2794
2795
2796
2797
2798
2799
2800
2801
2802
2803
2804
2805
2806
2807
2808
2809
2810
2811
2812
2813
2814
2815
2816
2817
2818
2819
2820
2821
2822
2823
2824
2825
2826
2827
2828
2829
2830
2831
2832
2833
2834
2835
2836
2837
2838
2839
2840
2841
2842
2843
2844
2845
2846
2847
2848
2849
2850
2851
2852
2853
2854
2855
2856
2857
2858
2859
2860
2861
2862
2863
2864
2865
2866
2867
2868
2869
2870
2871
2872
2873
2874
2875
2876
2877
2878
2879
2880
2881
2882
2883
2884
2885
2886
2887
2888
2889
2890
2891
2892
2893
2894
2895
2896
2897
2898
2899
2900
2901
2902
2903
2904
2905
2906
2907
2908
2909
2910
2911
2912
2913
2914
2915
2916
2917
2918
2919
2920
2921
2922
2923
2924
2925
2926
2927
2928
2929
2930
2931
2932
2933
2934
2935
2936
2937
2938
2939
2940
2941
2942
2943
2944
2945
2946
2947
2948
2949
2950
2951
2952
2953
2954
2955
2956
2957
2958
2959
2960
2961
2962
2963
2964
2965
2966
2967
2968
2969
2970
2971
2972
2973
2974
2975
2976
2977
2978
2979
2980
2981
2982
2983
2984
2985
2986
2987
2988
2989
2990
2991
2992
2993
2994
2995
2996
2997
2998
2999
3000
3001
3002
3003
3004
3005
3006
3007
3008
3009
3010
3011
3012
3013
3014
3015
3016
3017
3018
3019
3020
3021
3022
3023
3024
3025
3026
3027
3028
3029
3030
3031
3032
3033
3034
3035
3036
3037
3038
3039
3040
3041
3042
3043
3044
3045
3046
3047
3048
3049
3050
3051
3052
3053
3054
3055
3056
3057
3058
3059
3060
3061
3062
3063
3064
3065
3066
3067
3068
3069
3070
3071
3072
3073
3074
3075
3076
3077
3078
3079
3080
3081
3082
3083
3084
3085
3086
3087
3088
3089
3090
3091
3092
3093
3094
3095
3096
3097
3098
3099
3100
3101
3102
3103
3104
3105
3106
3107
3108
3109
3110
3111
3112
3113
3114
3115
3116
3117
3118
3119
3120
3121
3122
3123
3124
3125
3126
3127
3128
3129
3130
3131
3132
3133
3134
3135
3136
3137
3138
3139
3140
3141
3142
3143
3144
3145
3146
3147
3148
3149
3150
3151
3152
3153
3154
3155
3156
3157
3158
3159
3160
3161
3162
3163
3164
3165
3166
3167
3168
3169
3170
3171
3172
3173
3174
3175
3176
3177
3178
3179
3180
3181
3182
3183
3184
3185
3186
3187
3188
3189
3190
3191
3192
3193
3194
3195
3196
3197
3198
3199
3200
3201
3202
3203
3204
3205
3206
3207
3208
3209
3210
3211
3212
3213
3214
3215
3216
3217
3218
3219
3220
3221
3222
3223
3224
3225
3226
3227
3228
3229
3230
3231
3232
3233
3234
3235
3236
3237
3238
3239
3240
3241
3242
3243
3244
3245
3246
3247
3248
3249
3250
3251
3252
3253
3254
3255
3256
3257
3258
3259
3260
3261
3262
3263
3264
3265
3266
3267
3268
3269
3270
3271
3272
3273
3274
3275
3276
3277
3278
3279
3280
3281
3282
3283
3284
3285
3286
3287
3288
3289
3290
3291
3292
3293
3294
3295
3296
3297
3298
3299
3300
3301
3302
3303
3304
3305
3306
3307
3308
3309
3310
3311
3312
3313
3314
3315
3316
3317
3318
3319
3320
3321
3322
3323
3324
3325
3326
3327
3328
3329
3330
3331
3332
3333
3334
3335
3336
3337
3338
3339
3340
3341
3342
3343
3344
3345
3346
3347
3348
3349
3350
3351
3352
3353
3354
3355
3356
3357
3358
3359
3360
3361
3362
3363
3364
3365
3366
3367
3368
3369
3370
3371
3372
3373
3374
3375
3376
3377
3378
3379
3380
3381
3382
3383
3384
3385
3386
3387
3388
3389
3390
3391
3392
3393
3394
3395
3396
3397
3398
3399
3400
3401
3402
3403
3404
3405
3406
3407
3408
3409
3410
3411
3412
3413
3414
3415
3416
3417
3418
3419
3420
3421
3422
3423
3424
3425
3426
3427
3428
3429
3430
3431
3432
3433
3434
3435
3436
3437
3438
3439
3440
3441
3442
3443
3444
3445
3446
3447
3448
3449
3450
3451
3452
3453
3454
3455
3456
3457
3458
3459
3460
3461
3462
3463
3464
3465
3466
3467
3468
3469
3470
3471
3472
3473
3474
3475
3476
3477
3478
3479
3480
3481
3482
3483
3484
3485
3486
3487
3488
3489
3490
3491
3492
3493
3494
3495
3496
3497
3498
3499
3500
3501
3502
3503
3504
3505
3506
3507
3508
3509
3510
3511
3512
3513
3514
3515
3516
3517
3518
3519
3520
3521
3522
3523
3524
3525
3526
3527
3528
3529
3530
3531
3532
3533
3534
3535
3536
3537
3538
3539
3540
3541
3542
3543
3544
3545
3546
3547
3548
3549
3550
3551
3552
3553
3554
3555
3556
3557
3558
3559
3560
3561
3562
3563
3564
3565
3566
3567
3568
3569
3570
3571
3572
3573
3574
3575
3576
3577
3578
3579
3580
3581
3582
3583
3584
3585
3586
3587
3588
3589
3590
3591
3592
3593
3594
3595
3596
3597
3598
3599
3600
3601
3602
3603
3604
3605
3606
3607
3608
3609
3610
3611
3612
3613
3614
3615
3616
3617
3618
3619
3620
3621
3622
3623
3624
3625
3626
3627
3628
3629
3630
3631
3632
3633
3634
3635
3636
3637
3638
3639
3640
3641
3642
3643
3644
3645
3646
3647
3648
3649
3650
3651
3652
3653
3654
3655
3656
3657
3658
3659
3660
3661
3662
3663
3664
3665
3666
3667
3668
3669
3670
3671
3672
3673
3674
3675
3676
3677
3678
3679
3680
3681
3682
3683
3684
3685
3686
3687
3688
3689
3690
3691
3692
3693
3694
3695
3696
3697
3698
3699
3700
3701
3702
3703
3704
3705
3706
3707
3708
3709
3710
3711
3712
3713
3714
3715
3716
3717
3718
3719
3720
3721
3722
3723
3724
3725
3726
3727
3728
3729
3730
3731
3732
3733
3734
3735
3736
3737
3738
3739
3740
3741
3742
3743
3744
3745
3746
3747
3748
3749
3750
3751
3752
3753
3754
3755
3756
3757
3758
3759
3760
3761
3762
3763
3764
3765
3766
3767
3768
3769
3770
3771
3772
3773
3774
3775
3776
3777
3778
3779
3780
3781
3782
3783
3784
3785
3786
3787
3788
3789
3790
3791
3792
3793
3794
3795
3796
3797
3798
3799
3800
3801
3802
3803
3804
3805
3806
3807
3808
3809
3810
3811
3812
3813
3814
3815
3816
3817
3818
3819
3820
3821
3822
3823
3824
3825
3826
3827
3828
3829
3830
3831
3832
3833
3834
3835
3836
3837
3838
3839
3840
3841
3842
3843
3844
3845
3846
3847
3848
3849
3850
3851
3852
3853
3854
3855
3856
3857
3858
3859
3860
3861
3862
3863
3864
3865
3866
3867
3868
3869
3870
3871
3872
3873
3874
3875
3876
3877
3878
3879
3880
3881
3882
3883
3884
3885
3886
3887
3888
3889
3890
3891
3892
3893
3894
3895
3896
3897
3898
3899
3900
3901
3902
3903
3904
3905
3906
3907
3908
3909
3910
3911
3912
3913
3914
3915
3916
3917
3918
3919
3920
3921
3922
3923
3924
3925
3926
3927
3928
3929
3930
3931
3932
3933
3934
3935
3936
3937
3938
3939
3940
3941
3942
3943
3944
3945
3946
3947
3948
3949
3950
3951
3952
3953
3954
3955
3956
3957
3958
3959
3960
3961
3962
3963
3964
3965
3966
3967
3968
3969
3970
3971
3972
3973
3974
3975
3976
3977
3978
3979
3980
3981
3982
3983
3984
3985
3986
3987
3988
3989
3990
3991
3992
3993
3994
3995
3996
3997
3998
3999
4000
4001
4002
4003
4004
4005
4006
4007
4008
4009
4010
4011
4012
4013
4014
4015
4016
4017
4018
4019
4020
4021
4022
4023
4024
4025
4026
4027
4028
4029
4030
4031
4032
4033
4034
4035
4036
4037
4038
4039
4040
4041
4042
4043
4044
4045
4046
4047
4048
4049
4050
4051
4052
4053
4054
4055
4056
4057
4058
4059
4060
4061
4062
4063
4064
4065
4066
4067
4068
4069
4070
4071
4072
4073
4074
4075
4076
4077
4078
4079
4080
4081
4082
4083
4084
4085
4086
4087
4088
4089
4090
4091
4092
4093
4094
4095
4096
4097
4098
4099
4100
4101
4102
4103
4104
4105
4106
4107
4108
4109
4110
4111
4112
4113
4114
4115
4116
4117
4118
4119
4120
4121
4122
4123
4124
4125
4126
4127
4128
4129
4130
4131
4132
4133
4134
4135
4136
4137
4138
4139
4140
4141
4142
4143
4144
4145
4146
4147
4148
4149
4150
4151
4152
4153
4154
4155
4156
4157
4158
4159
4160
4161
4162
4163
4164
4165
4166
4167
4168
4169
4170
4171
4172
4173
4174
4175
4176
4177
4178
4179
4180
4181
4182
4183
4184
4185
4186
4187
4188
4189
4190
4191
4192
4193
4194
4195
4196
4197
4198
4199
4200
4201
4202
4203
4204
4205
4206
4207
4208
4209
4210
4211
4212
4213
4214
4215
4216
4217
4218
4219
4220
4221
4222
4223
4224
4225
4226
4227
4228
4229
4230
4231
4232
4233
4234
4235
4236
4237
4238
4239
4240
4241
4242
4243
4244
4245
4246
4247
4248
4249
4250
4251
4252
4253
4254
4255
4256
4257
4258
4259
4260
4261
4262
4263
4264
4265
4266
4267
4268
4269
4270
4271
4272
4273
4274
4275
4276
4277
4278
4279
4280
4281
4282
4283
4284
4285
4286
4287
4288
4289
4290
4291
4292
4293
4294
4295
4296
4297
4298
4299
4300
4301
4302
4303
4304
4305
4306
4307
4308
4309
4310
4311
4312
4313
4314
4315
4316
4317
4318
4319
4320
4321
4322
4323
4324
4325
4326
4327
4328
4329
4330
4331
4332
4333
4334
4335
4336
4337
4338
4339
4340
4341
4342
4343
4344
4345
4346
4347
4348
4349
4350
4351
4352
4353
4354
4355
4356
4357
4358
4359
4360
4361
4362
4363
4364
4365
4366
4367
4368
4369
4370
4371
4372
4373
4374
4375
4376
4377
4378
4379
4380
4381
4382
4383
4384
4385
4386
4387
4388
4389
4390
4391
4392
4393
4394
4395
4396
4397
4398
4399
4400
4401
4402
4403
4404
4405
4406
4407
4408
4409
4410
4411
4412
4413
4414
4415
4416
4417
4418
4419
4420
4421
4422
4423
4424
4425
4426
4427
4428
4429
4430
4431
4432
4433
4434
4435
4436
4437
4438
4439
4440
4441
4442
4443
4444
4445
4446
4447
4448
4449
4450
4451
4452
4453
4454
4455
4456
4457
4458
4459
4460
4461
4462
4463
4464
4465
4466
4467
4468
4469
4470
4471
4472
4473
4474
4475
4476
4477
4478
4479
4480
4481
4482
4483
4484
4485
4486
4487
4488
4489
4490
4491
4492
4493
4494
4495
4496
4497
4498
4499
4500
4501
4502
4503
4504
4505
4506
4507
4508
4509
4510
4511
4512
4513
4514
4515
4516
4517
4518
4519
4520
4521
4522
4523
4524
4525
4526
4527
4528
4529
4530
4531
4532
4533
4534
4535
4536
4537
4538
4539
4540
4541
4542
4543
4544
4545
4546
4547
4548
4549
4550
4551
4552
4553
4554
4555
4556
4557
4558
4559
4560
4561
4562
4563
4564
4565
4566
4567
4568
4569
4570
4571
4572
4573
4574
4575
4576
4577
4578
4579
4580
4581
4582
4583
4584
4585
4586
4587
4588
4589
4590
4591
4592
4593
4594
4595
4596
4597
4598
4599
4600
4601
4602
4603
4604
4605
4606
4607
4608
4609
4610
4611
4612
4613
4614
4615
4616
4617
4618
4619
4620
4621
4622
4623
4624
4625
4626
4627
4628
4629
4630
4631
4632
4633
4634
4635
4636
4637
4638
4639
4640
4641
4642
4643
4644
4645
4646
4647
4648
4649
4650
4651
4652
4653
4654
4655
4656
4657
4658
4659
4660
4661
4662
4663
4664
4665
4666
4667
4668
4669
4670
4671
4672
4673
4674
4675
4676
4677
4678
4679
4680
4681
4682
4683
4684
4685
4686
4687
4688
4689
4690
4691
4692
4693
4694
4695
4696
4697
4698
4699
4700
4701
4702
4703
4704
4705
4706
4707
4708
4709
4710
4711
4712
4713
4714
4715
4716
4717
4718
4719
4720
4721
4722
4723
4724
4725
4726
4727
4728
4729
4730
4731
4732
4733
4734
4735
4736
4737
4738
4739
4740
4741
4742
4743
4744
4745
4746
4747
4748
4749
4750
4751
4752
4753
4754
4755
4756
4757
4758
4759
4760
4761
4762
4763
4764
4765
4766
4767
4768
4769
4770
4771
4772
4773
4774
4775
4776
4777
4778
4779
4780
4781
4782
4783
4784
4785
4786
4787
4788
4789
4790
4791
4792
4793
4794
4795
4796
4797
4798
4799
4800
4801
4802
4803
4804
4805
4806
4807
4808
4809
4810
4811
4812
4813
4814
4815
4816
4817
4818
4819
4820
4821
4822
4823
4824
4825
4826
4827
4828
4829
4830
4831
4832
4833
4834
4835
4836
4837
4838
4839
4840
4841
4842
4843
4844
4845
4846
4847
4848
4849
4850
4851
4852
4853
4854
4855
4856
4857
4858
4859
4860
4861
4862
4863
4864
4865
4866
4867
4868
4869
4870
4871
4872
4873
4874
4875
4876
4877
4878
4879
4880
4881
4882
4883
4884
4885
4886
4887
4888
4889
4890
4891
4892
4893
4894
4895
4896
4897
4898
4899
4900
4901
4902
4903
4904
4905
4906
4907
4908
4909
4910
4911
4912
4913
4914
4915
4916
4917
4918
4919
4920
4921
4922
4923
4924
4925
4926
4927
4928
4929
4930
4931
4932
4933
4934
4935
4936
4937
4938
4939
4940
4941
4942
4943
4944
4945
4946
4947
4948
4949
4950
4951
4952
4953
4954
4955
4956
4957
4958
4959
4960
4961
4962
4963
4964
4965
4966
4967
4968
4969
4970
4971
4972
4973
4974
4975
4976
4977
4978
4979
4980
4981
4982
4983
4984
4985
4986
4987
4988
4989
4990
4991
4992
4993
4994
4995
4996
4997
4998
4999
5000
5001
5002
5003
5004
5005
5006
5007
5008
5009
5010
5011
5012
5013
5014
5015
5016
5017
5018
5019
5020
5021
5022
5023
5024
5025
5026
5027
5028
5029
5030
5031
5032
5033
5034
5035
5036
5037
5038
5039
5040
5041
5042
5043
5044
5045
5046
5047
5048
5049
5050
5051
5052
5053
5054
5055
5056
5057
5058
5059
5060
5061
5062
5063
5064
5065
5066
5067
5068
5069
5070
5071
5072
5073
5074
5075
5076
5077
5078
5079
5080
5081
5082
5083
5084
5085
5086
5087
5088
5089
5090
5091
5092
5093
5094
5095
5096
5097
5098
5099
5100
5101
5102
5103
5104
5105
5106
5107
5108
5109
5110
5111
5112
5113
5114
5115
5116
5117
5118
5119
5120
5121
5122
5123
5124
5125
5126
5127
5128
5129
5130
5131
5132
5133
5134
5135
5136
5137
5138
5139
5140
5141
5142
5143
5144
5145
5146
5147
5148
5149
5150
5151
5152
5153
5154
5155
5156
5157
5158
5159
5160
5161
5162
5163
5164
5165
5166
5167
5168
5169
5170
5171
5172
5173
5174
5175
5176
5177
5178
5179
5180
5181
5182
5183
5184
5185
5186
5187
5188
5189
5190
5191
5192
5193
5194
5195
5196
5197
5198
5199
5200
5201
5202
5203
5204
5205
5206
5207
5208
5209
5210
5211
5212
5213
5214
5215
5216
5217
5218
5219
5220
5221
5222
5223
5224
5225
5226
5227
5228
5229
5230
5231
5232
5233
5234
5235
5236
5237
5238
5239
5240
5241
5242
5243
5244
5245
5246
5247
5248
5249
5250
5251
5252
5253
5254
5255
5256
5257
5258
5259
5260
5261
5262
5263
5264
5265
5266
5267
5268
5269
5270
5271
5272
5273
5274
5275
5276
5277
5278
5279
5280
5281
5282
5283
5284
5285
5286
5287
5288
5289
5290
5291
5292
5293
5294
5295
5296
5297
5298
5299
5300
5301
5302
5303
5304
5305
5306
5307
5308
5309
5310
5311
5312
5313
5314
5315
5316
5317
5318
5319
5320
5321
5322
5323
5324
5325
5326
5327
5328
5329
5330
5331
5332
5333
5334
5335
5336
5337
5338
5339
5340
5341
5342
5343
5344
5345
5346
5347
5348
5349
5350
5351
5352
5353
5354
5355
5356
5357
5358
5359
5360
5361
5362
5363
5364
5365
5366
5367
5368
5369
5370
5371
5372
5373
5374
5375
5376
5377
5378
5379
5380
5381
5382
5383
5384
5385
5386
5387
5388
5389
5390
5391
5392
5393
5394
5395
5396
5397
5398
5399
5400
5401
5402
5403
5404
5405
5406
5407
5408
5409
5410
5411
5412
5413
5414
5415
5416
5417
5418
5419
5420
5421
5422
5423
5424
5425
5426
5427
5428
5429
5430
5431
5432
5433
5434
5435
5436
5437
5438
5439
5440
5441
5442
5443
5444
5445
5446
5447
5448
5449
5450
5451
5452
5453
5454
5455
5456
5457
5458
5459
5460
5461
5462
5463
5464
5465
5466
5467
5468
5469
5470
5471
5472
5473
5474
5475
5476
5477
5478
5479
5480
5481
5482
5483
5484
5485
5486
5487
5488
5489
5490
5491
5492
5493
5494
5495
5496
5497
5498
5499
5500
5501
5502
5503
5504
5505
5506
5507
5508
5509
5510
5511
5512
5513
5514
5515
5516
5517
5518
5519
5520
5521
5522
5523
5524
5525
5526
5527
5528
5529
5530
5531
5532
5533
5534
5535
5536
5537
5538
5539
5540
5541
5542
5543
5544
5545
5546
5547
5548
5549
5550
5551
5552
5553
5554
5555
5556
5557
5558
5559
5560
5561
5562
5563
5564
5565
5566
5567
5568
5569
5570
5571
5572
5573
5574
5575
5576
5577
5578
5579
5580
5581
5582
5583
5584
5585
5586
5587
5588
5589
5590
5591
5592
5593
5594
5595
5596
5597
5598
5599
5600
5601
5602
5603
5604
5605
5606
5607
5608
5609
5610
5611
5612
5613
5614
5615
5616
5617
5618
5619
5620
5621
5622
5623
5624
5625
5626
5627
5628
5629
5630
5631
5632
5633
5634
5635
5636
5637
5638
5639
5640
5641
5642
5643
5644
5645
5646
5647
5648
5649
5650
5651
5652
5653
5654
5655
5656
5657
5658
5659
5660
5661
5662
5663
5664
5665
5666
5667
5668
5669
5670
5671
5672
5673
5674
5675
5676
5677
5678
5679
5680
5681
5682
5683
5684
5685
5686
5687
5688
5689
5690
5691
5692
5693
5694
5695
5696
5697
5698
5699
5700
5701
5702
5703
5704
5705
5706
5707
5708
5709
5710
5711
5712
5713
5714
5715
5716
5717
5718
5719
5720
5721
5722
5723
5724
5725
5726
5727
5728
5729
5730
5731
5732
5733
5734
5735
5736
5737
5738
5739
5740
5741
5742
5743
5744
5745
5746
5747
5748
5749
5750
5751
5752
5753
5754
5755
5756
5757
5758
5759
5760
5761
5762
5763
5764
5765
5766
5767
5768
5769
5770
5771
5772
5773
5774
5775
5776
5777
5778
5779
5780
5781
5782
5783
5784
5785
5786
5787
5788
5789
5790
5791
5792
5793
5794
5795
5796
5797
5798
5799
5800
5801
5802
5803
5804
5805
5806
5807
5808
5809
5810
5811
5812
5813
5814
5815
5816
5817
5818
5819
5820
5821
5822
5823
5824
5825
5826
5827
5828
5829
5830
5831
5832
5833
5834
5835
5836
5837
5838
5839
5840
5841
5842
5843
5844
5845
5846
5847
5848
5849
5850
5851
5852
5853
5854
5855
5856
5857
5858
5859
5860
5861
5862
5863
5864
5865
5866
5867
5868
5869
5870
5871
5872
5873
5874
5875
5876
5877
5878
5879
5880
5881
5882
5883
5884
5885
5886
5887
5888
5889
5890
5891
5892
5893
5894
5895
5896
5897
5898
5899
5900
5901
5902
5903
5904
5905
5906
5907
5908
5909
5910
5911
5912
5913
5914
5915
5916
5917
5918
5919
5920
5921
5922
5923
5924
5925
5926
5927
5928
5929
5930
5931
5932
5933
5934
5935
5936
5937
5938
5939
5940
5941
5942
5943
5944
5945
5946
5947
5948
5949
5950
5951
5952
5953
5954
5955
5956
5957
5958
5959
5960
5961
5962
5963
5964
5965
5966
5967
5968
5969
5970
5971
5972
5973
5974
5975
5976
5977
5978
5979
5980
5981
5982
5983
5984
5985
5986
5987
5988
5989
5990
5991
5992
5993
5994
5995
5996
5997
5998
5999
6000
6001
6002
6003
6004
6005
6006
6007
6008
6009
6010
6011
6012
6013
6014
6015
6016
6017
6018
6019
6020
6021
6022
6023
6024
6025
6026
6027
6028
6029
6030
6031
6032
6033
6034
6035
6036
6037
6038
6039
6040
6041
6042
6043
6044
6045
6046
6047
6048
6049
6050
6051
6052
6053
6054
6055
6056
6057
6058
6059
6060
6061
6062
6063
6064
6065
6066
6067
6068
6069
6070
6071
6072
6073
6074
6075
6076
6077
6078
6079
6080
6081
6082
6083
6084
6085
6086
6087
6088
6089
6090
6091
6092
6093
6094
6095
6096
6097
6098
6099
6100
6101
6102
6103
6104
6105
6106
6107
6108
6109
6110
6111
6112
6113
6114
6115
6116
6117
6118
6119
6120
6121
6122
6123
6124
6125
6126
6127
6128
6129
6130
6131
6132
6133
6134
6135
6136
6137
6138
6139
6140
6141
6142
6143
6144
6145
6146
6147
6148
6149
6150
6151
6152
6153
6154
6155
6156
6157
6158
6159
6160
6161
6162
6163
6164
6165
6166
6167
6168
6169
6170
6171
6172
6173
6174
6175
6176
6177
6178
6179
6180
6181
6182
6183
6184
6185
6186
6187
6188
6189
6190
6191
6192
6193
6194
6195
6196
6197
6198
6199
6200
6201
6202
6203
6204
6205
6206
6207
6208
6209
6210
6211
6212
6213
6214
6215
6216
6217
6218
6219
6220
6221
6222
6223
6224
6225
6226
6227
6228
6229
6230
6231
6232
6233
6234
6235
6236
6237
6238
6239
6240
6241
6242
6243
6244
6245
6246
6247
6248
6249
6250
6251
6252
6253
6254
6255
6256
6257
6258
6259
6260
6261
6262
6263
6264
6265
6266
6267
6268
6269
6270
6271
6272
6273
6274
6275
6276
6277
6278
6279
6280
6281
6282
6283
6284
6285
6286
6287
6288
6289
6290
6291
6292
6293
6294
6295
6296
6297
6298
6299
6300
6301
6302
6303
6304
6305
6306
6307
6308
6309
6310
6311
6312
6313
6314
6315
6316
6317
6318
6319
6320
6321
6322
6323
6324
6325
6326
6327
6328
6329
6330
6331
6332
6333
6334
6335
6336
6337
6338
6339
6340
6341
6342
6343
6344
6345
6346
6347
6348
6349
6350
6351
6352
6353
6354
6355
6356
6357
6358
6359
6360
6361
6362
6363
6364
6365
6366
6367
6368
6369
6370
6371
6372
6373
6374
6375
6376
6377
6378
6379
6380
6381
6382
6383
6384
6385
6386
6387
6388
6389
6390
6391
6392
6393
6394
6395
6396
6397
6398
6399
6400
6401
6402
6403
6404
6405
6406
6407
6408
6409
6410
6411
6412
6413
6414
6415
6416
6417
6418
6419
6420
6421
6422
6423
6424
6425
6426
6427
6428
6429
6430
6431
6432
6433
6434
6435
6436
6437
6438
6439
6440
6441
6442
6443
6444
6445
6446
6447
6448
6449
6450
6451
6452
6453
6454
6455
6456
6457
6458
6459
6460
6461
6462
6463
6464
6465
6466
6467
6468
6469
6470
6471
6472
6473
6474
6475
6476
6477
6478
6479
6480
6481
6482
6483
6484
6485
6486
6487
6488
6489
6490
6491
6492
6493
6494
6495
6496
6497
6498
6499
6500
6501
6502
6503
6504
6505
6506
6507
6508
6509
6510
6511
6512
6513
6514
6515
6516
6517
6518
6519
6520
6521
6522
6523
6524
6525
6526
6527
6528
6529
6530
6531
6532
6533
6534
6535
6536
6537
6538
6539
6540
6541
6542
6543
6544
6545
6546
6547
6548
6549
6550
6551
6552
6553
6554
6555
6556
6557
6558
6559
6560
6561
6562
6563
6564
6565
6566
6567
6568
6569
6570
6571
6572
6573
6574
6575
6576
6577
6578
6579
6580
6581
6582
6583
6584
6585
6586
6587
6588
//! Parallel-iterator primitives: `for_each_chunk`, plus a thin
//! reduction helper.
//!
//! For fine-grain workloads (`FpN<N>` at K = 5..7, ~100 ns - 1 us
//! per op) submitting per-item jobs through `sched::join` would
//! pay scheduler overhead larger than the actual work. The right
//! pattern is to recursively bisect the input until chunks are at
//! a saturating "leaf size", then run a serial loop inside each
//! leaf so SIMD has a continuous run to operate on.
//!
//! Recursive bisection through [`crate::sched::join`] gives us:
//! - log2(n / leaf_size) recursion depth
//! - One `sched::join` per internal split
//! - Cooperative work-stealing during waits (free from the Local
//!   tier dispatch)
//! - Linear speedup up to `worker_count()` cores
//!
//! # Sweeping input sizes
//!
//! The seeded leaf count is a power of two, so a throughput curve
//! measured across many sizes has small steps in it at the sizes where
//! that count doubles. Per element the larger size is the faster one
//! across a step, which reads as the kernel speeding up as `n` grows.
//! The steps belong to the fan-out, not to the caller's kernel; see
//! `adaptive_seed_depth` for where they fall and how large they are.

use crate::sched::arena::{global_local_arena, join_context};
use crate::sched::plan::JobPlan;

/// The heartbeat quantum: how long a serial fill runs before it hands
/// its tail to the pool, and the estimated total below which an entry
/// gate skips the pool altogether. Below the 67µs canonical heartbeat
/// value: the hybrid design hands the tail off to SLAW on the first
/// tick (no further heartbeat recursion), so faster ticks mean an
/// earlier handover and more of the work runs in parallel.
///
/// At 20µs:
/// - matmul 16x16 (~25µs total serial) just barely fires one
///   heartbeat - fans the tiny tail through SLAW.
/// - matmul 32x32 (~1ms serial) fans out at iter ~20 of 1024;
///   97% of work runs through SLAW's parallel bisect.
/// - matmul 64x64 (~13ms serial) fans out at iter ~20 of 4096;
///   99% parallel.
const HEARTBEAT_NS: u64 = 20_000;

/// The quantum in timestamp-counter ticks while the counter's rate is
/// unmeasured, which is [`HEARTBEAT_NS`] at 3 GHz. [`heartbeat_ticks`]
/// takes the host's own rate once anything in the process has measured
/// it.
#[cfg(target_arch = "x86_64")]
const HEARTBEAT_CYCLES: u64 = 60_000;

/// The heartbeat quantum in [`read_tsc`] ticks on this host.
///
/// On x86_64 the counter's rate is read as [`tsc_per_ns_16`] stored it
/// and never measured from here, because the first measurement in a
/// process spins for three milliseconds and a fill must not pay that.
/// Until a leaf flush or a span with a site has measured it, the
/// quantum is `HEARTBEAT_CYCLES`. Off x86_64 the counter is a
/// nanosecond clock and the quantum is [`HEARTBEAT_NS`] exactly.
#[inline(always)]
fn heartbeat_ticks() -> u64 {
    #[cfg(target_arch = "x86_64")]
    {
        match TSC_PER_NS_16.load(std::sync::atomic::Ordering::Relaxed) {
            0 => HEARTBEAT_CYCLES,
            rate_16 => HEARTBEAT_NS.saturating_mul(rate_16) >> 4,
        }
    }
    #[cfg(not(target_arch = "x86_64"))]
    {
        HEARTBEAT_NS
    }
}

/// How often to check the heartbeat timer. `iter_count & POLL_MASK
/// == 0` triggers an rdtsc read. 0x1F = every 32 iterations,
/// tight enough that the first tick lands soon after the
/// heartbeat quantum. rdtsc cost is
/// ~20-30 cycles on Zen3; checking every 32 iters of 1us work is
/// 0.05% overhead, well below the bench noise floor.
const POLL_MASK: usize = 0x1F;

/// Helper: RAII guard that emits a `DispatchExit` trace event when
/// dropped, regardless of which return path `for_each_chunk` takes.
fn scopeguard_dispatch_exit(n: usize) -> DispatchExitGuard {
    DispatchExitGuard { n: n as u32 }
}

struct DispatchExitGuard {
    n: u32,
}

impl Drop for DispatchExitGuard {
    fn drop(&mut self) {
        crate::sched::trace::emit(
            crate::sched::trace::TraceEvent::DispatchExit,
            self.n,
        );
    }
}

#[inline(always)]
fn read_tsc() -> u64 {
    #[cfg(target_arch = "x86_64")]
    // SAFETY: `_rdtsc` is part of the base x86_64 ISA (Pentium
    // and newer). The intrinsic has no CPU-feature preconditions
    // and no architectural preconditions on operand state.
    unsafe {
        std::arch::x86_64::_rdtsc()
    }
    #[cfg(not(target_arch = "x86_64"))]
    {
        // Nanoseconds since a point fixed at first use, so two reads
        // differ by the time between them. Timing an Instant created in
        // the same expression measures the call itself, which is a few
        // nanoseconds of noise around zero however long the leaf took.
        static ORIGIN: std::sync::OnceLock<std::time::Instant> = std::sync::OnceLock::new();
        ORIGIN
            .get_or_init(std::time::Instant::now)
            .elapsed()
            .as_nanos() as u64
    }
}

/// Timestamp-counter ticks per nanosecond on this host, in
/// sixteenths, or zero before the probe has run.
///
/// Sixteenths rather than a float so the conversion at a flush is an
/// integer multiply and shift, and so the stored value is exact for
/// the tick rates this runs on: a 3.6 GHz counter is 57.6 sixteenths.
static TSC_PER_NS_16: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);

// The rate needs no exclusion to install. Two callers arriving before
// it is measured both measure and both store, and they store the same
// thing: the counter's rate is a property of the machine and does not
// depend on who asked. So the race is benign and costs one extra
// measurement, once in a process, against an acquire on every caller
// forever.

/// Ticks per nanosecond in sixteenths, measured once per process.
///
/// [`read_tsc`] returns raw counter ticks on x86_64, and every
/// consumer of a leaf time works in nanoseconds, so the two flushes in
/// [`LocalLeafBuffer`] convert with this. The counter advances at a
/// fixed rate independent of the core's current frequency, so one
/// measurement describes the process's whole life.
///
/// Off x86_64 the counter is already a nanosecond clock, so this is
/// one sixteenth-per-nanosecond times sixteen and the conversion is
/// the identity.
pub(crate) fn tsc_per_ns_16() -> u64 {
    use std::sync::atomic::Ordering;
    let installed = TSC_PER_NS_16.load(Ordering::Relaxed);
    if installed != 0 {
        return installed;
    }
    let rate = measure_tsc_per_ns_16();
    TSC_PER_NS_16.store(rate, Ordering::Relaxed);
    rate
}

/// Time a fixed spin against both clocks and report the ratio.
///
/// The spin runs for a millisecond, which is four orders of magnitude
/// above the pair of reads bracketing it, and the fastest of three is
/// taken because interference only ever adds wall time and so only
/// ever lowers the apparent rate.
///
/// A reading outside 1 to 16 ticks per nanosecond is refused and the
/// nanosecond clock's own rate is installed instead: no counter this
/// runs on is below 1 GHz or above 16 GHz, and a rate that wrong would
/// move every classifier boundary.
fn measure_tsc_per_ns_16() -> u64 {
    #[cfg(not(target_arch = "x86_64"))]
    {
        // The fallback in `read_tsc` already returns nanoseconds.
        return 16;
    }
    #[cfg(target_arch = "x86_64")]
    {
        let mut best = 0u64;
        for _ in 0..3 {
            let wall_start = std::time::Instant::now();
            let tsc_start = read_tsc();
            while wall_start.elapsed() < std::time::Duration::from_millis(1) {
                std::hint::spin_loop();
            }
            let ticks = read_tsc().wrapping_sub(tsc_start);
            let nanos = wall_start.elapsed().as_nanos() as u64;
            if nanos == 0 {
                continue;
            }
            let per_ns_16 = ticks.saturating_mul(16) / nanos;
            if per_ns_16 > best {
                best = per_ns_16;
            }
        }
        if !(16..=256).contains(&best) {
            notice!(
                "flynnel: the timestamp counter measured {} ticks per nanosecond in sixteenths, \
                 which is outside 1 to 16 per nanosecond; leaf times are read as nanoseconds \
                 instead",
                best
            );
            return 16;
        }
        best
    }
}

/// Bracket a leaf-body invocation with timestamp-counter reads and
/// record the delta. Every bisect-leaf emit site uses this, and the
/// two flushes in [`LocalLeafBuffer`] convert the ticks to nanoseconds
/// with [`tsc_per_ns_16`], so [`crate::sched::split_observer`] and the
/// site classifier receive the unit their thresholds are stated in.
/// Samples batch in a thread-local buffer
/// ([`LocalLeafBuffer::FLUSH_THRESHOLD`] = 4) before flushing:
/// unbatched, the three global fetch_adds cost ~100ns per leaf on a
/// 16-worker host; batched, ~30ns (2 TSC reads + thread-local access).
/// `items` is how many items the leaf covered, which is what makes the
/// classifier's reading a property of the work rather than of the split
/// that produced it: a job cut into leaves of different sizes has leaf
/// times that differ by the sizes alone.
#[inline(always)]
fn record_leaf<F: FnOnce() -> R, R>(
    site: Option<crate::sched::call_site::SiteRef>,
    items: usize,
    body: F,
) -> R {
    crate::sched::trace::emit(crate::sched::trace::TraceEvent::LeafStart, 0);
    let t0 = read_tsc();
    let out = body();
    let dt = read_tsc().wrapping_sub(t0);
    crate::sched::trace::emit(crate::sched::trace::TraceEvent::LeafEnd, 0);
    LOCAL_LEAF_BUFFER.with(|cell| {
        let mut buf = cell.borrow_mut();
        buf.add(site, dt, items as u64, None);
    });
    out
}

/// [`record_leaf`] with the thread's own clock read either side as
/// well, so the leaf carries its on-core time beside its wall time.
///
/// The two differ by whatever the leaf spent off a core, and that
/// difference is the whole reason a busy host moves the scheduler's
/// choice rather than only its speed: preemption lands on some leaves
/// and not others, so it enters a wall-time spread as variance that
/// cannot be told from the work's own irregularity.
///
/// Only the sampled path calls this, and only while
/// [`crate::sched::levers::oncore_spread`] is on. Two clock pairs are
/// added to the two counter reads the leaf already pays; on Linux and
/// FreeBSD the thread-clock half of a pair is a syscall, since the vDSO
/// serves the monotonic clocks and refuses the per-thread CPU clock.
/// `examples/clock_cost.rs` reports what a pair costs on a host and
/// what the bracket amortizes to at the sample stride. A leaf recorded
/// off this path carries a wall time and no on-core reading, and the
/// site divides each figure by the items that figure covers.
///
/// A platform with no thread clock records the wall time alone, which
/// is what it has always recorded.
#[inline(always)]
fn record_leaf_on_core<F: FnOnce() -> R, R>(
    site: Option<crate::sched::call_site::SiteRef>,
    items: usize,
    body: F,
) -> R {
    let (thread_before, _) = crate::sched::occupancy::clock_pair();
    crate::sched::trace::emit(crate::sched::trace::TraceEvent::LeafStart, 0);
    let t0 = read_tsc();
    let out = body();
    let dt = read_tsc().wrapping_sub(t0);
    crate::sched::trace::emit(crate::sched::trace::TraceEvent::LeafEnd, 0);
    let (thread_after, _) = crate::sched::occupancy::clock_pair();
    // Both ends must have carried a count for the difference to describe
    // an interval. One end without one leaves it unknowable, and a zero
    // here would be the reading of a leaf that never reached a core.
    let on_core = match (thread_before.ticks(), thread_after.ticks()) {
        (Some(before), Some(after)) => Some(after.saturating_sub(before)),
        _ => None,
    };
    LOCAL_LEAF_BUFFER.with(|cell| {
        let mut buf = cell.borrow_mut();
        buf.add(site, dt, items as u64, on_core);
    });
    out
}

/// A leaf's per-item squared time: its squared time over its item
/// count. Summed across leaves it gives an unbiased estimate of the
/// per-item variance whatever sizes the leaves came out at, because a
/// leaf of n items averages n of them. A leaf of no items contributes
/// time to the leaf statistics and nothing to the per-item ones, which
/// is what it can honestly say.
///
/// Taken from the raw nanoseconds in 128 bits and scaled once, rather
/// than from the scaled square: scaling first costs the low 8 bits of
/// every leaf, which is nothing on a millisecond leaf and 40 percent on
/// a one-item leaf of a microsecond. That error arrives as spread,
/// which is the one thing this figure exists to measure.
#[inline(always)]
fn per_item_square(nanos: u64, items: u64) -> u64 {
    (nanos as u128)
        .saturating_mul(nanos as u128)
        .checked_div((items as u128) << 16)
        .unwrap_or(0) as u64
}

/// Record a pre-measured serial-span duration against `site`
/// without bracketing a body. The heartbeat and token-bucket fillers
/// record the serial spans between their promotions, and the
/// whole-input serial passes record the pass, in place of closure
/// invocations.
///
/// A span carries the items it covered and enters the site's
/// statistics as a leaf of that many items does. The site's per-item
/// mean is its summed time over its summed items, so a span's time
/// counted without its items would be spread over items other leaves
/// covered.
///
/// Spans feed the SITE's statistics only, never the process-global
/// counters: a span is a whole serial stretch, and mixing it into the
/// global classifier's per-item-ns boundaries would migrate the
/// process profile off unrelated workloads. With no site attached the
/// sample is dropped.
///
/// `nanos` is nanoseconds and goes to the site unconverted: this path
/// bypasses [`LocalLeafBuffer`], where counter ticks are turned into
/// nanoseconds. A span timed on the counter comes through
/// [`record_leaf_span_ticks`].
#[inline(always)]
pub(crate) fn record_leaf_span_ns(
    site: Option<crate::sched::call_site::SiteRef>,
    nanos: u64,
    items: u64,
) {
    if let Some(site) = site {
        let scaled = nanos >> 8;
        site.get().record_batch_site_only(
            nanos,
            scaled.saturating_mul(scaled),
            1,
            items,
            per_item_square(nanos, items),
        );
    }
}

/// [`record_leaf_span_ns`] for a span timed with [`read_tsc`]: the
/// delta is counter ticks, turned into nanoseconds by the conversion
/// [`LocalLeafBuffer`] applies to leaves.
///
/// The site is checked before the rate is read, because on x86_64 the
/// first read in a process measures the rate with three
/// one-millisecond spins, and a span with no site is dropped anyway.
#[inline(always)]
fn record_leaf_span_ticks(site: Option<crate::sched::call_site::SiteRef>, ticks: u64, items: u64) {
    if site.is_some() {
        let (nanos, _) = LocalLeafBuffer::as_nanos(ticks, 0);
        record_leaf_span_ns(site, nanos, items);
    }
}

/// Per-thread accumulator that batches leaf-time samples before
/// flushing them onward. Two independent accumulations run side by
/// side:
///
/// - The **global half** feeds the process-wide observer counters
///   via [`crate::sched::split_observer::record_leaf_batch`] on a
///   fixed [`Self::FLUSH_THRESHOLD`]-leaf cadence, regardless of
///   which sites the samples belong to. Keeping this cadence
///   independent of site interleaving preserves the batch
///   granularity the global auto-classifier and the split-observer
///   statistics were tuned against.
/// - The **site half** feeds the per-call-site statistics via
///   [`crate::sched::call_site::CallSiteState::record_batch_site_only`],
///   flushing at the same threshold OR whenever a sample arrives
///   for a different site, so site batches never mix sites.
///
/// Each half also carries the items its leaves covered and the summed
/// per-item squared time, which are what a dispersion of per-item cost
/// is computed from. Leaf time alone cannot distinguish a workload whose
/// items differ from one the scheduler cut into leaves that differ.
struct LocalLeafBuffer {
    global_sum_ns: u64,
    global_sumsq_scaled: u64,
    global_count: u64,
    global_items: u64,
    global_sumsq_per_item: u64,
    site_sum_ns: u64,
    site_sumsq_scaled: u64,
    site_count: u64,
    site_items: u64,
    site_sumsq_per_item: u64,
    /// On-core and elapsed counts for this worker, spanning the leaves
    /// buffered for the current site. Read once when the first sample
    /// of a batch arrives and once at flush, so the pair costs two
    /// clock reads per batch rather than two per leaf.
    ///
    /// The on-core count carries whether it was measured at all, so a
    /// platform without the clock contributes nothing to the site's
    /// occupancy rather than contributing an elapsed interval with no
    /// on-core time against it, which would read as total contention.
    site_thread_at_start: crate::sched::occupancy::ThreadTicks,
    site_wall_at_start: u64,
    /// The same leaves timed on the thread's own clock, in raw ticks,
    /// for the leaves that carried a reading. Their own item count,
    /// because a site reaches the sampled path for some leaves and not
    /// others and each figure divides by the items it covers.
    site_oncore_sum: u64,
    site_oncore_sumsq_per_item: u64,
    site_oncore_items: u64,
    /// Call site the buffered site-half samples belong to; null
    /// when the recent samples carried no site. Site changes are
    /// rare within one worker (only when it steals across
    /// concurrently-running dispatches), so the batching
    /// amortization survives keying.
    site: *const crate::sched::call_site::CallSiteState,
}

impl LocalLeafBuffer {
    // Per-worker flush threshold. 4 lets the closing-loop auto-
    // classifier observe a single workload-iteration's leaves on
    // small-N workloads (16-chunk grep, 16-chunk histogram, etc.).
    // With 1-chunk-per-worker workloads each worker produces 1
    // sample per iteration, so 4 iterations are enough to publish
    // a flush; the migration hysteresis then needs ~12 iterations
    // to converge on a stable classification -- well within
    // criterion's 3-second warm-up.
    const FLUSH_THRESHOLD: u64 = 4;

    const fn new() -> Self {
        Self {
            global_sum_ns: 0,
            global_sumsq_scaled: 0,
            global_count: 0,
            global_items: 0,
            global_sumsq_per_item: 0,
            site_sum_ns: 0,
            site_sumsq_scaled: 0,
            site_count: 0,
            site_items: 0,
            site_sumsq_per_item: 0,
            site_thread_at_start: crate::sched::occupancy::ThreadTicks::Absent(
                crate::sched::occupancy::NoReading::NoClock,
            ),
            site_wall_at_start: 0,
            site_oncore_sum: 0,
            site_oncore_sumsq_per_item: 0,
            site_oncore_items: 0,
            site: core::ptr::null(),
        }
    }

    #[inline(always)]
    fn add(
        &mut self,
        site: Option<crate::sched::call_site::SiteRef>,
        nanos: u64,
        items: u64,
        oncore: Option<u64>,
    ) {
        let scaled = nanos >> 8;
        let sq = scaled.saturating_mul(scaled);
        let per_item_sq = per_item_square(nanos, items);

        self.global_sum_ns = self.global_sum_ns.saturating_add(nanos);
        self.global_sumsq_scaled = self.global_sumsq_scaled.saturating_add(sq);
        self.global_items = self.global_items.saturating_add(items);
        self.global_sumsq_per_item = self.global_sumsq_per_item.saturating_add(per_item_sq);
        self.global_count += 1;
        if self.global_count >= Self::FLUSH_THRESHOLD {
            self.flush_global();
        }

        if let Some(s) = site {
            let site_ptr: *const crate::sched::call_site::CallSiteState = s.get();
            if !core::ptr::eq(site_ptr, self.site) {
                self.flush_site();
                self.site = site_ptr;
            }
            if self.site_count == 0 {
                let (thread, wall) = crate::sched::occupancy::clock_pair();
                self.site_thread_at_start = thread;
                self.site_wall_at_start = wall;
            }
            self.site_sum_ns = self.site_sum_ns.saturating_add(nanos);
            self.site_sumsq_scaled = self.site_sumsq_scaled.saturating_add(sq);
            self.site_items = self.site_items.saturating_add(items);
            self.site_sumsq_per_item = self.site_sumsq_per_item.saturating_add(per_item_sq);
            // The on-core reading, where this leaf carried one. Kept in
            // raw ticks and summed the same shape as the wall side, so
            // the two spreads are computed by the same arithmetic over
            // different clocks. A leaf without one contributes to the
            // wall figures and to nothing here, which is why the item
            // count is separate: the on-core spread is divided by the
            // items that actually carried a reading.
            if let Some(ticks) = oncore {
                let per_item_oncore = (ticks as u128)
                    .saturating_mul(ticks as u128)
                    .checked_div((items as u128) << 16)
                    .unwrap_or(0) as u64;
                self.site_oncore_sum = self.site_oncore_sum.saturating_add(ticks);
                self.site_oncore_sumsq_per_item =
                    self.site_oncore_sumsq_per_item.saturating_add(per_item_oncore);
                self.site_oncore_items = self.site_oncore_items.saturating_add(items);
            }
            self.site_count += 1;
            if self.site_count >= Self::FLUSH_THRESHOLD {
                self.flush_site();
            }
        }
    }

    /// A batch of counter ticks as nanoseconds: the summed ticks, and
    /// the summed squares of `ticks >> 8`, which carry the rate
    /// squared.
    ///
    /// Both conversions are integer and lose at most the divisor, and
    /// the squared sum is scaled by the rate before the division so a
    /// rate below sixteen sixteenths cannot floor it to zero.
    fn as_nanos(sum_ticks: u64, sumsq_scaled: u64) -> (u64, u64) {
        let per_ns_16 = tsc_per_ns_16();
        if per_ns_16 == 0 {
            return (sum_ticks, sumsq_scaled);
        }
        let sum_ns = sum_ticks.saturating_mul(16) / per_ns_16;
        let sumsq_ns = sumsq_scaled
            .saturating_mul(256)
            / per_ns_16.saturating_mul(per_ns_16).max(1);
        (sum_ns, sumsq_ns)
    }

    fn flush_global(&mut self) {
        if self.global_count == 0 {
            return;
        }
        let (sum_ns, sumsq_scaled) =
            Self::as_nanos(self.global_sum_ns, self.global_sumsq_scaled);
        let (_, sumsq_per_item) = Self::as_nanos(0, self.global_sumsq_per_item);
        crate::sched::split_observer::record_leaf_batch(
            sum_ns,
            sumsq_scaled,
            self.global_count,
            self.global_items,
            sumsq_per_item,
        );
        self.global_sum_ns = 0;
        self.global_sumsq_scaled = 0;
        self.global_count = 0;
        self.global_items = 0;
        self.global_sumsq_per_item = 0;
    }

    fn flush_site(&mut self) {
        if self.site_count == 0 || self.site.is_null() {
            self.site_count = 0;
            self.site_sum_ns = 0;
            self.site_sumsq_scaled = 0;
            return;
        }
        // SAFETY: `site` was captured from a `SiteRef`, which only
        // wraps `&'static CallSiteState`, so the pointee lives for
        // the program's lifetime.
        let site: &'static crate::sched::call_site::CallSiteState =
            unsafe { &*self.site };
        let (thread, wall) = crate::sched::occupancy::clock_pair();
        // Both ends of the batch must have carried a count for the pair
        // to describe an interval. Where either did not, this batch
        // contributes neither figure: adding the elapsed side alone
        // would grow the site's wall total against an unchanged on-core
        // total, and the site would read as contended on precisely the
        // platforms that cannot measure contention.
        // The same pair gives the batch its weight, so a batch recorded
        // while its worker was off its core counts for the share of the
        // interval it actually ran. A pair that measured nothing weighs
        // a whole batch: no reading is not evidence of contention.
        let weight = if let (Some(start), Some(end)) =
            (self.site_thread_at_start.ticks(), thread.ticks())
        {
            let on_core = end.saturating_sub(start);
            let elapsed = wall.saturating_sub(self.site_wall_at_start);
            site.add_pool_ticks(on_core, elapsed);
            // The pool ticks are recorded either way - they cost nothing
            // beyond this pair, which is read regardless. Only the
            // WEIGHTING is switched, so the occupancy a site reports
            // does not depend on which arm is running.
            crate::sched::levers::batch_weight()
                .then(|| crate::sched::call_site::batch_weight_per_mille(on_core, elapsed))
                .flatten()
        } else {
            None
        };
        let (sum_ns, sumsq_scaled) =
            Self::as_nanos(self.site_sum_ns, self.site_sumsq_scaled);
        let (_, sumsq_per_item) = Self::as_nanos(0, self.site_sumsq_per_item);
        site.record_batch_weighted(
            sum_ns,
            sumsq_scaled,
            self.site_count,
            self.site_items,
            sumsq_per_item,
            weight,
        );
        // Unconverted, unlike the wall sums above: these are the thread
        // clock's own ticks and only a ratio is taken of them.
        site.record_oncore_batch(
            self.site_oncore_sum,
            self.site_oncore_sumsq_per_item,
            self.site_oncore_items,
        );
        self.site_oncore_sum = 0;
        self.site_oncore_sumsq_per_item = 0;
        self.site_oncore_items = 0;
        self.site_sum_ns = 0;
        self.site_sumsq_scaled = 0;
        self.site_count = 0;
        self.site_items = 0;
        self.site_sumsq_per_item = 0;
    }
}

impl Drop for LocalLeafBuffer {
    fn drop(&mut self) {
        // Flush any residual samples on thread exit so both stat
        // consumers stay current.
        self.flush_global();
        self.flush_site();
    }
}

thread_local! {
    static LOCAL_LEAF_BUFFER: std::cell::RefCell<LocalLeafBuffer>
        = const { std::cell::RefCell::new(LocalLeafBuffer::new()) };
}

/// Publishes the calling thread's buffered leaf samples on drop.
/// Dispatch entries hold one so short dispatches (a probe plus a
/// serial tail, or a small fan-out) feed the observer and the site
/// before returning instead of waiting for a later dispatch to
/// cross the batch threshold; without it a dispatch recording
/// fewer than FLUSH_THRESHOLD leaves on the calling thread
/// publishes nothing.
/// Reports one dispatch's wall time against the routing arm it ran, so
/// the site's A/B has an outcome to weigh rather than a class to trust.
///
/// Held by the entry and dropped on every path out of it, including the
/// inline-collapse and probe returns, because a dispatch that took one
/// of those is still a dispatch that arm produced.
struct RecordRoutingArm {
    site: Option<crate::sched::call_site::SiteRef>,
    arm: Option<crate::sched::call_site::PolicyArm>,
    started: std::time::Instant,
}

impl RecordRoutingArm {
    fn start(
        site: Option<crate::sched::call_site::SiteRef>,
        arm: Option<crate::sched::call_site::PolicyArm>,
    ) -> Self {
        Self { site, arm, started: std::time::Instant::now() }
    }
}

impl Drop for RecordRoutingArm {
    fn drop(&mut self) {
        let Some(site) = self.site else {
            return;
        };
        let Some(arm) = self.arm else {
            return;
        };
        site.get()
            .record_routing_arm(arm, self.started.elapsed().as_nanos() as u64);
    }
}

struct FlushLeafStatsOnExit;

impl Drop for FlushLeafStatsOnExit {
    fn drop(&mut self) {
        LOCAL_LEAF_BUFFER.with(|cell| {
            let mut buf = cell.borrow_mut();
            buf.flush_global();
            buf.flush_site();
        });
    }
}

/// Lower bound on chunk size for the recursive bisect. Below this
/// the bisect emits a leaf and runs `op` serially on the chunk.
///
/// Tuning evidence (rayon-crossover bench): 256 is the stable
/// point, winning at n=1k-100k and losing ~6% at n=1M (acceptable
/// trade). 64 wins 7% at n=1M heavy but regresses 5.6x at n=10k
/// heavy because the adaptive cascade (replenish-to-max on
/// migrated) creates 256-512 leaves with steal-traffic-dominated
/// coordination. 32 leaves `log2(N/32)` recursion depth to
/// dominate wherever per-join overhead is high.
const MIN_LEAF_ITEMS: usize = 256;

/// Sample stride for [`record_leaf_sampled`] used by the default
/// lazy-steal bisect path. The TSC bracket + global stat update
/// fires for every Nth leaf; the other N-1 just run the body.
/// Reduces per-leaf instrumentation from ~30ns amortized to ~4ns,
/// at the cost of cv^2 sampling granularity that the observer can
/// still compute meaningfully.
const LEAF_SAMPLE_STRIDE: u32 = 8;

/// Target per-leaf wall-clock work used by [`adaptive_seed_depth`].
/// Smaller targets produce more leaves (better load balance, more
/// dispatch overhead). Tuned empirically across VM Zen3 16-thread
/// sweep + local Windows 4-thread:
///
/// 1ms target: VM Zen3 hinted scorecard 4W/5T/7L. Local Windows
/// showed 100us x 1024 splitting into 128 leaves of 8 items each
/// = 800us/leaf (mid-N regime), but VM data showed the cells
/// fluctuating in the noise floor between 1ms and 5ms targets.
/// 1ms keeps the same scorecard and matches the rayon default
/// chunk-size target more closely.
const TARGET_LEAF_WORK_NS: u64 = 1_000_000;

/// On unless `FLYNNEL_SEED_HYSTERESIS=0` turns it off, then settable.
static SEED_HYSTERESIS: std::sync::atomic::AtomicBool =
    std::sync::atomic::AtomicBool::new(true);
/// Reads the environment once, before the setting is answered.
static STABILIZERS_FROM_ENV: std::sync::Once = std::sync::Once::new();

fn stabilizers_from_env() {
    STABILIZERS_FROM_ENV.call_once(|| {
        let off = matches!(
            std::env::var_os("FLYNNEL_SEED_HYSTERESIS").as_deref(),
            Some(v) if v == "0"
        );
        SEED_HYSTERESIS.store(!off, std::sync::atomic::Ordering::Relaxed);
    });
}

/// Whether a change of seed depth needs corroboration before it takes
/// effect.
///
/// Public so a reader does not have to go through
/// [`set_seed_hysteresis`] to find out. Setting the switch to what it
/// already holds answers the same question, but it is two atomic
/// read-modify-writes on a line every dispatch reads, so a caller
/// polling it in a loop charges the scheduler rather than only itself.
pub fn seed_hysteresis() -> bool {
    stabilizers_from_env();
    SEED_HYSTERESIS.load(std::sync::atomic::Ordering::Relaxed)
}

/// Whether `FLYNNEL_SEED_DEPTH` reporting is on, read once.
///
/// Every dispatch reaches this, and `env::var_os` takes the process
/// environment lock and allocates a string on each read, so reading it
/// per call would charge the hot path for a switch that is off.
fn seed_depth_reporting() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var_os("FLYNNEL_SEED_DEPTH").is_some())
}

/// Turn seed-depth hysteresis on or off, returning what it was.
///
/// Settable rather than read once so a bench can put both arms in one
/// process and measure them back to back. Arms compared across
/// processes carry whatever else differed between those processes,
/// which for a decision driven by a measured estimate is the thing
/// being measured.
pub fn set_seed_hysteresis(on: bool) -> bool {
    stabilizers_from_env();
    SEED_HYSTERESIS.swap(on, std::sync::atomic::Ordering::Relaxed)
}

/// Compute the eager seed-split depth for bisect_lazy_steal_driven.
/// Returns the number of initial split levels before lazy mode
/// (steal-pressure-driven) kicks in.
///
/// `leaf_count = max(workers, items * per_item / TARGET_LEAF_WORK_NS)`,
/// capped at `items`; `seed_depth = ceil(log2(leaf_count))`, never
/// below `log2(workers)`.
///
/// The light and heavy regimes are the two ends of that one
/// expression, not separate branches. Light items leave
/// `items * per_item` under the target, so the worker floor supplies
/// the whole count and the depth is `log2(workers)`: one leaf per core,
/// fewest dispatches. Heavy items drive the quotient past `items`,
/// where the cap binds and the depth is `log2(items)`: one leaf per
/// item, so a slow worker holds up one item's work before the rest can
/// be stolen.
///
/// The division is integer, so a quotient below one contributes
/// nothing and the floor decides alone.
///
/// Example workloads (workers=16):
///   100us * 32 items: max(16, 3) = 16 -> 4 levels -> 16 leaves
///   10ms * 128 items: max(16, 1280) = 1280, capped at 128
///                     -> 7 levels -> 128 leaves
///
/// Without a per-item hint the count is `workers`, the fewest-leaves
/// choice; lazy-steal mode subdivides on demand after.
///
/// A caller who set an oversubscription factor with
/// [`crate::sched::plan::JobPlan::with_oversubscription_log2`] shifts
/// the resulting depth by that log2, capped at one leaf per item. A
/// factor that arrived from a profile or a learned class does not,
/// and neither does the process-global split multiplier: on this route
/// the only oversubscription applied is one the caller asked for.
///
/// The count rounds up to a power of two, so it is a step function of
/// `items`: two sizes a percent apart can seed 32 leaves and 64, and
/// the same size always gets the same count. Per element the larger
/// size is the faster one across a boundary - 2.6 to 6.8 percent at
/// 655_000 against 665_000, 24 workers, 50 ns an item - because 32
/// leaves on 24 workers leaves eight workers holding two and a tail
/// about twice one leaf, where 64 leaves is 2.67 each and averages the
/// imbalance out. The step sits under the run-to-run spread on a quiet
/// host.
#[inline]
fn adaptive_seed_depth(plan: &JobPlan, items: usize, workers: usize) -> usize {
    let workers = workers.max(1);
    let items = items.max(1);
    let workers_log2 = (workers as u64).next_power_of_two().trailing_zeros() as usize;
    let estimate_ns = plan.estimated_per_item_ns;
    let target_leaf_count = match estimate_ns {
        Some(ns) if ns > 0 => {
            let total_work_ns = (ns as u64).saturating_mul(items as u64);
            let from_work = (total_work_ns / TARGET_LEAF_WORK_NS).max(workers as u64);
            (from_work as usize).min(items)
        }
        _ => workers,
    };
    let depth = (target_leaf_count as u64).next_power_of_two().trailing_zeros() as usize;
    let depth = depth.max(workers_log2);
    // A change of depth needs corroboration, so one estimate landing
    // the far side of a boundary cannot halve or double the leaf count
    // by itself.
    let depth = match plan.site {
        Some(site) if seed_hysteresis() => site.get().stabilize_seed_depth(depth),
        _ => depth,
    };
    // A factor the caller set is them describing their own workload,
    // which a profile-derived or learned one is not, so it shifts the
    // depth the cost model settled on. It is applied after the worker
    // floor and the stabilizer because both of those work in depth, and
    // capped at one leaf per item, which is the same ceiling the target
    // carries.
    //
    // The process-global split multiplier is not consulted here at any
    // point. On this route the only oversubscription that applies is one
    // the caller asked for.
    let depth = if plan.oversubscription_log2_explicit {
        let items_log2 = (items as u64).next_power_of_two().trailing_zeros() as usize;
        depth
            .saturating_add(plan.effective_oversubscription_log2() as usize)
            .min(items_log2)
    } else {
        depth
    };
    // Counted after both stabilizers, so it is the depth the dispatch
    // actually seeds with rather than the one the cost model proposed.
    if let Some(site) = plan.site {
        site.get().record_seed_depth(depth);
    }
    // The seeded leaf count is a power of two, so a target between two
    // of them is served by the next one up: a target of 33 seeds 64.
    // `FLYNNEL_SEED_DEPTH` reports the target beside what it rounded
    // to, which is the difference a caller pays for in dispatch.
    //
    // One line per distinct shape rather than per dispatch. The key is
    // the pair the result depends on, so a bench sweeping sizes prints
    // once per size and a caller repeating one shape prints once.
    if seed_depth_reporting() {
        static LAST_SHAPE: core::sync::atomic::AtomicU64 =
            core::sync::atomic::AtomicU64::new(u64::MAX);
        let ns = plan.estimated_per_item_ns.unwrap_or(0);
        let key = ((items as u64) << 32) | ns as u64;
        if LAST_SHAPE.swap(key, core::sync::atomic::Ordering::Relaxed) != key {
            notice!(
                "seed depth: items {items} per_item_ns {:?} effective_ns {:?} \
                 workers {workers} target_leaves {target_leaf_count} \
                 seeded_leaves {} depth {depth}",
                plan.estimated_per_item_ns,
                estimate_ns,
                1usize << depth
            );
        }
    }
    depth
}

/// The recursion floor for a bisect: the smallest chunk it splits to,
/// never above `caller_floor`.
///
/// Which of three regimes applies turns on whether the plan's per-item
/// estimate is authoritative, not on how heavy the items are.
///
/// Without an authoritative estimate, a site that has measured its own
/// per-item cost supplies one: the floor is sized from it exactly as
/// from a caller's hint. That reading is a property of the work, so a
/// fine-grain site keeps its floor however its class routes, where
/// `use_smt` alone would drop the floor to 1 and hand a 5 ns item its
/// own dispatch.
///
/// With neither, `use_smt` stands in for a figure: it is the static
/// classifier's signal that items are heavy, and the floor drops to 1
/// so a batch narrower than `caller_floor` still splits instead of
/// running serially on the calling worker. Every other hintless plan
/// takes `caller_floor` unchanged, leaving fine-grain chunked work its
/// per-leaf amortization.
///
/// With an authoritative estimate a leaf carries about one dispatch's
/// worth of work: `pool_dispatch_cost_ns() / per_item_ns`, floored at
/// one item. The numerator is measured on the running host and cached,
/// so this floor is only as steady as that measurement was; it is a
/// figure this process holds, not a constant of the machine, and the
/// spread across processes on one host reaches a factor of two.
///
/// The result is capped at `caller_floor` in every regime, so a site
/// passing `min_leaf=1` gets one item per leaf whatever the plan says.
#[inline]
fn adaptive_min_leaf(plan: &JobPlan, caller_floor: usize) -> usize {
    if !plan.estimated_per_item_ns_explicit {
        // The site's own measurement first: it is per item, so it says
        // what use_smt was standing in for and says it about this
        // workload rather than about the class the last window
        // produced. A site running 5 ns items keeps its floor here even
        // while its class holds SMT on, which is the case the floor
        // used to get wrong: the class turned SMT on, the floor fell to
        // 1, and the split that produced fed the class that chose it.
        if let Some(site) = plan.site
            && let Some(per_item_ns) = site.get().per_item_ns()
            && per_item_ns > 0
        {
            let from_cost = (pool_dispatch_cost_ns() / per_item_ns).max(1) as usize;
            return from_cost.min(caller_floor);
        }
        // No authoritative ns hint and nothing measured, but the static
        // classifier may have signaled heavy per-item work via
        // use_smt=true (LatencyBound profile). For heavy items each
        // leaf is its own item -- min_leaf=1 lets the bisect split N=5
        // NMFD-shape batches into 5 leaves so the worker pool
        // actually parallelizes. Without the min_leaf=1 floor the
        // caller_floor (typically 256) caps min_leaf above
        // batch_size and the bisect never splits: serial execution
        // on the wrap-and-park primary, measured 2x slower than
        // rayon on cold_workloads NMFD-like cells with
        // `flynnel_def` (no caller ns hint).
        if plan.use_smt {
            return 1;
        }
        // For hint-less PortBound batches, the probe-and-decide path
        // in `for_each_chunk_min_leaf` already adaptively measures
        // per-item cost when n < workers*MIN_LEAF and feeds an
        // amended_plan with explicit ns into the bisect. Above
        // that threshold (n >= workers*MIN_LEAF, e.g. 16k items
        // on 8 workers) the static caller_floor=256 default
        // applies and the workload runs at 1.04-1.16x of rayon,
        // a bench-noise band that doesn't justify the per-leaf
        // bucketing overhead a finer adaptive sizing would
        // require. The probe path is the adaptation mechanism
        // for cells where it matters.
        return caller_floor;
    }
    // A leaf carries about one dispatch cost of work, measured on
    // this host, so the dispatch amortizes; never above the caller's
    // floor.
    match plan.estimated_per_item_ns {
        Some(ns) if ns > 0 => {
            let raw = (pool_dispatch_cost_ns() / ns as u64).max(1) as usize;
            raw.min(caller_floor)
        }
        _ => caller_floor,
    }
}

/// What one calibration pass measures on the running host's pool.
/// Every value is in nanoseconds and was measured, never assumed.
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct HostDispatchProfile {
    /// The pool's cost of one dispatch: the wall time of one join whose
    /// two halves are a single item each, median of `DIRECT_SAMPLES`.
    /// Observed, not a difference between two larger timings.
    pub dispatch_cost_ns: u64,
    /// Total work below which a body runs faster on the calling
    /// thread than dispatched: the serial time at the crossover of
    /// the doubling sweep, never below `dispatch_cost_ns`.
    pub collapse_threshold_ns: u64,
    /// Total work from which dispatching with the sleep-counter wake
    /// path finishes no later than dispatching with polling only:
    /// the crossover of the same sweep run under both scopes, or the
    /// serial time at the largest count swept when the wake path
    /// never catches up.
    pub jec_wake_threshold_ns: u64,
}

/// The measured profile, installed by the calibration; zero until it
/// has run.
static HOST_DISPATCH_COST_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static HOST_COLLAPSE_THRESHOLD_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
static HOST_JEC_WAKE_THRESHOLD_NS: std::sync::atomic::AtomicU64 = std::sync::atomic::AtomicU64::new(0);
// The three are published rather than guarded. The collapse threshold
// is written last with Release and read first with Acquire, so a
// reader that sees it set is guaranteed the other two alongside it.
//
// That is a stronger guarantee than the exclusion it replaces: the
// reader below never took the lock, so two relaxed loads could see a
// freshly published threshold beside a dispatch cost still at zero,
// and excluding a second calibration never fixed that. Two racing
// calibrations each publish a whole triple and the later one wins; a
// reader sees one of them entire or neither.

/// The published dispatch cost in nanoseconds, or zero while nothing is
/// published. Never measures and never waits, so a pool worker can read
/// it from its idle loop; [`host_dispatch_profile`] is the query that
/// calibrates when nobody has.
pub(crate) fn installed_dispatch_cost_ns() -> u64 {
    use std::sync::atomic::Ordering;
    if HOST_COLLAPSE_THRESHOLD_NS.load(Ordering::Acquire) == 0 {
        return 0;
    }
    HOST_DISPATCH_COST_NS.load(Ordering::Relaxed)
}

/// This host's dispatch profile, measured once per process by the
/// first query (10-40 ms on the bench hosts) or earlier by
/// [`calibrate_host_dispatch`] at start-up. Measured on this host
/// unless `FLYNNEL_HOST_PROFILE_NS` pins it, which a comparison
/// against a non-adaptive scheduler needs so both sides face the same
/// dispatch cost.
///
/// A first query from one of the pool's own workers is measured on a
/// thread outside the pool while that worker runs pool jobs, so the
/// figure is an outside caller's dispatch whichever thread asks.
pub fn host_dispatch_profile() -> HostDispatchProfile {
    use std::sync::atomic::Ordering;
    // Acquire, and first: it pairs with the Release in
    // calibrate_host_dispatch, so a non-zero read here is what makes
    // the other two safe to read relaxed.
    let collapse = HOST_COLLAPSE_THRESHOLD_NS.load(Ordering::Acquire);
    if collapse != 0 {
        return HostDispatchProfile {
            dispatch_cost_ns: HOST_DISPATCH_COST_NS.load(Ordering::Relaxed),
            collapse_threshold_ns: collapse,
            jec_wake_threshold_ns: HOST_JEC_WAKE_THRESHOLD_NS.load(Ordering::Relaxed),
        };
    }
    // Nobody has published yet, and only one caller may go and
    // measure. Calibrating dispatches into the pool and waits, and a
    // worker helping inside that wait can pick up an outside job that
    // arrives here, reads the same zero, and starts a second
    // calibration on its own stack, whose waits pick up more. Nothing
    // stores until the innermost one finishes, so the nesting is
    // bounded only by how many dispatchers arrive during the window,
    // and a worker's stack is not.
    if CALIBRATING
        .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
        .is_ok()
    {
        let _in_flight = CalibrationInFlight;
        return calibrate_host_dispatch();
    }
    // Somebody else is measuring. Who this caller is decides what it
    // gets, and the two cases must not be merged: the mutex this
    // replaced made every second arrival wait, and handing every
    // second arrival zeros instead broke the tests that expect a small
    // job to collapse inline, because they arrived during another
    // test's calibration and routed on a threshold of zero.
    //
    // A pool worker cannot wait. The calibration is waiting on the
    // pool, possibly on this very worker, so a worker that blocked
    // here would deadlock it. It takes zeros, which route
    // conservatively for the one job it is running: nothing collapses
    // inline, every leaf sizes from one item, the dispatch takes the
    // wake path. That is the whole cost of breaking the recursion.
    if !crate::sched::arena_local::current_worker_ctx().is_null() {
        return HostDispatchProfile {
            dispatch_cost_ns: 0,
            collapse_threshold_ns: 0,
            jec_wake_threshold_ns: 0,
        };
    }
    // Anyone else waits for the real figures, as it always did. The
    // wait is the tens of milliseconds one calibration takes, once per
    // process, and it spins first because the publish is usually a
    // few microseconds away, then yields so a preempted calibrator gets
    // its core back.
    //
    // A calibrator that panics drops the flag without publishing. A
    // waiter that only watched the threshold would then spin for the
    // life of the process, so when the flag is down and nothing is
    // published the waiter takes the flag and measures itself.
    let mut spins = 0u32;
    loop {
        let collapse = HOST_COLLAPSE_THRESHOLD_NS.load(Ordering::Acquire);
        if collapse != 0 {
            return HostDispatchProfile {
                dispatch_cost_ns: HOST_DISPATCH_COST_NS.load(Ordering::Relaxed),
                collapse_threshold_ns: collapse,
                jec_wake_threshold_ns: HOST_JEC_WAKE_THRESHOLD_NS.load(Ordering::Relaxed),
            };
        }
        if CALIBRATING
            .compare_exchange(false, true, Ordering::AcqRel, Ordering::Acquire)
            .is_ok()
        {
            let _in_flight = CalibrationInFlight;
            return calibrate_host_dispatch();
        }
        if spins < 1_000 {
            spins += 1;
            std::hint::spin_loop();
        } else {
            std::thread::yield_now();
        }
    }
}

/// Set for as long as a calibration started from
/// [`host_dispatch_profile`] is measuring, so a second arrival in
/// that window does not start another.
static CALIBRATING: std::sync::atomic::AtomicBool = std::sync::atomic::AtomicBool::new(false);

/// Clears [`CALIBRATING`] when the measuring caller leaves, by any
/// path. A calibration that panics must not leave the flag set, or
/// every later caller would read zeros for the life of the process.
struct CalibrationInFlight;

impl Drop for CalibrationInFlight {
    fn drop(&mut self) {
        CALIBRATING.store(false, std::sync::atomic::Ordering::Release);
    }
}

/// Total work, in nanoseconds from the caller's explicit per-item
/// estimate, below which a data-parallel entry runs its body on the
/// calling thread instead of dispatching:
/// [`HostDispatchProfile::collapse_threshold_ns`]. Classifier
/// defaults never trigger the collapse; only
/// [`JobPlan::with_estimated_per_item_ns`] does.
///
/// Measured once per process on first use and cached for that
/// process's life, so it is a per-process value rather than a constant
/// of the machine: a figure read in one process does not describe
/// another. The `FLYNNEL_HOST_PROFILE_NS` environment variable pins
/// all three figures and skips the measurement, which is what an
/// experiment comparing arms across processes needs.
///
/// It prices this pool's dispatch and nothing else. A caller whose
/// parallel route has setup of its own - a prescan, a chunking pass, a
/// merge across chunk boundaries - is not described by this number,
/// and a serial-versus-parallel threshold derived from it alone will
/// sit far below where that caller's own route starts winning. Declare
/// that cost with [`JobPlan::with_task_overhead_ns`], which is what
/// [`JobPlan::optimal_chunk_count`] divides the work against.
pub fn inline_collapse_threshold_ns() -> u64 {
    host_dispatch_profile().collapse_threshold_ns
}

/// The pool's measured cost of one dispatch in this process:
/// [`HostDispatchProfile::dispatch_cost_ns`]. A leaf is sized to
/// carry about this much work, and the probe's dispatch decision
/// compares against [`inline_collapse_threshold_ns`].
///
/// Per process rather than per host: four processes on one Ryzen 9
/// 7900X read 900 to 1100 ns, twelve on a Ryzen 7 2700 read 2100 to
/// 5500 ns. A figure read in one process does not describe another,
/// and the first read in a loaded process fixes that load's cost for
/// the rest of that process.
pub fn pool_dispatch_cost_ns() -> u64 {
    host_dispatch_profile().dispatch_cost_ns
}

/// The measured collapse threshold when the calibration has run,
/// `None` before. For pool workers, which must not start the
/// calibration: it dispatches into the pool and waits for them.
pub fn measured_collapse_threshold_ns() -> Option<u64> {
    match HOST_COLLAPSE_THRESHOLD_NS.load(std::sync::atomic::Ordering::Relaxed) {
        0 => None,
        ns => Some(ns),
    }
}

/// Total work from which a dispatch takes the sleep-counter wake
/// path instead of polling: [`HostDispatchProfile::jec_wake_threshold_ns`].
pub fn jec_wake_threshold_ns() -> u64 {
    host_dispatch_profile().jec_wake_threshold_ns
}

/// Measure this host's dispatch profile now and install it.
///
/// The draw runs on the calling thread, or, when that thread is one of
/// the pool's workers, on a thread outside the pool while the worker
/// runs pool jobs. A worker's own joins pop back the half they pushed
/// before a parked peer can take it, so drawn there the figure would be
/// an inline pass rather than a dispatch.
///
/// A compute-bound body, eight dependent multiply-rotate steps per
/// item, is timed serial against dispatched through the pool after
/// sixteen discarded warm-up dispatches. Every timed point and every
/// crossover sweep is the fastest of nine, not a median: interference
/// only adds time to a measurement, so the fastest run is the closest
/// reading of the cost itself.
///
/// Three values come out. The dispatch cost is one join over two
/// leaves minus the same body serial. The collapse threshold is the
/// serial time at which the dispatched body first finishes no later
/// than serial, found over doubling counts from four leaves and
/// interpolated between the last two, floored at the dispatch cost.
/// The wake threshold is the same sweep run under the polling scope
/// against the wake scope.
///
/// A sweep whose second side never catches up reports the serial time
/// at the largest count swept, so a calibration taken on a loaded
/// host collapses more work inline and wakes later, never the
/// reverse.
///
/// Callers arriving while one calibration runs wait for its values; a
/// call after it re-measures and replaces them.
///
/// # Installing is the side effect, not the return
///
/// All three figures are stored into the process globals before this
/// returns, so calling it arms them for every dispatch in the process
/// whether or not the caller keeps what comes back. A call site that
/// binds the result and only prints it has still changed how the whole
/// program routes.
///
/// [`pinned_host_profile`] is the way to hold them fixed: with
/// `FLYNNEL_HOST_PROFILE_NS` set, this installs those values and
/// measures nothing, which is what a caller comparing arms wants - a
/// measured profile is a per-run variable, and one taken on a loaded
/// host differs from one taken on an idle host.
pub fn calibrate_host_dispatch() -> HostDispatchProfile {
    use std::sync::atomic::Ordering;
    let p = match pinned_host_profile() {
        Some(pinned) => pinned,
        None => stored_or_measured(),
    };
    HOST_DISPATCH_COST_NS.store(p.dispatch_cost_ns, Ordering::Relaxed);
    HOST_JEC_WAKE_THRESHOLD_NS.store(p.jec_wake_threshold_ns, Ordering::Relaxed);
    // Last, and Release: this is the one the reader tests, so it is
    // what makes the other two visible.
    HOST_COLLAPSE_THRESHOLD_NS.store(p.collapse_threshold_ns, Ordering::Release);
    p
}

/// The host's profile from the persisted table when one is there and
/// was measured on a quiet host, otherwise a fresh measurement, which
/// is published for the next process to read.
///
/// Every path that cannot reach the table still returns a profile,
/// because measuring is what the caller would have done with no table
/// at all. What it does not do is stay quiet about why: a table that is
/// unreadable and a host that has never been measured produce the same
/// numbers, and only the diagnostic tells them apart.
///
/// A stored record whose samples spread like a loaded host is passed
/// over and measured again. Persisting a measurement taken under load
/// would hand every later process on the machine that load's numbers.
#[cfg(feature = "persisted-calibration")]
fn stored_or_measured() -> HostDispatchProfile {
    use crate::sched::calibration_store::{
        CalibrationStore, CpuCalibration, HostStamp, StoreError, calibration_dir,
    };
    let Some(dir) = calibration_dir() else {
        // No cache location on this platform; nothing to report.
        return draw_host_dispatch().profile;
    };
    let stamp = HostStamp::detect();
    let store = match CalibrationStore::open_or_create(&dir, &stamp) {
        Ok(s) => s,
        Err(e) => {
            notice!(
                "flynnel: the calibration table under {} is unusable ({e:?}); \
                 this process measures its own dispatch profile",
                dir.display()
            );
            return draw_host_dispatch().profile;
        }
    };
    if let Some((cpu, _accel)) = store.read()
        && stored_record_serves(&cpu, configured_max_age_s(), now_unix_s())
    {
        // A read and a measurement produce the same three numbers and
        // nothing else distinguished them, so a stale record looked
        // exactly like a host that calibrates consistently. Said under
        // the same variable that reports a draw, because the question
        // a reader has is which of the two happened.
        if std::env::var_os("FLYNNEL_OCCUPANCY").is_some() {
            let age_s = now_unix_s().saturating_sub(cpu.measured_unix_s);
            let lasts = match configured_max_age_s() {
                Some(bound) if bound > 0 => format!("serves until it is {bound} s old"),
                _ => "serves until the layout version changes".to_string(),
            };
            notice!(
                "flynnel: host profile {},{},{} READ from the stored record, not \
                 measured; it was drawn {age_s} s ago at {} per mille occupancy with \
                 spread {} per mille and {lasts}",
                cpu.dispatch_cost_ns,
                cpu.collapse_threshold_ns,
                cpu.jec_wake_threshold_ns,
                cpu.occupancy().map_or_else(|| "an unrecorded".to_string(), |o| o.to_string()),
                cpu.spread_per_mille,
            );
        }
        return HostDispatchProfile {
            dispatch_cost_ns: cpu.dispatch_cost_ns,
            collapse_threshold_ns: cpu.collapse_threshold_ns,
            jec_wake_threshold_ns: cpu.jec_wake_threshold_ns,
        };
    }
    // Occupancy is reported beside the draw and gates nothing: what
    // figure marks a contended measurement is not known, and a threshold
    // picked ahead of the distribution describes whoever picked it. It is
    // sampled once, on the thread that measured, and both reported and
    // stored, so the figure a reader sees in the note is the one the
    // record carries.
    let Draw { profile, spread, occupancy: drawn_at, publishable } = draw_host_dispatch();
    if std::env::var_os("FLYNNEL_OCCUPANCY").is_some() {
        // An unmeasured interval is reported as unmeasured. Printing a
        // number for it would tell a reader the calibration ran on a
        // quiet host when what happened is that nobody looked.
        // The three figures are printed in the form
        // FLYNNEL_HOST_PROFILE_NS takes, so a draw taken under known
        // conditions can be pinned into a later process and compared
        // against another draw. Without this the occupancy beside a
        // draw is readable and the draw itself is not, so two draws
        // cannot be told apart by anything except the line that says
        // what they ran under.
        notice!(
            "flynnel: host profile {},{},{}",
            profile.dispatch_cost_ns,
            profile.collapse_threshold_ns,
            profile.jec_wake_threshold_ns,
        );
        match drawn_at {
            Some(share) => notice!(
                "flynnel: host calibration ran at {share} per mille occupancy, \
                 spread {spread} per mille",
            ),
            None => notice!(
                "flynnel: host calibration measured no occupancy on this platform, \
                 spread {spread} per mille",
            ),
        }
    }
    if !publishable {
        notice!(
            "flynnel: this draw was taken on a pool worker, whose joins time its own inline \
             pass rather than a dispatch, so it routes this process and is not stored"
        );
        return profile;
    }
    match store.try_acquire_writer() {
        Ok(writer) => {
            writer.beat();
            let outcome = writer.publish_if_better(
                &CpuCalibration::new(
                    profile.dispatch_cost_ns,
                    profile.collapse_threshold_ns,
                    profile.jec_wake_threshold_ns,
                    spread,
                    SAMPLES as u32,
                    drawn_at,
                ),
                &[],
            );
            // This process still runs on what it measured; only the
            // table keeps the older record. Said rather than passed
            // over, because a measurement that was taken and then
            // discarded is not the same event as one that was stored,
            // and the two are indistinguishable from the outside.
            use crate::sched::calibration_store::PublishOutcome;
            match outcome {
                PublishOutcome::KeptIncumbent { incumbent, offered, confirmations } => {
                    notice!(
                        "flynnel: this host's stored calibration dispatches in {incumbent} \
                         ns and this one in {offered}; the cheaper record stands and this \
                         process uses what it measured. It carries {confirmations} \
                         agreeing draw(s){}",
                        if confirmations == 0 {
                            ", so it stays provisional and the next start measures again"
                        } else {
                            " and serves the next start"
                        }
                    );
                }
                PublishOutcome::Published { confirmations: 0 } => notice!(
                    "flynnel: this draw is stored and provisional; nothing has agreed \
                     with it yet, so the next start measures again and the two are \
                     compared"
                ),
                PublishOutcome::Published { confirmations } => notice!(
                    "flynnel: this draw is stored with {confirmations} agreeing draw(s) \
                     and serves the next start"
                ),
            }
        }
        // Another process on this host is measuring the same table. Its
        // record serves the next start; this process keeps the profile
        // it measured. Expected under a parallel test run, so it is not
        // worth a note.
        Err(StoreError::WriterActive) => {}
        Err(e) => notice!(
            "flynnel: measured this host but could not publish it ({e:?}); \
             the next process measures again"
        ),
    }
    profile
}

/// Whether a stored record is returned to the caller, or the host is
/// measured again.
///
/// The decision, separate from the environment and the clock that feed
/// it, so it can be exercised with any record at any age. What matters
/// about it cannot be tested otherwise: a record this returns true for
/// is returned to the caller before the branch that would offer one to
/// `WriterGuard::publish_if_better`, so that guard's comparison never
/// sees a record in this state.
///
/// `max_age_s` of `None` is no age bound: a record that clears its
/// trust check stands until the layout version changes. The shipped
/// bound is [`DEFAULT_CALIBRATION_MAX_AGE_S`], which
/// `configured_max_age_s` answers unless `FLYNNEL_CALIBRATION_MAX_AGE_S`
/// says otherwise. A bound of zero is also read as no bound, since
/// re-measuring at every start is what the store exists to avoid and is
/// not something a caller asks for by typing a number.
///
/// A record stamped in the future is a clock that moved rather than a
/// fresh draw, so saturating leaves it aged zero and it stands.
#[cfg(feature = "persisted-calibration")]
fn stored_record_serves(
    cpu: &crate::sched::calibration_store::CpuCalibration,
    max_age_s: Option<u64>,
    now_s: u64,
) -> bool {
    if !crate::sched::levers::serve_policy().admits(cpu) {
        return false;
    }
    match max_age_s {
        None | Some(0) => true,
        Some(max) => now_s.saturating_sub(cpu.measured_unix_s) <= max,
    }
}

/// How long a stored calibration serves before a fresh draw is taken.
///
/// One day. A record that clears the trust check is otherwise permanent
/// for its stamp, and that check passes more readily for a draw taken
/// under load, whose samples agree because they were all slowed
/// together. This bound is what limits how long such a draw can route.
///
/// The cost is one calibration sweep per process start whose record has
/// aged out, so a day is short enough to cap the exposure and long
/// enough that repeated starts read the stored record.
#[cfg(feature = "persisted-calibration")]
pub const DEFAULT_CALIBRATION_MAX_AGE_S: u64 = 86_400;

/// The age bound: `FLYNNEL_CALIBRATION_MAX_AGE_S` when it parses,
/// otherwise [`DEFAULT_CALIBRATION_MAX_AGE_S`]. Zero means no bound and
/// a record then stands until [`crate::sched::calibration_store::LAYOUT_VERSION`]
/// rises.
///
/// Every way of failing to read it says which one happened and falls
/// back to the default rather than to no bound, so a mistyped value
/// costs at most one extra sweep a day instead of restoring the
/// permanence the bound exists to end.
#[cfg(feature = "persisted-calibration")]
fn configured_max_age_s() -> Option<u64> {
    let Some(raw) = std::env::var_os("FLYNNEL_CALIBRATION_MAX_AGE_S") else {
        return Some(DEFAULT_CALIBRATION_MAX_AGE_S);
    };
    let Some(text) = raw.to_str() else {
        notice!(
            "flynnel: FLYNNEL_CALIBRATION_MAX_AGE_S is not valid UTF-8; using the \
             default of {DEFAULT_CALIBRATION_MAX_AGE_S}s"
        );
        return Some(DEFAULT_CALIBRATION_MAX_AGE_S);
    };
    match text.trim().parse::<u64>() {
        Ok(v) => Some(v),
        Err(e) => {
            notice!(
                "flynnel: FLYNNEL_CALIBRATION_MAX_AGE_S wants a whole number of \
                 seconds and got {text:?} ({e}); using the default of \
                 {DEFAULT_CALIBRATION_MAX_AGE_S}s"
            );
            Some(DEFAULT_CALIBRATION_MAX_AGE_S)
        }
    }
}

/// Seconds since the epoch, or zero where the clock reads before it.
///
/// Zero ages every record to nothing, so a host whose clock is that
/// wrong keeps serving what it has rather than re-measuring at every
/// start off a figure that cannot be trusted.
#[cfg(feature = "persisted-calibration")]
fn now_unix_s() -> u64 {
    match std::time::SystemTime::now().duration_since(std::time::UNIX_EPOCH) {
        Ok(d) => d.as_secs(),
        Err(e) => {
            notice!(
                "flynnel: this host's clock reads before the epoch ({e}), so a stored \
                 calibration's age cannot be taken and it stands"
            );
            0
        }
    }
}

/// The measured profile, with nothing persisted.
#[cfg(not(feature = "persisted-calibration"))]
fn stored_or_measured() -> HostDispatchProfile {
    draw_host_dispatch().profile
}

/// One draw of this host's dispatch profile.
#[cfg_attr(
    not(feature = "persisted-calibration"),
    expect(dead_code, reason = "only the persisted path stores a draw, so only it reads these")
)]
struct Draw {
    profile: HostDispatchProfile,
    /// The spread of the samples behind it, in parts per thousand.
    spread: u32,
    /// The share of a core the measuring thread held while it drew, in
    /// parts per thousand, or `None` where the platform has no thread
    /// clock.
    occupancy: Option<u32>,
    /// Whether it timed an outside caller's dispatch, and so may be
    /// stored for other processes to read.
    publishable: bool,
}

/// Draw the profile as an outside caller's dispatch sees it, whichever
/// thread asks.
///
/// On a pool worker the draw's joins would be the worker's own. It
/// pushes the right half to its own deque, runs the one-item left half
/// and pops the right half back before a parked peer can take it, so the
/// median times an inline pass: 100 ns on a 24-thread Windows host and
/// 170 to 181 ns on a 16-vCPU Linux guest, where an outside caller's
/// draw reads 1300 to 4800 and 1300 to 2200. So a worker hands the draw
/// to a thread outside the pool and runs pool jobs until it ends. That
/// is also what keeps a pool of one worker from waiting on itself: the
/// draw's dispatches land in queues this worker's search reaches, its
/// own mailbox and the held outside-caller slots among them.
///
/// A worker that cannot start the thread draws on itself and says so,
/// and that draw routes its own process but is not stored.
fn draw_host_dispatch() -> Draw {
    use std::sync::atomic::{AtomicBool, Ordering};
    let ctx_ptr = crate::sched::arena_local::current_worker_ctx();
    if ctx_ptr.is_null() {
        return draw_on_this_thread(true);
    }
    // SAFETY: worker_loop set this pointer on this thread, and it stays
    // valid until worker_loop returns, which it cannot do while this
    // frame is on the worker's stack.
    let ctx = unsafe { &*ctx_ptr };
    let ended = AtomicBool::new(false);
    std::thread::scope(|scope| {
        let spawned = std::thread::Builder::new()
            .name("flynnel-calibration".to_string())
            .spawn_scoped(scope, || {
                let _ended = SetOnDrop(&ended);
                draw_on_this_thread(true)
            });
        let helper = match spawned {
            Ok(helper) => helper,
            Err(e) => {
                notice!(
                    "flynnel: could not start a thread to calibrate outside the pool ({e}); \
                     this worker draws on itself, so its dispatch cost times its own inline \
                     pass, and the draw is not stored"
                );
                return draw_on_this_thread(false);
            }
        };
        // The same wait an outside caller of the profile takes while
        // another thread calibrates, spinning first and yielding after,
        // with the pool's work run in between.
        let mut idle_rounds = 0u32;
        while !ended.load(Ordering::Acquire) {
            match ctx.find_work() {
                Some(job) => {
                    // SAFETY: find_work hands each JobRef out once, and
                    // this executes it exactly once.
                    unsafe { job.execute() };
                    idle_rounds = 0;
                }
                None if idle_rounds < 1_000 => {
                    idle_rounds += 1;
                    std::hint::spin_loop();
                }
                None => std::thread::yield_now(),
            }
        }
        match helper.join() {
            Ok(draw) => draw,
            Err(panic) => std::panic::resume_unwind(panic),
        }
    })
}

/// Measure on the calling thread, with that thread's occupancy.
fn draw_on_this_thread(publishable: bool) -> Draw {
    let window = crate::sched::occupancy::OccupancyWindow::start();
    let (profile, spread) = measure_host_dispatch();
    Draw { profile, spread, occupancy: window.sample().per_mille(), publishable }
}

/// Sets its flag when dropped, which includes a panic unwinding past it.
struct SetOnDrop<'a>(&'a std::sync::atomic::AtomicBool);

impl Drop for SetOnDrop<'_> {
    fn drop(&mut self) {
        self.0.store(true, std::sync::atomic::Ordering::Release);
    }
}

/// Place the crossover between the last two timed counts.
///
/// `prev` is the previous count's `first` time and the gap that count
/// showed; `a` and `diff` are the same for the count where the gap
/// reached zero. The crossover falls between them in proportion to how
/// much of the gap each closed.
///
/// Both orderings are required before interpolating, and neither is
/// incidental. `d_prev > diff` says the gap actually narrowed. `a >
/// a_prev` says the larger count cost more, which is what puts the
/// crossover between the two - a contended host can time the larger
/// count faster, and `a - a_prev` is over u64: it panics in debug and
/// wraps in release, where the wrapped span produces a threshold that
/// then governs every dispatch on that host. A pair that fails either
/// test carries no crossover to place, so the measurement is reported
/// as it stands.
fn interpolate_crossover(prev: Option<(u64, i64)>, a: u64, diff: i64) -> u64 {
    match prev {
        Some((a_prev, d_prev)) if d_prev > diff && a > a_prev => {
            let span = (a - a_prev) as f64;
            let frac = d_prev as f64 / (d_prev - diff) as f64;
            a_prev + (span * frac) as u64
        }
        _ => a,
    }
}

/// [`calibrate_host_dispatch`], returning the collapse threshold.
pub fn calibrate_inline_collapse_threshold() -> u64 {
    calibrate_host_dispatch().collapse_threshold_ns
}

/// The profile pinned by `FLYNNEL_HOST_PROFILE_NS`, or `None` when
/// the variable is unset and the host is to be measured.
///
/// Reading a pinned profile skips the calibration entirely, so a
/// caller comparing this scheduler against one that never measures
/// its dispatch cost can hold that cost equal on both sides. Without
/// it the two differ in whether they adapt as well as in how they
/// schedule, and the ratio mixes the two.
fn pinned_host_profile() -> Option<HostDispatchProfile> {
    let raw = std::env::var_os("FLYNNEL_HOST_PROFILE_NS")?;
    match raw.to_str() {
        Some(text) => parse_pinned_profile(text),
        None => {
            notice!(
                "flynnel: FLYNNEL_HOST_PROFILE_NS is not valid UTF-8; measuring this host instead"
            );
            None
        }
    }
}

/// `dispatch,collapse,wake` in nanoseconds, all three required.
///
/// All three because pinning some and measuring the rest yields a
/// profile that is neither pinned nor measured, which defeats the
/// comparison the pin exists to make. Zero is refused because the
/// installed collapse threshold doubles as the "not yet calibrated"
/// marker. A value that does not parse is reported and the caller
/// measures the host, so a mistyped pin is loud rather than silently
/// honored as something else.
fn parse_pinned_profile(text: &str) -> Option<HostDispatchProfile> {
    let mut fields = text.split(',');
    let mut take = |what: &str| -> Option<u64> {
        let Some(field) = fields.next() else {
            notice!(
                "flynnel: FLYNNEL_HOST_PROFILE_NS wants dispatch,collapse,wake in ns; the {what} field is missing, so measuring this host instead"
            );
            return None;
        };
        match field.trim().parse::<u64>() {
            Ok(0) => {
                notice!(
                    "flynnel: FLYNNEL_HOST_PROFILE_NS {what} is zero, which marks an uncalibrated profile; measuring this host instead"
                );
                None
            }
            Ok(ns) => Some(ns),
            Err(e) => {
                notice!(
                    "flynnel: FLYNNEL_HOST_PROFILE_NS {what} field {field:?} is not a nanosecond count ({e}); measuring this host instead"
                );
                None
            }
        }
    };
    let dispatch_cost_ns = take("dispatch")?;
    let collapse_threshold_ns = take("collapse")?;
    let jec_wake_threshold_ns = take("wake")?;
    if fields.next().is_some() {
        notice!(
            "flynnel: FLYNNEL_HOST_PROFILE_NS takes exactly dispatch,collapse,wake; measuring this host instead"
        );
        return None;
    }
    Some(HostDispatchProfile {
        dispatch_cost_ns,
        collapse_threshold_ns,
        jec_wake_threshold_ns,
    })
}

/// Reports a dispatch's occupancy to its call site as the dispatch
/// ends, however it ends.
///
/// It reports and routes nothing. What it is for is the reader: leaf
/// times gathered while the pool was off its cores carry the machine's
/// other tenants, and the figure the classifier derives from those
/// times is a variance, which preemption inflates by landing on some
/// leaves and not others. The occupancy says whether that happened; no
/// learner refuses a window on it and no dispatch is shaped by it.
///
/// A dispatch whose workers never carried an on-core count contributes
/// no ticks at all, so the elapsed total stays zero and this reports
/// nothing rather than reporting a dispatch that never left its cores.
struct ReportOccupancy {
    thread_at_start: u64,
    wall_at_start: u64,
    site: crate::sched::call_site::SiteRef,
}

impl ReportOccupancy {
    fn start(site: crate::sched::call_site::SiteRef) -> Self {
        let (thread_at_start, wall_at_start) = site.get().pool_ticks();
        Self { thread_at_start, wall_at_start, site }
    }
}

impl Drop for ReportOccupancy {
    fn drop(&mut self) {
        let (thread, wall) = self.site.get().pool_ticks();
        let elapsed = wall.saturating_sub(self.wall_at_start);
        // No leaves reached the buffered path, so this dispatch says
        // nothing about the pool. Reporting zero would read as total
        // contention rather than as no reading.
        if elapsed == 0 {
            return;
        }
        let on_core = thread.saturating_sub(self.thread_at_start);
        let pct = on_core.saturating_mul(100) / elapsed;
        self.site.get().record_occupancy(pct.min(100) as u32);
    }
}

/// Samples per timed point, and sweeps per crossover.
///
/// A persisted record carries this count beside the spread those
/// samples showed, so a later reader can weigh how much the stored
/// median is worth.
const SAMPLES: usize = 9;

/// The measurements behind [`host_dispatch_profile`]. Dispatch goes
/// through a join bisect of its own with a fixed leaf and plans
/// without an explicit estimate, so nothing here consults the values
/// it produces.
fn measure_host_dispatch() -> (HostDispatchProfile, u32) {
    const LEAF: usize = 256;
    const MAX_ITEMS: usize = 1 << 17;

    /// Samples behind the dispatch cost. Larger than [`SAMPLES`]
    /// because one of these times a single join rather than a sweep
    /// over doubling sizes, so the whole set costs a few milliseconds.
    const DIRECT_SAMPLES: usize = 1024;

    /// The fastest of [`SAMPLES`] timings of `f`.
    ///
    /// The fastest of n samples estimates the left tail, not the
    /// central tendency, and the two sides of a sweep are not equally
    /// noisy: a serial timing repeats to within a percent while a
    /// dispatched one is right-skewed by steal timing and worker wake.
    /// So this reads a dispatch cost below what a dispatch typically
    /// costs, measured on a Ryzen 7 2700 at a dispatched point of 6200
    /// ns whose median over the same nine samples was 9600.
    ///
    /// The bias is accepted because the alternative is worse where it
    /// matters. A median of the same samples tracks the heavy tail
    /// upward, and a high draw raises the collapse threshold and keeps
    /// work on the calling thread that the pool should have had: on
    /// that host a median-estimator calibration drew a 38700 ns
    /// dispatch cost and the light 10k cell fell from 25.8 to 49.4 us
    /// against a 2.76x baseline. Under-estimating dispatch spends some
    /// overhead on work that did not need the pool; over-estimating it
    /// forfeits the parallelism outright.
    ///
    /// `FLYNNEL_PROFILE_SAMPLES=1` writes every sample of each timed
    /// point as a note, so the spread and this bias can be read off
    /// one calibration rather than inferred from repeated runs.
    fn fastest<F: FnMut()>(mut f: F) -> u64 {
        let mut samples = [0u64; SAMPLES];
        for slot in &mut samples {
            let t0 = std::time::Instant::now();
            f();
            *slot = t0.elapsed().as_nanos() as u64;
        }
        samples.sort_unstable();
        if std::env::var_os("FLYNNEL_PROFILE_SAMPLES").is_some() {
            notice!(
                "profile point: min {} median {} max {} samples {samples:?}",
                samples[0],
                samples[SAMPLES / 2],
                samples[SAMPLES - 1]
            );
        }
        samples[0]
    }
    // Eight dependent multiply-rotate steps per item: compute-bound
    // at about eight nanoseconds an item, so the dispatched version
    // scales with cores the way the work callers estimate does. A
    // streaming body would measure the memory system instead, and on
    // a 24-thread host the pool never beat a serial stream below the
    // cap, which would have collapsed compute the pool runs eight
    // times faster.
    fn body(items: &mut [u64]) {
        for x in items.iter_mut() {
            let mut v = *x;
            for _ in 0..8 {
                v = v.wrapping_mul(0x9E37_79B9_7F4A_7C15).rotate_left(13);
            }
            *x = v;
        }
        std::hint::black_box(&*items);
    }
    fn dispatched(plan: &JobPlan, items: &mut [u64]) {
        if items.len() <= LEAF {
            body(items);
            return;
        }
        let mid = items.len() >> 1;
        let (lo, hi) = items.split_at_mut(mid);
        join_context(plan, |_| dispatched(plan, lo), |_| dispatched(plan, hi));
    }
    // One sweep: doubling counts from four leaves, timing `first`
    // against `second` at each; the crossover is the `first` time
    // where `second` minus `first` reaches zero, interpolated
    // linearly between the last two counts so the doubling steps do
    // not quantize it to a factor of two. A sweep whose second side
    // never catches up reports the `first` time at the largest count
    // it timed.
    fn sweep<A, B>(v: &mut [u64], mut first: A, mut second: B) -> u64
    where
        A: FnMut(&JobPlan, &mut [u64]),
        B: FnMut(&JobPlan, &mut [u64]),
    {
        let mut n = 4 * LEAF;
        let mut prev: Option<(u64, i64)> = None;
        loop {
            let plan = JobPlan::new(0, n as u32);
            let a = fastest(|| first(&plan, &mut v[..n]));
            let b = fastest(|| second(&plan, &mut v[..n]));
            let diff = b as i64 - a as i64;
            if diff <= 0 {
                return interpolate_crossover(prev, a, diff);
            }
            if n >= v.len() {
                return a;
            }
            prev = Some((a, diff));
            n <<= 1;
        }
    }
    /// The median of [`SAMPLES`] runs of `sweep_once`.
    ///
    /// A crossover is derived from two timings rather than being one,
    /// so the minimum of several runs is whichever run's cancellation
    /// noise happened to fall furthest in one direction. That is biased
    /// low and spreads wider than the median of the same runs. The
    /// timing points inside a sweep keep their own minimum, which is
    /// where suppressing that noise is the right move.
    /// Returns the median beside what the rest of the samples say about
    /// it: the spread is how far apart the extremes fell, and a host
    /// carrying other work spreads an order wider than an idle one.
    fn median_sweep<F: FnMut() -> u64>(mut sweep_once: F) -> (u64, u32) {
        let mut samples = [0u64; SAMPLES];
        for slot in &mut samples {
            *slot = sweep_once();
        }
        samples.sort_unstable();
        if std::env::var_os("FLYNNEL_PROFILE_SAMPLES").is_some() {
            // Both dispersion figures, so a set of draws taken across
            // known conditions can be compared on either without being
            // re-run. The spread is what the trust check reads today;
            // the IQR is the candidate it would be read against.
            notice!(
                "profile sweep: min {} median {} max {} spread {} iqr {} per mille samples {samples:?}",
                samples[0],
                samples[SAMPLES / 2],
                samples[SAMPLES - 1],
                sample_spread_per_mille(&samples),
                sample_iqr_per_mille(&samples),
            );
        }
        (samples[SAMPLES / 2], sample_spread_per_mille(&samples))
    }
    let mut v: Vec<u64> = (0..MAX_ITEMS as u64).collect();
    // The first dispatches wake a cold pool and fault the buffer in;
    // sixteen are discarded.
    let warm = JobPlan::new(0, (4 * LEAF) as u32);
    for _ in 0..16 {
        dispatched(&warm, &mut v[..4 * LEAF]);
    }
    body(&mut v);
    // The pool's own dispatch cost: one join whose two halves are a
    // single item each, so the time is the dispatch rather than a
    // difference between two larger numbers. Below this much work no
    // dispatch can pay, whatever the sweep read.
    //
    // Observed rather than differenced, and the median rather than the
    // fastest, for the same reason. The sweeps above subtract one
    // timing from another of comparable size, where the fastest of
    // each side suppresses cancellation noise; nothing is subtracted
    // here, so the median estimates the cost a dispatch actually pays.
    // On a Ryzen 7 2700 across ten processes this reads 1900 to 2700
    // ns where the subtraction read 1 to 2300, the 1 being a floor
    // engaging on a difference that had gone to zero.
    let plan_two = JobPlan::new(0, 2);
    let mut samples = [0u64; DIRECT_SAMPLES];
    for slot in &mut samples {
        let (lo, hi) = v[..2].split_at_mut(1);
        let t0 = std::time::Instant::now();
        join_context(&plan_two, |_| body(lo), |_| body(hi));
        *slot = t0.elapsed().as_nanos() as u64;
    }
    samples.sort_unstable();
    let dispatch_cost_ns = samples[DIRECT_SAMPLES / 2].max(1);
    let serial_vs_pool = |v: &mut [u64]| {
        sweep(v, |_, items| body(items), dispatched)
    };
    let (collapse_median, collapse_spread) = median_sweep(|| serial_vs_pool(&mut v));
    let collapse_threshold_ns = collapse_median.max(dispatch_cost_ns);
    // The wake path against polling: the same dispatched body under
    // the two dispatch scopes.
    let polling_vs_wake = |v: &mut [u64]| {
        sweep(
            v,
            |plan, items| {
                let _scope = crate::sched::arena_local::DispatchScope::new_if_change(false);
                dispatched(plan, items);
            },
            |plan, items| {
                let _scope = crate::sched::arena_local::DispatchScope::new_if_change(true);
                dispatched(plan, items);
            },
        )
    };
    let (jec_wake_threshold_ns, wake_spread) = median_sweep(|| polling_vs_wake(&mut v));
    (
        HostDispatchProfile { dispatch_cost_ns, collapse_threshold_ns, jec_wake_threshold_ns },
        collapse_spread.max(wake_spread),
    )
}

/// Spread of a sorted sample set in parts per thousand of its median.
///
/// A calibration keeps the median of its samples; this is what the rest
/// of them say about how far that median can be trusted. An idle host
/// lands in the tens, a host carrying other work in the hundreds.
pub fn sample_spread_per_mille(sorted: &[u64]) -> u32 {
    if sorted.len() < 2 {
        return 0;
    }
    let median = sorted[sorted.len() / 2].max(1);
    let span = sorted[sorted.len() - 1].saturating_sub(sorted[0]);
    (span.saturating_mul(1000) / median).min(u32::MAX as u64) as u32
}

/// Interquartile range of a sorted sample set, in parts per thousand of
/// its median.
///
/// Reported beside [`sample_spread_per_mille`] because the two describe
/// different things and the difference decides whether a record is
/// usable. The full range is defined by the extremes, so a single
/// scheduling hiccup in nine samples sets it; the median a calibration
/// keeps is chosen precisely because it survives that sample. Judging
/// the one by the other refuses records whose medians are reproducible.
///
/// Measured on a 99.9-percent idle guest: five independent draws agreed
/// on their medians to within 8 percent while each reported a range of
/// 160 to 272 percent. Those cannot both describe the same dispersion.
///
/// This does not solve that, and neither can any figure taken over one
/// draw's samples. Across three load levels on two hosts it inverts
/// exactly as the range does: under saturation every sample is slowed
/// by about the same factor, so the samples agree with each other
/// while the medians independent draws produce scatter by thousands of
/// parts per thousand. On a Zen3 guest this read 128 quiet against 86
/// saturated while the medians went 83 to 432,299. The property a
/// bound wants is whether the median reproduces, and that is a
/// statement about two draws rather than about nine samples.
///
/// Reported beside the range because the two together are what show
/// the inversion belongs to within-draw dispersion rather than to the
/// range in particular. Nothing gates on either.
pub fn sample_iqr_per_mille(sorted: &[u64]) -> u32 {
    if sorted.len() < 4 {
        return 0;
    }
    let median = sorted[sorted.len() / 2].max(1);
    // Nearest-rank quartiles. Exact interpolation would be a choice
    // about a nine-sample set that the bound derivation should make, not
    // this function.
    let q1 = sorted[sorted.len() / 4];
    let q3 = sorted[(sorted.len() * 3) / 4];
    (q3.saturating_sub(q1).saturating_mul(1000) / median).min(u32::MAX as u64) as u32
}

/// True when the caller capped this dispatch to a single worker, which
/// [`JobPlan::with_workers`] documents as serial execution on the
/// calling thread. Read from the plan alone, so a capped call touches
/// neither the pool nor the host profile, at every data-parallel entry.
#[inline]
fn capped_to_caller(plan: &JobPlan) -> bool {
    plan.capped_to_caller()
}

/// True when this dispatch runs its body on the calling thread: the
/// caller capped it to a single worker, or the caller's own estimate
/// puts the total under the collapse threshold.
#[inline]
fn runs_on_caller(plan: &JobPlan, n: usize) -> bool {
    capped_to_caller(plan) || collapses_inline(plan, n)
}

/// True when the caller's explicit estimate puts `n` items under
/// [`inline_collapse_threshold_ns`], and this site has not already
/// run a collapsed body slower than that threshold.
///
/// An estimate low enough to admit work that then overruns costs the
/// difference between running inline and running on the pool, which
/// is unbounded in the size of the underestimate. One overrun at a
/// site is enough to stop trusting it there.
///
/// The latch is per site and lives for the process, so a site that
/// overran once dispatches every later call from that source
/// location whatever its size. A benchmark that sweeps sizes through
/// one call site measures its own order as well as the sizes.
#[inline]
fn collapses_inline(plan: &JobPlan, n: usize) -> bool {
    if plan.site.is_some_and(|s| s.get().collapse_overran()) {
        return false;
    }
    plan.estimated_per_item_ns_explicit
        && plan.effective_ns_per_elem().is_some_and(|per| {
            (per as u64).saturating_mul(n as u64) < inline_collapse_threshold_ns()
        })
}

/// Run `body` on the calling thread and record it as a leaf.
///
/// Called only where [`runs_on_caller`] answered true, so a call the
/// worker cap did not put here is one the caller's estimate admitted,
/// and a body of that kind which outruns the collapse threshold latches
/// the site against its estimate. A body the cap put here says nothing
/// about the estimate, so it leaves the latch alone and never reads the
/// host profile, whose first read measures by dispatching into the pool.
#[inline]
fn run_on_caller<R>(plan: &JobPlan, items: usize, body: impl FnOnce() -> R) -> R {
    let t0 = std::time::Instant::now();
    let out = record_leaf(plan.site, items, body);
    if !capped_to_caller(plan)
        && let Some(site) = plan.site
    {
        site.get()
            .note_collapsed_body(t0.elapsed().as_nanos() as u64, inline_collapse_threshold_ns());
    }
    out
}

/// Apply `op` to every element of `items` in parallel by
/// recursively bisecting the slice. Each leaf chunk is processed
/// serially by `op`, which is the right granule for SIMD loops.
///
/// The leaf count is derived per dispatch and this entry does not
/// expose it. Which mechanism derives it depends on what the plan
/// carries.
///
/// With an authoritative per-item estimate the recursion seeds
/// `2^adaptive_seed_depth` leaves, where the target is
/// `max(workers, items * per_item_ns / TARGET_LEAF_WORK_NS)` capped
/// at `items.len()`, rounded up to a power of two, and floored at
/// `workers` rounded the same way. Past that seed
/// `bisect_lazy_steal_driven` splits further only on observed steal
/// pressure, down to the recursion floor. A factor the caller set with
/// `with_oversubscription_log2` shifts that depth; one from a profile
/// or a learned class does not.
///
/// Without one, a probe measures a prefix and the tail runs under a
/// split budget of `workers * effective_leaves_per_worker()`, halved
/// per level and replenished to the full budget when a subtree is
/// stolen. That multiplier belongs to the split observer, which
/// retunes it from the pool's steal rate within `[1, 8]` unless the
/// caller pinned one with `with_oversubscription_log2`, so the budget
/// is a figure this process holds at that moment rather than a
/// constant. A pinned [`crate::sched::plan::BisectVariant`] takes the
/// same budget shape with a multiplier of one.
///
/// That variant is not pinned only by a caller. On a host whose vendor
/// resolves to [`crate::sched::adaptive_variant_routing::VariantRouting::ComputeBatchAdaptive`],
/// which is the CPUID default on AMD, a plan carrying
/// [`crate::dispatch_profile::DispatchProfile::PortBound`] is given one
/// at construction, chosen by the item count either side of
/// [`crate::sched::adaptive_variant_routing::COMPUTE_BATCH_LARGE_N`].
/// Every other profile is left on the seed-depth route.
///
/// So on such a host PortBound takes a different bisect from the other
/// profiles, and a caller comparing profiles is comparing routes as
/// well. PortBound is also what an unclassified site collapses to, so
/// this is the common case rather than a corner of it.
///
/// So no route carries a fixed ceiling of the form
/// `worker_count() * k`. The bound that holds on all of them is
/// `items.len() / floor`.
///
/// A caller cannot ask these entries for a leaf count.
/// [`for_each_chunk_min_leaf`] sets the floor, which bounds the count
/// from above through that quotient rather than choosing it.
///
/// `op` must be `Sync` because the same closure body is invoked
/// in parallel from multiple workers (each on its own chunk).
#[track_caller]
pub fn for_each_chunk<T, F>(plan: &JobPlan, items: &mut [T], op: F)
where
    T: Send,
    F: Fn(&mut [T]) + Sync,
{
    for_each_chunk_min_leaf(plan, items, MIN_LEAF_ITEMS, op)
}

/// Same as [`for_each_chunk`] but the recursion floor is
/// caller-supplied.
///
/// The floor is a ceiling on the leaf size `adaptive_min_leaf` may
/// pick, so a smaller value lets the seed depth and steal pressure
/// decide the count instead of the floor deciding it. Heavy
/// per-element ops pass 1 and get a leaf per item, where the default
/// [`MIN_LEAF_ITEMS`] bounds the count at `n / MIN_LEAF_ITEMS` and
/// runs a small batch serially.
///
/// Passing a floor does not request a leaf count. It sets the point
/// the recursion stops, and the count that results depends on the
/// per-item estimate and on how much the pool steals.
///
/// `min_leaf` bounds leaf size from the opposite side to the sibling
/// helpers: `adaptive_min_leaf` may only pick at or below it, so
/// `items.len() / min_leaf` is a lower bound on the leaf count. In
/// [`for_each_chunk_indexed_min_leaf`] and
/// [`for_each_chunk_triple_min_leaf`] it reaches the bisect unchanged
/// and the same quotient is an upper bound.
#[track_caller]
pub fn for_each_chunk_min_leaf<T, F>(
    plan: &JobPlan,
    items: &mut [T],
    min_leaf: usize,
    op: F,
)
where
    T: Send,
    F: Fn(&mut [T]) + Sync,
{
    let n = items.len();
    crate::sched::trace::emit(
        crate::sched::trace::TraceEvent::DispatchEnter,
        n as u32,
    );
    let _trace_exit_guard = scopeguard_dispatch_exit(n);
    if n == 0 {
        return;
    }
    // Per-call-site identity from the caller's source location
    // (track_caller chain). An outer entry's attachment (or a
    // caller's explicit with_site) wins.
    let (plan_owned, routing_arm) = plan
        .with_site_if_none(crate::sched::call_site::caller_site())
        .apply_site_class_measured();
    let _routing = RecordRoutingArm::start(plan_owned.site, routing_arm);
    let plan = &plan_owned;
    // Declared before the flush guard so it drops after it: the pool
    // totals it differences are published by that flush, and a
    // dispatch's last batch of leaves is in it.
    let _occupancy = plan.site.map(ReportOccupancy::start);
    let _flush_on_exit = FlushLeafStatsOnExit;
    // Wake path against polling: at small total work the sleep
    // counter's CAS and condvar wake cost more than they save, and
    // workers find pushed work by spinning (`ROUNDS_UNTIL_SLEEPING`
    // keeps them spinning across typical inter-dispatch gaps); at
    // large total work the wake cascade wins. The switch point is
    // this host's measured crossover of the two.
    let estimated_total_ns: u64 = plan
        .effective_ns_per_elem()
        .map(|ns| (ns as u64).saturating_mul(n as u64))
        // Unknown cost: assume large workload, take JEC path
        // (safe default - JEC is correct for any size, just
        // higher overhead on small).
        .unwrap_or(u64::MAX);
    let use_jec_wake = estimated_total_ns >= jec_wake_threshold_ns();
    let _jec_scope = crate::sched::arena_local::DispatchScope::new_if_change(use_jec_wake);

    // Inline-collapse fast path: when the caller has supplied an
    // authoritative per-item cost (via with_estimated_per_item_ns or
    // with_cost_ns_per_elem) and total work is under this host's
    // measured collapse threshold, skip the pool entirely.
    //
    // Gating on plan.estimated_per_item_ns_explicit (not just
    // effective_ns_per_elem) is critical: JobPlan::new auto-populates
    // estimated_per_item_ns with the classifier default 12 / 50 /
    // 600 ns. For N=4 BigFloat-heavy items the classifier default
    // (12 * 4 = 48ns < 50us) would falsely trigger inline-collapse
    // even though each item is 10ms of real compute. The probe path
    // downstream measures actual cost; let it run instead of
    // shortcutting here based on a guess.
    if runs_on_caller(plan, n) {
        run_on_caller(plan, items.len(), || op(items));
        return;
    }

    let workers = plan.effective_workers(global_local_arena().total_workers());

    // The recursion floor. A fine-grain op at around 10ns an element
    // wants a floor in the hundreds, because per-leaf dispatch
    // dominates below that; heavy per-element work wants a floor of
    // one, since four items at 10ms each saturate four cores as four
    // leaves where a 256-item floor runs them on the calling thread.
    //
    // With an authoritative per-item estimate `adaptive_min_leaf`
    // sizes a leaf to carry about one dispatch's worth of work,
    // `pool_dispatch_cost_ns() / per_item_ns`, floored at one item;
    // without one it takes the floor passed in here. Either way the
    // result is capped at that floor, so a caller passing 1 gets 1.
    //
    // The numerator is measured once per process, so the floor is a
    // figure this process holds rather than a constant of the machine.
    let effective_min_leaf = adaptive_min_leaf(plan, min_leaf.max(1));

    // `FLYNNEL_SEED_DEPTH` reports the floor beside the seeded leaf
    // count that `adaptive_seed_depth` prints. They are two separate
    // decisions and only the second was observable, so a floor moving
    // with a per-process measurement changed the fan-out while every
    // printed figure stayed still.
    //
    // Reads nothing that would start the calibration: a print that
    // measured the host would change what it reports on.
    //
    // One line per distinct shape, keyed on what the floor depends on,
    // so a sweep prints once per size rather than once per dispatch.
    if seed_depth_reporting() {
        static LAST_FLOOR_SHAPE: core::sync::atomic::AtomicU64 =
            core::sync::atomic::AtomicU64::new(u64::MAX);
        let ns = plan.estimated_per_item_ns.unwrap_or(0);
        let key = ((n as u64) << 32)
            ^ ((ns as u64) << 8)
            ^ effective_min_leaf as u64;
        if LAST_FLOOR_SHAPE.swap(key, core::sync::atomic::Ordering::Relaxed) != key {
            notice!(
                "leaf floor: items {n} per_item_ns {:?} explicit {} \
                 use_smt {} caller_floor {} effective_floor {effective_min_leaf}",
                plan.estimated_per_item_ns,
                plan.estimated_per_item_ns_explicit,
                plan.use_smt,
                min_leaf.max(1)
            );
        }
    }

    // Probe-and-decide path: when the caller hasn't given us a cost
    // estimate and the workload is small relative to the worker
    // count (n < workers * MIN_LEAF_ITEMS, meaning we can't even
    // make one MIN_LEAF-sized chunk per worker), we don't know
    // whether this work is dispatch-dominated (Light bench) or
    // genuinely parallelizable (Heavy bench).
    //
    // Probe a tiny leaf inline, measure actual per-element cost,
    // then decide:
    //   * the tail's measured work is below this host's collapse
    //     crossover -> finish inline serially, skipping the pool.
    //   * otherwise -> bisect the tail with the observed cost as the
    //     leaves_per_worker driver.
    //
    // That crossover is `inline_collapse_threshold_ns()`, measured on
    // the running host and cached for the process, and the comparison
    // is against the whole tail rather than a per-worker share.
    //
    // Gated on `plan.oversubscription_log2.is_none()` too: if the
    // caller explicitly set an oversub override they opted in to
    // a specific dispatch shape and we honor it. The probe is
    // 32 items - small enough that the overhead is in the
    // single-digit-microsecond range even on Heavy ops (~19us
    // for 600ns/elem), but big enough that per-call function
    // overhead doesn't dominate the timing.
    const PROBE_SIZE: usize = 32;
    // Probe-and-decide fires when we lack an authoritative per-item cost
    // information. JobPlan::new + set_profile auto-populate
    // `estimated_per_item_ns` with a classifier default (12 / 50 /
    // 600 ns); those values are routing hints, not measurements. When
    // the caller hasn't explicitly called with_estimated_per_item_ns,
    // we treat the auto-populated value as a guess and probe to learn
    // actual cost. Caller-explicit hints (the bench's
    // with_estimated_per_item_ns calls, or production code that
    // measured the workload) skip the probe -- they're authoritative.
    let cost_authoritative = plan.estimated_per_item_ns_explicit;
    // The probe path runs item 0 serially first to measure cost,
    // then bisects the tail. For heavy items this adds a full
    // item's runtime to the critical path: NMFD 5x100ms takes
    // 100ms-of-serial-probe + 50ms-of-parallel-tail = 150ms vs
    // rayon's 62ms (8-core ideal). That serial probe is the main
    // reason `flynnel_def` measured 2x slower than rayon on
    // cold_workloads NMFD-like cells.
    //
    // Skip the probe when the static classifier has already
    // signaled heavy per-item via use_smt=true (LatencyBound).
    // In that case the workload character is known; jumping
    // straight to the bisect with min_leaf=1 (from
    // adaptive_min_leaf's use_smt shortcut) gives full parallel
    // distribution from the very first call. When the classifier's
    // heavy guess is wrong, the observer migrates the global
    // class within a few iterations.
    if !cost_authoritative
        && !plan.use_smt
        && n < workers.saturating_mul(MIN_LEAF_ITEMS)
    {
        // Cold-path entry: production callers either supply
        // with_estimated_per_item_ns (cost_authoritative=true) or
        // hit the LatencyBound classifier shortcut (use_smt=true).
        // The probe path only fires for hint-less PortBound calls
        // at small N -- the rare measurement-detour case. Mark it
        // cold so LLVM (>= 21 since rust 1.96) keeps the
        // hint/SMT-classified fast paths icache-warm and reorders
        // this probe-and-amend-plan sequence into a cold section.
        core::hint::cold_path();
        // Probe must leave a non-empty tail so the bisect path has
        // items to parallelize after measurement. A `n.min(PROBE_SIZE)`
        // probe consumes the entire input when n <= PROBE_SIZE
        // (n=5 -> probe=5, tail=0, return before bisect),
        // serializing small-N heavy-per-item workloads (BigFloat
        // verify, NMFD with n<=8 instances) that rayon
        // parallelizes 3-4x.
        //
        // Formula: probe is at most 1/8 of the input, floored at 1,
        // capped at PROBE_SIZE. For n=5: probe=1, tail=4. For n=32:
        // probe=4, tail=28. For n=256: probe=32, tail=224. Tail
        // always has at least 7/8 of the items so the post-probe
        // bisect can drive parallel work-stealing.
        let probe_size = (n / 8).clamp(1, PROBE_SIZE);
        // Staged probe: time one item first. A single item at or
        // above the trust floor (20x the measured timer bracket,
        // bracket error <= ~5%) is a reliable heavy measurement,
        // and the remaining probe_size - 1 items skip the serial
        // detour: at 128 items x 500us the full n/8 probe costs
        // ~8 ms of serial work vs ~0.5 ms for one item. A light
        // first item is bracket-noise-dominated, so the rest of
        // the n/8 batch is probed for a reliable average (cheap,
        // the items are light).
        let (first, after_first) = items.split_at_mut(1);
        let start = std::time::Instant::now();
        record_leaf(plan.site, first.len(), || op(first));
        let first_ns = start.elapsed().as_nanos() as u64;
        if after_first.is_empty() {
            // n=1: the single item was just processed inline.
            return;
        }
        let (probe_ns, probed, tail) = if first_ns >= probe_trust_floor_ns() || probe_size == 1 {
            (first_ns, 1usize, after_first)
        } else {
            let extra = (probe_size - 1).min(after_first.len().saturating_sub(1));
            if extra == 0 {
                (first_ns, 1usize, after_first)
            } else {
                let (more, rest) = after_first.split_at_mut(extra);
                let t2 = std::time::Instant::now();
                record_leaf(plan.site, more.len(), || op(more));
                let more_ns = t2.elapsed().as_nanos() as u64;
                (first_ns + more_ns, 1 + extra, rest)
            }
        };
        let mut per_elem_ns = probe_ns.max(1) / probed as u64;
        let mut tail = tail;
        // The tail dispatches when its measured work is past this
        // host's collapse crossover, the same question the explicit
        // estimate answers at entry.
        let dispatch_floor_ns = inline_collapse_threshold_ns();
        // Confirmation probe: a probe preempted by the OS
        // mid-measurement overstates per-item cost by orders of
        // magnitude and would misroute a trivial workload to the
        // pool. When the probe says "dispatch" and the light-path
        // batch ran (probed >= 8, so re-measurement is cheap
        // relative to it), time a tiny second probe and take the
        // MIN per-item estimate: preemption inflates a
        // measurement, never deflates one. The staged-heavy path
        // (probed == 1) skips confirmation: a second heavy item
        // on the critical path is the cost the staged probe
        // exists to avoid, and an inflated single-item reading
        // still routes to the pool, where the bisect's leaf
        // timing corrects the site.
        const CONFIRM_SIZE: usize = 4;
        // A single trusted item is confirmed when it could be a cold
        // first call rather than a heavy item: the first call at a
        // site pays one-time costs (code and site state faulting in)
        // that measured 30 us on a quiet 16-worker host and 79 us
        // under a saturated one for a one-add item, against an 80 us
        // dispatch floor there and a 40 us floor on 8 workers, so the
        // allowance is four floors. Up to three further items are
        // timed one at a time and the minimum taken, since preemption
        // only ever inflates a reading (a Windows quantum runs to
        // milliseconds); the loop stops at the first reading that
        // puts the tail under the floor, so a light item costs one
        // confirmation and a heavy one at most three, each at most
        // the allowance. It runs only when the tail is at least a
        // worker's width, where a misreading would dispatch a wide
        // tail of light items; a small tail costs little either way
        // and keeps the single-item critical path. An item beyond
        // the allowance is heavy on its own evidence.
        const CONFIRM_SINGLE_MAX: usize = 3;
        let cold_start_allowance_ns = dispatch_floor_ns.saturating_mul(4);
        if probed == 1 && first_ns < cold_start_allowance_ns && tail.len() >= workers {
            for _ in 0..CONFIRM_SINGLE_MAX {
                if per_elem_ns.saturating_mul(tail.len() as u64) < dispatch_floor_ns
                    || tail.len() <= 1
                {
                    break;
                }
                let (one, rest) = tail.split_at_mut(1);
                let start = std::time::Instant::now();
                record_leaf(plan.site, one.len(), || op(one));
                per_elem_ns = per_elem_ns.min(start.elapsed().as_nanos().max(1) as u64);
                tail = rest;
            }
        }
        if per_elem_ns.saturating_mul(tail.len() as u64) >= dispatch_floor_ns
            && probed >= 8
            && tail.len() > CONFIRM_SIZE
        {
            let (confirm, rest) = tail.split_at_mut(CONFIRM_SIZE);
            let start = std::time::Instant::now();
            record_leaf(plan.site, confirm.len(), || op(confirm));
            let confirm_ns = start.elapsed().as_nanos() as u64;
            per_elem_ns = per_elem_ns.min(confirm_ns.max(1) / CONFIRM_SIZE as u64);
            tail = rest;
        }
        let est_tail_ns = per_elem_ns.saturating_mul(tail.len() as u64);
        if est_tail_ns < dispatch_floor_ns {
            // Dispatch overhead would exceed remaining work; complete
            // the tail serially on the calling thread.
            record_leaf(plan.site, tail.len(), || op(tail));
            return;
        }
        // Tail is worth parallelizing. Match the observer-tuned
        // multiplier (default 2 leaves per worker) - the probe path
        // fires when no profile was provided, so we shouldn't
        // over-bisect to clamp(.., 8) like the cost-derived branch
        // would. Measured on Genoa EPYC 9B14, the
        // 8-clamp regressed Heavy/10k by 8% via probe-path
        // over-bisection vs the conservative observer-multiplier
        // choice.
        let leaves_per_worker = plan.effective_leaves_per_worker();
        let max_budget = workers.saturating_mul(leaves_per_worker).max(1);
        // Probe path: use the per-element cost we just measured to
        // pick min_leaf adaptively (same formula as the main path
        // above). per_elem_ns came from the probe measurement so
        // it's authoritative for this workload's actual cost on
        // this host.
        let probe_min_leaf = pool_dispatch_cost_ns()
            .checked_div(per_elem_ns)
            .map(|raw| (raw.max(1) as usize).min(MIN_LEAF_ITEMS))
            .unwrap_or(MIN_LEAF_ITEMS);
        // Amend the plan with the measured per_item_ns so the bisect's
        // join_context -> pick_tier path observes a heavy-per-item
        // signal and routes to the worker pool. Without this, pick_tier
        // returns Inline for batch_size < 32 and the bisect runs
        // serially regardless of what min_leaf we computed. The probe
        // already learned the workload is heavy; record that knowledge
        // in the plan we hand down.
        let mut amended_plan = *plan;
        amended_plan.estimated_per_item_ns = Some(per_elem_ns.min(u32::MAX as u64) as u32);
        // Mark the probe-derived estimate as authoritative: it came
        // from a real measurement on this host, not a classifier
        // default. pick_tier's heavy_override consults this flag
        // when deciding small-N parallel dispatch.
        amended_plan.estimated_per_item_ns_explicit = true;
        amended_plan.batch_size = tail.len() as u32;
        bisect(&amended_plan, tail, &op, max_budget, max_budget, false, probe_min_leaf);
        return;
    }

    // Default path: continuation-steal-lazy bisect. First level
    // always splits to seed initial fanout (workers eager leaves);
    // subsequent levels run serially inline unless the per-deque
    // steal counter on the dispatching worker has incremented since
    // the last check. Wins +12.7% Xeon Heavy/10k, +10.4% Xeon
    // Compute/100k, +5.1% Zen3 Compute/100k, never regresses across
    // the 12-cell (host x workload x size) bench matrix.
    if plan.bisect_variant.is_none() {
        let seed_depth = adaptive_seed_depth(plan, items.len(), workers);
        bisect_lazy_steal_driven(plan, items, &op, seed_depth, 0, effective_min_leaf);
        return;
    }

    // The two pinned BisectVariants drive an alternative
    // (leaves_per_worker, use_rayon_replenish) resolution into the
    // baseline bisect / bisect_rayon_style helpers. Routed
    // automatically by `adaptive_variant_routing` on AMD Compute;
    // callers can also pin per-plan via `with_bisect_variant`.
    let (leaves_per_worker, use_rayon_replenish) = match plan.bisect_variant {
        // ProducerMaxLenWorkers: upfront leaves exactly match rayon's
        // LengthSplitter default initial count = workers * 1. Wins
        // +19.6% on Zen3 Compute/100k.
        Some(crate::sched::plan::BisectVariant::ProducerMaxLenWorkers) => {
            (1usize, false)
        }
        // RayonStyleReplenish: start with one leaf per worker; on
        // each observed steal, replenish to max(workers, splits/2)
        // instead of the upfront budget. Wins +37.6% on Zen3
        // Compute/10k.
        Some(crate::sched::plan::BisectVariant::RayonStyleReplenish) => {
            (1usize, true)
        }
        // None is handled by the lazy-steal early-return above.
        None => unreachable!("None branch handled above"),
    };
    let max_budget = workers.saturating_mul(leaves_per_worker).max(1);
    if use_rayon_replenish {
        bisect_rayon_style(plan, items, &op, max_budget, workers, false, effective_min_leaf);
    } else {
        bisect(plan, items, &op, max_budget, max_budget, false, effective_min_leaf);
    }
}

/// Apply `op` to each chunk of size at most `chunk_size`, in
/// parallel. Chunks at the slice's tail may be smaller than
/// `chunk_size`. Useful when the caller knows a SIMD-optimal
/// chunk shape (e.g., AVX-512 16-lane FpN<16> = 64 items per
/// chunk to fill 16 ZMM registers x 4 lanes each).
#[track_caller]
pub fn for_each_fixed_chunk<T, F>(
    plan: &JobPlan,
    items: &mut [T],
    chunk_size: usize,
    op: F,
)
where
    T: Send,
    F: Fn(&mut [T]) + Sync,
{
    let n = items.len();
    if n == 0 {
        return;
    }
    // Per-call-site identity from the caller's source location
    // (track_caller chain); an outer attachment wins.
    let (plan_owned, routing_arm) = plan
        .with_site_if_none(crate::sched::call_site::caller_site())
        .apply_site_class_measured();
    let _routing = RecordRoutingArm::start(plan_owned.site, routing_arm);
    let plan = &plan_owned;
    let chunk = chunk_size.max(1);
    if n <= chunk {
        // Whole-input serial pass: one span for the SITE's
        // statistics only. It is not a chunk leaf, so it stays out
        // of the process-global per-leaf counters.
        let t0 = std::time::Instant::now();
        op(items);
        record_leaf_span_ns(plan.site, t0.elapsed().as_nanos() as u64, n as u64);
        return;
    }
    if capped_to_caller(plan) {
        // The same chunks a dispatch would hand out, in order: a caller
        // such as par_zip_apply builds its body around the width.
        run_on_caller(plan, n, || {
            for c in items.chunks_mut(chunk) {
                op(c);
            }
        });
        return;
    }
    let n_chunks = n.div_ceil(chunk);
    let max_budget = n_chunks.max(1);
    bisect_fixed(plan, items, &op, max_budget, max_budget, chunk, false);
}

/// Recursive bisection driver for `for_each_chunk`. Cuts the
/// slice in half on each level, dispatching the halves via
/// `sched::join_context`.
///
/// # Adaptive splitter (SLAW-style)
///
/// `splits` is the remaining split budget; `max_budget` is the
/// full budget to replenish to when migrated. On each call:
///
/// - If `migrated == true`, we observed steal pressure (some
///   peer dequeued this side from another worker's deque). The
///   pool is hungrier than the original split estimate; reset
///   `splits` to `max_budget` so this subtree can split log2(N)
///   more levels and produce many more leaves for distribution.
/// - Otherwise, the budget halves per level until it hits zero.
///
/// Bottoms out when either `items.len() <= MIN_LEAF_ITEMS` (too
/// small to split further) or `splits == 0` and `!migrated` (no
/// pressure, no budget).
///
/// This is the SLAW ("Scalable Locality-aware Adaptive Work-
/// stealing", Guo-Zhao-Cavé-Sarkar, IPDPS 2010) adaptive work-
/// stealing pattern: more splits where there is observed
/// contention, fewer where workers are saturated.
fn bisect<T, F>(
    plan: &JobPlan,
    items: &mut [T],
    op: &F,
    splits: usize,
    max_budget: usize,
    migrated: bool,
    min_leaf: usize,
)
where
    T: Send,
    F: Fn(&mut [T]) + Sync,
{
    if items.len() <= min_leaf {
        record_leaf(plan.site, items.len(), || op(items));
        return;
    }
    // Replenish on stealing pressure.
    let cur_splits = if migrated { max_budget } else { splits };
    // Leaf when we're out of budget. The check uses `<= 1` rather
    // than `== 0`: a split with `cur_splits == 1` would emit two
    // children each holding `splits = 0`, and each would
    // immediately leaf. That produces 2*max_budget leaves total
    // instead of the intended max_budget, doubling per-leaf
    // dispatch overhead at fine-grain rapid-fire workloads. The
    // `<= 1` form stops splitting at the level where the budget
    // is consumed, capping the leaf count at the intended
    // max_budget.
    if cur_splits <= 1 {
        record_leaf(plan.site, items.len(), || op(items));
        return;
    }
    let mid = items.len() / 2;
    let (left, right) = items.split_at_mut(mid);
    let half = cur_splits >> 1;
    join_context(
        plan,
        // `left_injected` propagates the migrated/injection signal
        // from the parent (left always runs inline, so it doesn't
        // observe its own steal flag - but if the parent itself
        // was stolen / injected, left should know).
        |left_injected| bisect(plan, left, op, half, max_budget, left_injected, min_leaf),
        // `right_stolen` is the freshly-observed stolen flag for
        // the right half. If a peer dequeued this job, we know
        // the pool is hungry, so this subtree splits more.
        |right_stolen| bisect(plan, right, op, half, max_budget, right_stolen, min_leaf),
    );
}

/// Rayon-style replenish variant of [`bisect`]. Differs from `bisect`
/// in the steal-replenish formula: on observed steal, the new
/// budget is `max(workers_floor, splits / 2)` instead of resetting
/// to the upfront `max_budget`. This mirrors rayon-1.12.0's
/// `Splitter::try_split` formula at
/// `src/iter/plumbing/mod.rs::258-283`:
///
/// ```text
/// if stolen { splits = max(thread_count, splits / 2) }
/// else if splits > 0 { splits /= 2 } else { return false }
/// ```
///
/// `workers_floor` plays the role of `current_num_threads()` in the
/// rayon formula. Used only when `for_each_chunk` is called with
/// [`crate::sched::plan::BisectVariant::RayonStyleReplenish`].
fn bisect_rayon_style<T, F>(
    plan: &JobPlan,
    items: &mut [T],
    op: &F,
    splits: usize,
    workers_floor: usize,
    migrated: bool,
    min_leaf: usize,
)
where
    T: Send,
    F: Fn(&mut [T]) + Sync,
{
    if items.len() <= min_leaf {
        record_leaf(plan.site, items.len(), || op(items));
        return;
    }
    // Rayon's replenish formula: max(workers, splits/2).
    let cur_splits = if migrated {
        workers_floor.max(splits / 2)
    } else {
        splits
    };
    if cur_splits <= 1 {
        record_leaf(plan.site, items.len(), || op(items));
        return;
    }
    let mid = items.len() / 2;
    let (left, right) = items.split_at_mut(mid);
    let half = cur_splits >> 1;
    join_context(
        plan,
        |left_injected| {
            bisect_rayon_style(plan, left, op, half, workers_floor, left_injected, min_leaf)
        },
        |right_stolen| {
            bisect_rayon_style(plan, right, op, half, workers_floor, right_stolen, min_leaf)
        },
    );
}

/// Sampled leaf recorder used by the default lazy-steal bisect
/// path. Only every Nth call goes through the full TSC bracket;
/// the other N-1 calls just run the body.
#[inline]
fn record_leaf_sampled<F: FnOnce() -> R, R>(
    site: Option<crate::sched::call_site::SiteRef>,
    items: usize,
    body: F,
) -> R {
    thread_local! {
        static STRIDE_TICK: std::cell::Cell<u32> = const { std::cell::Cell::new(0) };
    }
    let should_sample = STRIDE_TICK.with(|c| {
        let v = c.get().wrapping_add(1);
        c.set(if v >= LEAF_SAMPLE_STRIDE { 0 } else { v });
        v >= LEAF_SAMPLE_STRIDE
    });
    if should_sample {
        // The on-core bracket is two thread-clock reads on every sampled
        // leaf, which a dispatch pays once per sampled leaf and not once
        // per dispatch. Off by default until a measurement says the
        // spread it buys is worth that.
        if crate::sched::levers::oncore_spread() {
            record_leaf_on_core(site, items, body)
        } else {
            record_leaf(site, items, body)
        }
    } else {
        // Unsampled leaves still appear in the trace (one cached
        // load each when tracing is off) so a traced dispatch shows
        // every leaf and the thread it ran on.
        crate::sched::trace::emit(crate::sched::trace::TraceEvent::LeafStart, 0);
        let out = body();
        crate::sched::trace::emit(crate::sched::trace::TraceEvent::LeafEnd, 0);
        out
    }
}

/// Advance a stride counter and say whether this call is the one in
/// [`LEAF_SAMPLE_STRIDE`] that takes the bracket.
///
/// Separate from [`record_leaf_bracket_sampled`] because the cadence is
/// the whole of what that recorder adds, and the switch it also
/// consults is read once per process and cached. A test driving the
/// recorder could only ever observe whichever arm the process started
/// in, so the cadence would go unasserted while looking covered.
#[inline(always)]
fn advance_bracket_tick(prev: u32) -> (u32, bool) {
    let v = prev.wrapping_add(1);
    let take = v >= LEAF_SAMPLE_STRIDE;
    (if take { 0 } else { v }, take)
}

/// Leaf recorder for the entries that time every leaf: the wall
/// reading is taken on every call, the on-core bracket on one call in
/// [`LEAF_SAMPLE_STRIDE`].
///
/// Those entries decline to sample for a reason that is about the wall
/// cadence alone. The observer needs a reading from every leaf to
/// converge on a workload's shape within the first few iterations, and
/// a strided sample never accumulates enough flushes for
/// auto-migration to fire on small-N workloads. That argument says
/// nothing about how often a leaf is bracketed, so the two cadences
/// can differ, and without a recorder that lets them differ these
/// entries cannot reach the bracket at all.
///
/// [`record_leaf`] and [`record_leaf_on_core`] differ only by the
/// bracket, which is two thread-clock reads and enters the kernel on
/// Linux and FreeBSD. It is taken only while
/// [`crate::sched::levers::oncore_spread`] is on.
#[inline]
fn record_leaf_bracket_sampled<F: FnOnce() -> R, R>(
    site: Option<crate::sched::call_site::SiteRef>,
    items: usize,
    body: F,
) -> R {
    thread_local! {
        static BRACKET_TICK: std::cell::Cell<u32> = const { std::cell::Cell::new(0) };
    }
    // The switch is read first and the counter only advances behind it.
    // Both orders pick the same leaves, but the counter is a
    // thread-local read, modify and write on every leaf of these
    // entries, and with the switch off that is work the path did not
    // used to do. Reading the switch first leaves one cached bool on
    // the default path.
    if !crate::sched::levers::oncore_spread() {
        return record_leaf(site, items, body);
    }
    let bracket = BRACKET_TICK.with(|c| {
        let (next, take) = advance_bracket_tick(c.get());
        c.set(next);
        take
    });
    if bracket {
        record_leaf_on_core(site, items, body)
    } else {
        record_leaf(site, items, body)
    }
}

/// Direction B: continuation-steal lazy bisect.
///
/// Splits only when this worker observes that someone has stolen from
/// its own deque since the last check. Reads
/// `WorkerCtx::stats::times_stolen_from` as a Relaxed atomic - a
/// counter incremented by thieves at the steal site (see
/// `arena_local.rs::WorkerCtx::find_work` and the worker_loop steal
/// path).
///
/// The opening `seed_splits_remaining` calls always split to seed
/// initial fanout to ~`workers` leaves (so workers have something to
/// steal from before the lazy logic kicks in). Subsequent calls
/// consult the steal counter and only split if it advanced.
fn bisect_lazy_steal_driven<T, F>(
    plan: &JobPlan,
    items: &mut [T],
    op: &F,
    seed_splits_remaining: usize,
    last_seen_steals: u64,
    min_leaf: usize,
)
where
    T: Send,
    F: Fn(&mut [T]) + Sync,
{
    if items.len() <= min_leaf {
        record_leaf_sampled(plan.site, items.len(), || op(items));
        return;
    }
    // Decide whether to split: always split during the seed phase
    // (so the deque fills with stealable work), then run lazy.
    let ctx_ptr = crate::sched::arena_local::current_worker_ctx();
    let (should_split, new_last_seen) = if seed_splits_remaining > 0 {
        (true, last_seen_steals)
    } else if ctx_ptr.is_null() {
        // External caller path: outside any worker, no ctx to read.
        // Split eagerly until execution lands inside a worker via
        // external_dispatch's wrapper job.
        (true, last_seen_steals)
    } else {
        // SAFETY: ctx_ptr set by worker_loop on this thread; valid
        // until worker_loop returns.
        let ctx = unsafe { &*ctx_ptr };
        let cur = ctx
            .stats
            .times_stolen_from
            .load(core::sync::atomic::Ordering::Relaxed);
        if cur != last_seen_steals {
            (true, cur)
        } else {
            (false, last_seen_steals)
        }
    };

    if !should_split {
        // No steal pressure observed, so the entire remaining slice
        // runs inline as one leaf rather than splitting further.
        record_leaf_sampled(plan.site, items.len(), || op(items));
        return;
    }
    let mid = items.len() / 2;
    let (left, right) = items.split_at_mut(mid);
    let next_seed = seed_splits_remaining.saturating_sub(1);
    join_context(
        plan,
        |_| bisect_lazy_steal_driven(plan, left, op, next_seed, new_last_seen, min_leaf),
        |_| bisect_lazy_steal_driven(plan, right, op, next_seed, new_last_seen, min_leaf),
    );
}

/// Apply `op` to every chunk-triple of `(out, a, b)` in parallel by
/// recursively bisecting the three slices in lockstep. Each leaf
/// chunk is processed serially by `op(out_c, a_c, b_c)`, which is
/// the right granule for SIMD slice kernels (mul_slice, add_slice,
/// sub_slice in the consumer crate's SIMD primitives).
///
/// All three slices must have the same length (asserted). The
/// bisect always cuts at the midpoint so the three sub-slices
/// stay aligned by index.
///
/// Same SLAW splitter as [`for_each_chunk`]: budget halves per
/// level; replenishes to `max_budget` on observed steal.
#[track_caller]
pub fn for_each_chunk_triple<T1, T2, T3, F>(
    plan: &JobPlan,
    out: &mut [T1],
    a: &[T2],
    b: &[T3],
    op: F,
)
where
    T1: Send,
    T2: Send + Sync,
    T3: Send + Sync,
    F: Fn(&mut [T1], &[T2], &[T3]) + Sync,
{
    let leaf = adaptive_min_leaf(plan, MIN_LEAF_ITEMS);
    for_each_chunk_triple_min_leaf(plan, out, a, b, leaf, op)
}

/// Leaf width from the Tiny-Tasks model over `n` items: `n` items in
/// `c` chunks, rounded up so `c` chunks still cover `n`.
///
/// `None` unless the plan carries a `task_overhead_ns` and a per-item
/// cost the caller stated or the entry probe measured. A classifier
/// default for the profile does not count as stated.
///
/// Both hints are checked before the worker count is resolved, because
/// resolving it reads the SMT variance decision and starts the arena,
/// and a plan carrying neither hint must pay neither.
///
/// Divides by the workers that run rather than the pool's width: the
/// model's parallelism term is how many workers split the work at
/// once, and the SMT siblings are parked unless this plan wakes them.
fn model_leaf(plan: &JobPlan, n: usize) -> Option<usize> {
    if !plan.estimated_per_item_ns_explicit || plan.task_overhead_ns.is_none() {
        report_unreachable_model(plan, n);
        return None;
    }
    let chunks = plan.optimal_chunk_count_for(plan.resolved_workers(), n)? as usize;
    Some(n.div_ceil(chunks.max(1)))
}

/// Report a plan that declared a per-task overhead and still could not
/// reach the model, which means the overhead bought nothing.
///
/// A per-item cost without an overhead is not reported: that cost
/// steers the tier pick, the collapse decision, the leaf floor and the
/// seed depth on its own, so the plan is doing work the model is not
/// party to. Only the overhead is inert alone, and only the walk can
/// tell whether the probe supplied the missing half.
///
/// One line per distinct shape, so a sweep prints once per size rather
/// than once per dispatch.
fn report_unreachable_model(plan: &JobPlan, n: usize) {
    if plan.task_overhead_ns.is_none() || !model_reporting() {
        return;
    }
    static LAST_SHAPE: core::sync::atomic::AtomicU64 = core::sync::atomic::AtomicU64::new(u64::MAX);
    let key = ((n as u64) << 32) ^ u64::from(plan.task_overhead_ns.unwrap_or(0));
    if LAST_SHAPE.swap(key, core::sync::atomic::Ordering::Relaxed) != key {
        notice!(
            "tiny-tasks model not consulted: items {n} task_overhead_ns {:?} \
             estimated_per_item_ns {:?} stated {}. The model needs a per-item \
             cost the caller stated or the entry probe measured, so the \
             overhead sized nothing and the split used the caller's min_leaf.",
            plan.task_overhead_ns, plan.estimated_per_item_ns, plan.estimated_per_item_ns_explicit
        );
    }
}

/// Whether to report a plan whose Tiny-Tasks hints could not be used.
/// Off unless `FLYNNEL_MODEL_REPORT` is set, because a plan that never
/// asked for the model is the common case and is not a defect.
fn model_reporting() -> bool {
    static ON: std::sync::OnceLock<bool> = std::sync::OnceLock::new();
    *ON.get_or_init(|| std::env::var_os("FLYNNEL_MODEL_REPORT").is_some())
}

/// Same as [`for_each_chunk_triple`] but the recursion floor is
/// caller-supplied. At small N (say n=1k with the default
/// `MIN_LEAF_ITEMS=256`), the bisect floor caps the chunk count at
/// `n / MIN_LEAF_ITEMS = 4` leaves - only 4-way parallelism on a
/// 16-thread host. Slice ops can pass a smaller floor (e.g.,
/// `PAR_MIN_ELEMS = 64`) so the SLAW budget (`workers * multiplier`)
/// caps chunk count instead, matching rayon's per-chunk fanout at
/// small N.
///
/// `min_leaf` is a floor on leaf size, not a granularity leaves are
/// multiples of: the recursion stops at `min_leaf` items, so
/// `out.len() / min_leaf` caps how many leaves can exist at any worker
/// count.
///
/// With both `estimated_per_item_ns` and `task_overhead_ns` set, the
/// width comes from [`JobPlan::optimal_chunk_count_for`] and
/// `min_leaf` is the floor under it. With either unset, the floor is
/// the whole answer.
#[track_caller]
pub fn for_each_chunk_triple_min_leaf<T1, T2, T3, F>(
    plan: &JobPlan,
    out: &mut [T1],
    a: &[T2],
    b: &[T3],
    min_leaf: usize,
    op: F,
)
where
    T1: Send,
    T2: Send + Sync,
    T3: Send + Sync,
    F: Fn(&mut [T1], &[T2], &[T3]) + Sync,
{
    assert_eq!(out.len(), a.len(), "for_each_chunk_triple: out.len() != a.len()");
    assert_eq!(out.len(), b.len(), "for_each_chunk_triple: out.len() != b.len()");
    let n = out.len();
    if n == 0 {
        return;
    }
    // Per-call-site identity from the caller's source location
    // (track_caller chain); an outer attachment wins.
    let (plan_owned, routing_arm) = plan
        .with_site_if_none(crate::sched::call_site::caller_site())
        .apply_site_class_measured();
    let _routing = RecordRoutingArm::start(plan_owned.site, routing_arm);
    let plan = &plan_owned;
    // Capped to one worker, or under the dispatch floor by the
    // caller's own estimate: the body runs here, as in
    // for_each_chunk.
    if runs_on_caller(plan, n) {
        run_on_caller(plan, out.len(), || op(out, a, b));
        return;
    }
    let leaf = min_leaf.max(1);
    let workers = plan.effective_workers(global_local_arena().total_workers());
    // The caller's floor bounds the model's width from below.
    let dispatch_leaf = model_leaf(plan, n).map_or(leaf, |derived| leaf.max(derived));
    let max_budget = workers.saturating_mul(plan.effective_leaves_per_worker()).max(1);
    bisect_triple(plan, out, a, b, &op, dispatch_leaf, max_budget, max_budget, false);
}

#[allow(clippy::too_many_arguments)]
fn bisect_triple<T1, T2, T3, F>(
    plan: &JobPlan,
    out: &mut [T1],
    a: &[T2],
    b: &[T3],
    op: &F,
    min_leaf: usize,
    splits: usize,
    max_budget: usize,
    migrated: bool,
)
where
    T1: Send,
    T2: Send + Sync,
    T3: Send + Sync,
    F: Fn(&mut [T1], &[T2], &[T3]) + Sync,
{
    if out.len() <= min_leaf {
        record_leaf_bracket_sampled(plan.site, out.len(), || op(out, a, b));
        return;
    }
    let cur_splits = if migrated { max_budget } else { splits };
    // Leaf at `<= 1`, not `== 0`. See `bisect` for the rationale:
    // splitting at cur_splits=1 emits two children each holding
    // 0 budget, both immediately leaf, doubling leaf count.
    if cur_splits <= 1 {
        record_leaf_bracket_sampled(plan.site, out.len(), || op(out, a, b));
        return;
    }
    let mid = out.len() >> 1;
    let (out_lo, out_hi) = out.split_at_mut(mid);
    let (a_lo, a_hi) = a.split_at(mid);
    let (b_lo, b_hi) = b.split_at(mid);
    let half = cur_splits >> 1;
    join_context(
        plan,
        |left_inj| bisect_triple(plan, out_lo, a_lo, b_lo, op, min_leaf, half, max_budget, left_inj),
        |right_stolen| bisect_triple(plan, out_hi, a_hi, b_hi, op, min_leaf, half, max_budget, right_stolen),
    );
}

/// Apply `op(start_index, chunk)` to every chunk of `items` in
/// parallel by recursively bisecting the slice. The closure receives
/// the global starting index of the chunk so per-element work that
/// needs the absolute index (matmul row/col decode, SpMV row id,
/// LU trailing-update row id) can compute it from
/// `start + i_in_chunk`.
///
/// Same SLAW splitter as [`for_each_chunk`]: budget halves per
/// level; replenishes to `max_budget` on observed steal.
#[track_caller]
pub fn for_each_chunk_indexed<T, F>(plan: &JobPlan, items: &mut [T], op: F)
where
    T: Send,
    F: Fn(usize, &mut [T]) + Sync,
{
    let leaf = adaptive_min_leaf(plan, MIN_LEAF_ITEMS);
    for_each_chunk_indexed_min_leaf(plan, items, leaf, op)
}

/// One-time measurement of the `Instant` bracket cost on this host
/// (Windows QPC measured ~91 ns on Zen+ R7 2700; rdtsc-backed
/// clocks run ~20-40 ns). Measured at first use: 4096 empty
/// brackets, ~0.4 ms once per process.
fn timer_bracket_ns() -> u64 {
    static BRACKET_NS: std::sync::OnceLock<u64> = std::sync::OnceLock::new();
    *BRACKET_NS.get_or_init(|| {
        let iters = 4_096u32;
        let t0 = std::time::Instant::now();
        for _ in 0..iters {
            std::hint::black_box(std::time::Instant::now().elapsed());
        }
        ((t0.elapsed().as_nanos() as u64) / iters as u64).max(1)
    })
}

/// Trust floor for a single-item probe measurement: 20x the
/// measured timer bracket, bounding the bracket's contribution to
/// the reading at ~5%.
fn probe_trust_floor_ns() -> u64 {
    timer_bracket_ns().saturating_mul(20)
}

/// Probe size for the min_leaf-style entry probes, floored at the
/// caller's min_leaf.
const PROBE_SIZE_FLOOR: usize = 16;
/// Item count from which the min_leaf-style entries probe
/// `max(PROBE_SIZE_FLOOR, min_leaf)` items.
const PROBE_MIN_N: usize = 64;
/// Item count from which a one-min_leaf-unit probe fires (subject
/// to `n >= 2 * min_leaf` so the tail stays non-empty). Swept at
/// 2/4/8/16 on both bench hosts (PortBound heavy ~2-3ms items,
/// min_leaf 1, cold fresh-process sites, median of 5): 2 wins or
/// ties every cell. Zen+ R7 2700: n=3 heavy 5.70ms vs 8.19ms at
/// gate 4, n=4 heavy 5.70ms vs 11.60ms at gate 8. AVX-512 host:
/// n=3 heavy 3.80ms vs 5.06ms, n=8 heavy 4.0ms vs 13.4ms at gate
/// 16. Light items cost 4-16us at every gate on both hosts.
const PROBE_SMALL_MIN_N: usize = 2;

/// Same as [`for_each_chunk_indexed`] but the recursion floor is
/// supplied by the caller instead of [`MIN_LEAF_ITEMS`]. Use this
/// for coarse-grained workloads where each item carries enough work
/// that bisecting all the way to single-element leaves is justified
/// (e.g., per-row dense-GEMM strips where one strip is a full
/// `gemm_via_schemes` call, and the total strip count is small
/// relative to MIN_LEAF_ITEMS).
///
/// `min_leaf >= 1`. Setting `min_leaf == 1` produces one job per
/// item - the right choice when the body is heavy and the total
/// item count equals the worker count.
///
/// `min_leaf` is a floor on leaf size, not a granularity leaves are
/// multiples of: the recursion stops at `min_leaf` items, so
/// `items.len() / min_leaf` caps how many leaves can exist at any
/// worker count. Tying it to a cache blocking width caps the
/// parallelism at `items.len() / width`.
///
/// With both `estimated_per_item_ns` and `task_overhead_ns` set, the
/// width comes from [`JobPlan::optimal_chunk_count_for`] and
/// `min_leaf` is the floor under it. A cost the entry probe measured
/// counts, so `task_overhead_ns` alone reaches the model on a plan
/// that lets the probe run. With neither, or with only the per-item
/// cost, the floor is the whole answer.
///
/// Probe sizing shared by the min_leaf-style entries (this and
/// [`collect_indexed`]): at `n >= PROBE_MIN_N` the probe is
/// `max(PROBE_SIZE_FLOOR, min_leaf)` items; at
/// `PROBE_SMALL_MIN_N.max(2 * min_leaf) <= n < PROBE_MIN_N` it is
/// one min_leaf unit, which is a complete work unit by the
/// min_leaf contract and leaves a non-empty tail. Heavy measured
/// cost promotes the tail to the pool; light measured cost
/// reclassifies to FineGrain and the tail runs inline.
#[track_caller]
pub fn for_each_chunk_indexed_min_leaf<T, F>(
    plan: &JobPlan,
    items: &mut [T],
    min_leaf: usize,
    op: F,
)
where
    T: Send,
    F: Fn(usize, &mut [T]) + Sync,
{
    let n = items.len();
    if n == 0 {
        return;
    }
    // Per-call-site identity from the caller's source location
    // (track_caller chain); an outer attachment wins.
    let (plan_owned, routing_arm) = plan
        .with_site_if_none(crate::sched::call_site::caller_site())
        .apply_site_class_measured();
    let _routing = RecordRoutingArm::start(plan_owned.site, routing_arm);
    let plan = &plan_owned;
    let _flush_on_exit = FlushLeafStatsOnExit;
    // Capped to one worker, or under the dispatch floor by the
    // caller's own estimate: the body runs here, as in
    // for_each_chunk.
    if runs_on_caller(plan, n) {
        run_on_caller(plan, items.len(), || op(0, items));
        return;
    }
    let leaf = min_leaf.max(1);
    let workers = plan.effective_workers(global_local_arena().total_workers());

    // Probe-and-decide: when the caller hasn't supplied a per-item
    // cost estimate and the workload is big enough to amortize a
    // probe, run a tiny prefix through the closure to measure the
    // actual per-item cost. The measurement feeds the static
    // classifier (via JobPlan::with_estimated_per_item_ns) so the
    // bulk dispatch already routes to the right WorkloadClass on
    // the first call -- no caller hint, no observer warm-up
    // iterations required.
    //
    // The probe runs on items[0..probe_n] with start_idx=0,
    // executes the work for those items (it isn't wasted), and the
    // bulk dispatch then operates on items[probe_n..] with
    // start_idx=probe_n to preserve absolute-index semantics for
    // the caller's closure.
    //
    // probe_n must be at least min_leaf. The caller's closure body
    // is sized around the leaf granularity: a stride-w row kernel,
    // a chunked accumulator over min_leaf-sized blocks, etc. A
    // probe shorter than min_leaf would feed the closure a
    // half-complete unit and produce a near-zero per-item estimate,
    // which would mis-classify the workload into FineGrain /
    // Streaming when the actual work is much heavier.
    let plan_holder: JobPlan;
    // Probe path consults estimated_per_item_ns_explicit (not just
    // is_none) so classifier-default 12/50/600 ns doesn't suppress
    // measurement for JobPlan::new callers. Same fix as for_each_chunk.
    // use_smt gate matches for_each_chunk's probe: when the
    // classifier already signals heavy per-item work, skip the
    // serial probe so every item starts concurrently (racing and
    // fan-out callers rely on this).
    let cost_authoritative = plan.estimated_per_item_ns_explicit;
    let probe_quota: usize = if cost_authoritative || plan.use_smt {
        0
    } else if n >= PROBE_MIN_N.max(leaf) {
        PROBE_SIZE_FLOOR.max(leaf).min(n)
    } else if n >= PROBE_SMALL_MIN_N.max(2 * leaf) {
        leaf
    } else {
        0
    };
    let (effective_plan, items_to_dispatch, start_offset): (&JobPlan, &mut [T], usize) =
        if probe_quota > 0 {
            // Staged: one leaf unit first. A unit reading at or
            // above the trust floor is reliable on its own and the
            // rest of the quota skips the serial detour; a light
            // unit is bracket-noise-dominated, so the remaining
            // quota is probed for a reliable average.
            let (first, rest) = items.split_at_mut(leaf.min(probe_quota));
            let first_n = first.len();
            let t0 = std::time::Instant::now();
            record_leaf(plan.site, first_n, || op(0, first));
            let first_ns = t0.elapsed().as_nanos() as u64;
            let (probe_ns, probed, tail): (u64, usize, &mut [T]) =
                if first_ns >= probe_trust_floor_ns() || probe_quota <= first_n {
                    (first_ns, first_n, rest)
                } else {
                    let extra = (probe_quota - first_n).min(rest.len().saturating_sub(1));
                    if extra == 0 {
                        (first_ns, first_n, rest)
                    } else {
                        let (more, rest2) = rest.split_at_mut(extra);
                        let t2 = std::time::Instant::now();
                        record_leaf(plan.site, more.len(), || op(first_n, more));
                        let more_ns = t2.elapsed().as_nanos() as u64;
                        (first_ns + more_ns, first_n + extra, rest2)
                    }
                };
            let per_item_ns = (probe_ns.max(1) / probed as u64).min(u32::MAX as u64) as u32;
            plan_holder = (*plan).with_estimated_per_item_ns(per_item_ns);
            (&plan_holder, tail, probed)
        } else {
            (plan, items, 0)
        };

    if items_to_dispatch.is_empty() {
        return;
    }

    // The caller's floor bounds the model's width from below. Read
    // from `effective_plan` over the items left after the probe, so a
    // probed cost counts and the width sizes the remaining run.
    let dispatch_leaf = model_leaf(effective_plan, items_to_dispatch.len())
        .map_or(leaf, |derived| leaf.max(derived));

    // Default: continuation-steal-lazy bisect with absolute-index
    // passthrough. Uses adaptive_seed_depth so heavy items get more
    // eager splits for load-balance headroom; light items get
    // log2(workers) to minimize dispatch overhead.
    if effective_plan.bisect_variant.is_none() {
        let seed_depth = adaptive_seed_depth(effective_plan, items_to_dispatch.len(), workers);
        bisect_lazy_steal_driven_indexed(
            effective_plan, items_to_dispatch, start_offset, &op, dispatch_leaf, seed_depth, 0,
        );
        return;
    }
    // Pinned variant: fall back to the eager all-the-way-to-min_leaf
    // bisect_indexed for callers who explicitly opted into a fixed
    // split shape.
    let max_budget = workers
        .saturating_mul(effective_plan.effective_leaves_per_worker())
        .max(1);
    bisect_indexed(
        effective_plan,
        items_to_dispatch,
        start_offset,
        &op,
        dispatch_leaf,
        max_budget,
        max_budget,
        false,
    );
}

/// Call `f(i)` for every `i` in `0..n` in parallel, each index
/// exactly once, with no slice to mutate: the read-only and
/// side-effect shapes (declare a cell per id through a shared
/// resolver, fault a page per index, fill a row of a buffer the
/// closure addresses itself). `min_leaf` is the bisect floor in
/// indices, as in [`for_each_chunk_indexed_min_leaf`], which this
/// runs over a zero-sized slice of length `n` so the probe, the
/// per-call-site statistics and the lazy-steal bisect all apply
/// unchanged. Nothing is allocated for the slice.
///
/// `min_leaf` is a floor on leaf size, so `n / min_leaf` caps the leaf
/// count, and the Tiny-Tasks sizing on
/// [`for_each_chunk_indexed_min_leaf`] applies here too.
///
/// For a per-chunk body over a read-only slice use
/// [`for_each_chunk_ref`].
#[track_caller]
pub fn for_each_indexed<F>(plan: &JobPlan, n: usize, min_leaf: usize, f: F)
where
    F: Fn(usize) + Sync,
{
    if n == 0 {
        return;
    }
    // A Vec of unit is length without storage: the bisect halves
    // lengths and hands out absolute starts, which is all the
    // indexed walk needs.
    let mut units: Vec<()> = vec![(); n];
    for_each_chunk_indexed_min_leaf(plan, &mut units, min_leaf, |start, chunk| {
        for i in start..start + chunk.len() {
            f(i);
        }
    });
}

/// Call `f(start, chunk)` over consecutive read-only chunks of
/// `items` in parallel: `chunk` is `items[start..start + len]` with
/// `len == min_leaf` except on the trailing chunk, and the chunks
/// tile the slice exactly once. The read-only counterpart of
/// [`for_each_chunk_indexed_min_leaf`] for a body that needs a fixed
/// batch width (a tile kernel, a resolver that takes a run of ids)
/// and never writes the input. Built on [`for_each_indexed`] over
/// the chunk count, one chunk per index.
///
/// `min_leaf` is the chunk width here, not a bisect bound as in the
/// sibling helpers: the leaf count is
/// `items.len().div_ceil(min_leaf)`.
#[track_caller]
pub fn for_each_chunk_ref<T, F>(plan: &JobPlan, items: &[T], min_leaf: usize, f: F)
where
    T: Sync,
    F: Fn(usize, &[T]) + Sync,
{
    let n = items.len();
    if n == 0 {
        return;
    }
    let width = min_leaf.max(1);
    let n_chunks = n.div_ceil(width);
    for_each_indexed(plan, n_chunks, 1, |i| {
        let lo = i * width;
        let hi = (lo + width).min(n);
        f(lo, &items[lo..hi]);
    });
}

/// Continuation-steal-lazy bisect with absolute-index passthrough.
/// Mirrors [`bisect_lazy_steal_driven`] but threads the start
/// offset through so leaves know their absolute slot in the
/// original input. Used by [`for_each_chunk_indexed_min_leaf`] as
/// the default path; closes the per-dispatch overhead gap between
/// the indexed and non-indexed paths at fine-grain leaves.
#[allow(clippy::too_many_arguments)]
fn bisect_lazy_steal_driven_indexed<T, F>(
    plan: &JobPlan,
    items: &mut [T],
    start: usize,
    op: &F,
    min_leaf: usize,
    seed_splits_remaining: usize,
    last_seen_steals: u64,
)
where
    T: Send,
    F: Fn(usize, &mut [T]) + Sync,
{
    if items.len() <= min_leaf {
        // Every leaf is wall-timed here: the closing-loop auto-classifier
        // observer converges on the workload's shape (mean_ns + cv^2)
        // within the first few iterations only if it sees them all, and
        // on a small-N workload such as a 16-chunk grep a strided sample
        // never accumulates enough flushes for auto-migration to fire.
        // The on-core bracket rides a stride of its own inside the
        // recorder, which that requirement does not constrain.
        record_leaf_bracket_sampled(plan.site, items.len(), || op(start, items));
        return;
    }
    let ctx_ptr = crate::sched::arena_local::current_worker_ctx();
    let (should_split, new_last_seen) = if seed_splits_remaining > 0 {
        (true, last_seen_steals)
    } else if ctx_ptr.is_null() {
        // External caller path: outside any worker, no ctx to read.
        // Split eagerly until execution lands inside a worker via
        // external_dispatch's wrapper job.
        (true, last_seen_steals)
    } else {
        // SAFETY: ctx_ptr set by worker_loop on this thread; valid
        // until worker_loop returns.
        let ctx = unsafe { &*ctx_ptr };
        let cur = ctx
            .stats
            .times_stolen_from
            .load(core::sync::atomic::Ordering::Relaxed);
        if cur != last_seen_steals {
            (true, cur)
        } else {
            (false, last_seen_steals)
        }
    };

    if !should_split {
        // No steal pressure observed, so the entire remaining slice
        // runs inline as one leaf rather than splitting further. This is the rayon continuation-stealing
        // pattern: only fork further when somebody is starving.
        record_leaf_bracket_sampled(plan.site, items.len(), || op(start, items));
        return;
    }
    let mid = items.len() / 2;
    let (left, right) = items.split_at_mut(mid);
    let right_start = start + mid;
    let next_seed = seed_splits_remaining.saturating_sub(1);
    join_context(
        plan,
        |_| bisect_lazy_steal_driven_indexed(plan, left, start, op, min_leaf, next_seed, new_last_seen),
        |_| bisect_lazy_steal_driven_indexed(plan, right, right_start, op, min_leaf, next_seed, new_last_seen),
    );
}

#[allow(clippy::too_many_arguments)]
fn bisect_indexed<T, F>(
    plan: &JobPlan,
    items: &mut [T],
    start: usize,
    op: &F,
    min_leaf: usize,
    splits: usize,
    max_budget: usize,
    migrated: bool,
)
where
    T: Send,
    F: Fn(usize, &mut [T]) + Sync,
{
    if items.len() <= min_leaf {
        record_leaf_bracket_sampled(plan.site, items.len(), || op(start, items));
        return;
    }
    let cur_splits = if migrated { max_budget } else { splits };
    // Leaf at `<= 1` (see `bisect` for the doubling-bug rationale).
    if cur_splits <= 1 {
        record_leaf_bracket_sampled(plan.site, items.len(), || op(start, items));
        return;
    }
    let mid = items.len() >> 1;
    let (left, right) = items.split_at_mut(mid);
    let right_start = start + mid;
    let half = cur_splits >> 1;
    join_context(
        plan,
        |left_inj| {
            bisect_indexed(plan, left, start, op, min_leaf, half, max_budget, left_inj)
        },
        |right_stolen| {
            bisect_indexed(
                plan, right, right_start, op, min_leaf, half, max_budget, right_stolen,
            )
        },
    );
}

/// Parallel indexed collect: compute `f(idx)` for `idx in 0..n` in
/// parallel and return the result `Vec<R>`. Matches the cost profile
/// of `rayon`'s `(0..n).into_par_iter().map(f).collect()` by
/// allocating the result buffer without zero-init - uses
/// `MaybeUninit<R>` + ptr writes inside the SLAW bisect. This is the
/// right helper for indexed-collect ops (matmul, spmv, LU row update,
/// Jacobi rotation, eigenvalue rotation, FFT butterfly, block-sparse
/// GEMM tiles) where the pre-zero-init cost on the indexed slice
/// path dominated for small-to-mid N.
///
/// `min_leaf` controls the bisect floor:
/// - `1` for heavy-per-item work (matmul O(k), spmv O(nnz_per_row),
///   row updates) - the SLAW budget (`workers * multiplier`) is the
///   right cap, not the leaf floor.
/// - `MIN_LEAF_ITEMS` (~256) for fine-grain work where the floor
///   amortizes join overhead.
///
/// It is a floor on leaf size, not a granularity leaves are multiples
/// of: the recursion stops at `min_leaf` indices, so `n / min_leaf`
/// caps the leaf count at any worker count.
///
/// # Safety contract
///
/// `R` is dropped only on the success path. If `f` panics mid-bisect,
/// the partial result Vec leaks (no double-drop). This is sound for
/// `Copy` / POD result types (FpN, tuples of FpN, etc.); for
/// resource-holding `R` types, callers should ensure panics don't
/// occur or use a panic-safe wrapper.
#[track_caller]
pub fn collect_indexed<R, F>(plan: &JobPlan, n: usize, min_leaf: usize, f: F) -> Vec<R>
where
    R: Send,
    F: Fn(usize) -> R + Sync,
{
    if n == 0 {
        return Vec::new();
    }
    // Per-call-site identity from the caller's source location
    // (track_caller chain). An outer entry (heartbeat /
    // token-bucket / tiny-tasks) that already attached its own
    // site wins via with_site_if_none.
    let (plan_owned, routing_arm) = plan
        .with_site_if_none(crate::sched::call_site::caller_site())
        .apply_site_class_measured();
    let _routing = RecordRoutingArm::start(plan_owned.site, routing_arm);
    let plan = &plan_owned;
    let _flush_on_exit = FlushLeafStatsOnExit;
    if capped_to_caller(plan) {
        return run_on_caller(plan, n, || (0..n).map(&f).collect());
    }

    let leaf = min_leaf.max(1);
    let workers = plan.effective_workers(global_local_arena().total_workers());
    let max_budget = workers.saturating_mul(plan.effective_leaves_per_worker()).max(1);

    // Allocate buffer of `MaybeUninit<R>` with exact capacity, NO
    // zero-init. We promise to write every slot via the probe +
    // collect_inner before transmuting back to `Vec<R>`.
    let mut buf: Vec<std::mem::MaybeUninit<R>> = Vec::with_capacity(n);
    // SAFETY: `MaybeUninit<R>` does not require initialization.
    // `set_len` is valid for any value <= capacity. After the probe
    // and `collect_inner` fill every slot, we transmute to `Vec<R>`.
    unsafe { buf.set_len(n); }

    // Entry probe, same sizing contract as
    // [`for_each_chunk_indexed_min_leaf`]: measures per-item cost
    // when the caller supplied none so the bulk dispatch routes on
    // a measurement instead of a classifier guess.
    // Same use_smt gate as the for_each_chunk probes: a
    // classifier-signaled heavy plan skips the serial probe so
    // every item starts concurrently (race_any / race_quorum
    // dispatch through here).
    let cost_authoritative = plan.estimated_per_item_ns_explicit;
    let probe_quota: usize = if cost_authoritative || plan.use_smt {
        0
    } else if n >= PROBE_MIN_N.max(leaf) {
        PROBE_SIZE_FLOOR.max(leaf).min(n)
    } else if n >= PROBE_SMALL_MIN_N.max(2 * leaf) {
        leaf
    } else {
        0
    };
    let plan_holder: JobPlan;
    let (effective_plan, start_offset): (&JobPlan, usize) = if probe_quota > 0 {
        let base = buf.as_mut_ptr() as *mut R;
        // Staged, matching for_each_chunk_indexed_min_leaf: one
        // leaf unit first; a trust-floor reading skips the rest of
        // the quota, a light reading probes the remainder for a
        // reliable average. Every probed slot is written exactly
        // once; the bulk dispatch below starts past them.
        let first_n = leaf.min(probe_quota);
        let t0 = std::time::Instant::now();
        record_leaf(plan.site, first_n, || {
            for i in 0..first_n {
                // SAFETY: `i < first_n <= n = buf.len()`; slots
                // below `probed` are written only here and in the
                // extra-probe loop over a disjoint range.
                unsafe {
                    base.add(i).write(f(i));
                }
            }
        });
        let first_ns = t0.elapsed().as_nanos() as u64;
        let (probe_ns, probed): (u64, usize) =
            if first_ns >= probe_trust_floor_ns() || probe_quota <= first_n {
                (first_ns, first_n)
            } else {
                let extra = (probe_quota - first_n).min(n - first_n - 1);
                if extra == 0 {
                    (first_ns, first_n)
                } else {
                    let t2 = std::time::Instant::now();
                    record_leaf(plan.site, extra, || {
                        for i in first_n..first_n + extra {
                            // SAFETY: `i < first_n + extra < n`;
                            // disjoint from the first-unit range.
                            unsafe {
                                base.add(i).write(f(i));
                            }
                        }
                    });
                    let more_ns = t2.elapsed().as_nanos() as u64;
                    (first_ns + more_ns, first_n + extra)
                }
            };
        let per_item_ns = (probe_ns.max(1) / probed as u64).min(u32::MAX as u64) as u32;
        plan_holder = (*plan).with_estimated_per_item_ns(per_item_ns);
        (&plan_holder, probed)
    } else {
        (plan, 0)
    };

    if start_offset < n {
        collect_inner(
            effective_plan,
            &mut buf[start_offset..],
            start_offset,
            &f,
            leaf,
            max_budget,
            max_budget,
            false,
        );
    }

    let mut buf = std::mem::ManuallyDrop::new(buf);
    // SAFETY: every slot was written exactly once by
    // `collect_inner`. `MaybeUninit<R>` has the same layout as `R`,
    // so reconstructing the Vec with the same allocation is sound.
    unsafe { Vec::from_raw_parts(buf.as_mut_ptr() as *mut R, n, buf.capacity()) }
}

#[allow(clippy::too_many_arguments)]
fn collect_inner<R, F>(
    plan: &JobPlan,
    items: &mut [std::mem::MaybeUninit<R>],
    start: usize,
    f: &F,
    min_leaf: usize,
    splits: usize,
    max_budget: usize,
    migrated: bool,
)
where
    R: Send,
    F: Fn(usize) -> R + Sync,
{
    // Leaf fill via raw pointer arithmetic, matching rayon's
    // `CollectConsumer`-`consume_iter` byte-for-byte:
    // `self.start.0.add(i).write(item)`. This avoids the
    // `MaybeUninit::write` API which (per MICRO 2024 / rayon-demo
    // measurements) can occasionally inhibit LLVM auto-vec via
    // the `&mut MaybeUninit<R>` reference dance.
    #[inline(always)]
    fn fill_leaf<R, F>(items: &mut [std::mem::MaybeUninit<R>], start: usize, f: &F)
    where
        F: Fn(usize) -> R,
    {
        let len = items.len();
        let base = items.as_mut_ptr() as *mut R;
        for i in 0..len {
            // SAFETY: `base.add(i)` is in-bounds (`i < len`) and
            // points to a `MaybeUninit<R>` slot, which is valid
            // for writing an `R` by transmute (same layout).
            // Caller's contract: every slot is written exactly
            // once before `collect_indexed` transmutes back to
            // `Vec<R>`.
            unsafe { base.add(i).write(f(start + i)); }
        }
    }

    if items.len() <= min_leaf {
        record_leaf(plan.site, items.len(), || fill_leaf(items, start, f));
        return;
    }
    let cur_splits = if migrated { max_budget } else { splits };
    // Leaf at `<= 1` (see `bisect` for the doubling-bug rationale).
    if cur_splits <= 1 {
        record_leaf(plan.site, items.len(), || fill_leaf(items, start, f));
        return;
    }
    let mid = items.len() >> 1;
    let (left, right) = items.split_at_mut(mid);
    let right_start = start + mid;
    let half = cur_splits >> 1;
    join_context(
        plan,
        |left_inj| {
            collect_inner(plan, left, start, f, min_leaf, half, max_budget, left_inj)
        },
        |right_stolen| {
            collect_inner(
                plan, right, right_start, f, min_leaf, half, max_budget, right_stolen,
            )
        },
    );
}

/// Heartbeat-gate threshold for sites with **no cv^2 evidence
/// yet**: below this item count a fresh site falls through to
/// plain SLAW [`collect_indexed`] with `min_leaf=1`; at or above
/// it, heartbeat runs. Once the site has variance evidence the
/// cv^2 routing in [`collect_indexed_heartbeat`] replaces this
/// gate entirely (uniform leaves force SLAW at any n; irregular
/// leaves A/B via policy arms from [`HEARTBEAT_MIN_ITEMS`]).
/// Uniform-per-iter workloads (matmul, slice ops) regress 3-4x
/// under heartbeat at small/mid N, so the cold default stays
/// conservative; heartbeat's value is **heterogeneous** work
/// where per-iter cost varies by 10x+ - the rdtsc-driven
/// promotion adapts to actual elapsed time without prior
/// knowledge of the cost distribution.
const HEARTBEAT_GATE_ITEMS: usize = 100_000;

/// Parallel indexed collect using **heartbeat scheduling** (Acar,
/// Charguéraud, Guatto, Rainey, Sieczkowski, "Heartbeat Scheduling:
/// Provable Efficiency for Nested Parallelism", PLDI 2018, §4
/// "Native support for parallel loops"). On each rdtsc tick that
/// crosses the quantum, [`HEARTBEAT_NS`] in this host's counter ticks
/// (see [`heartbeat_ticks`]), the current loop's remaining range
/// is split in half and the upper half is forked via
/// [`crate::sched::join_context`]. Each forked sub-range runs its
/// own polling counter so the promotion tree builds incrementally.
///
/// # When to use
///
/// Call this for workloads whose per-item cost is **clustered or
/// irregular** (front-loaded heavy items, depth-dependent
/// recursion, work-list iteration) when no accurate per-item
/// weight is available. The heartbeat-vs-SLAW choice is then made
/// per call from the call site's learned statistics:
///
/// - Caller set [`JobPlan::estimated_per_item_ns`] explicitly:
///   entry-only static decision - fully serial when the estimated
///   total is below the ~20us heartbeat quantum, plain SLAW
///   [`collect_indexed`] otherwise. Site-learned profile defaults
///   do not trip this gate; only the explicit flag does.
/// - Site cv^2 known and below the calibrated
///   `cv2_high_per_mille` threshold (uniform leaves): SLAW at any
///   `n` - heartbeat loses 3-4x on uniform-per-iter work.
/// - Site cv^2 known and high (irregular leaves): the site's
///   policy arms A/B heartbeat against SLAW from
///   `n >= HEARTBEAT_MIN_ITEMS` (`4_096`), adopt whichever EWMA
///   wins, and re-trial on a fixed cadence.
/// - Site cv^2 unreadable because its mean leaf is under the 256 ns
///   unit the spread is kept in: SLAW at any `n`, decided from that
///   mean.
/// - Site cv^2 unknown (fresh site): legacy item-count gate -
///   SLAW below [`HEARTBEAT_GATE_ITEMS`] (`100_000`), heartbeat
///   at or above it.
///
/// # Alternatives for other shapes
///
/// - Uniform-per-iter workloads (matmul, slice ops, FFT butterfly
///   groups, LU trailing updates, HODLR tile compress): use
///   [`collect_indexed`] - SLAW's static budget is optimal. The
///   cv^2 route above converges to the same choice after a few
///   calls; calling `collect_indexed` directly skips the learning
///   window.
/// - Heterogeneous with known per-item weight (spmv with
///   `nnz_per_row`, etc.): use [`collect_indexed_token_bucket`]
///   which routes via the supplied weight instead of discovering
///   cost via rdtsc polling.
#[track_caller]
pub fn collect_indexed_heartbeat<R, F>(plan: &JobPlan, n: usize, f: F) -> Vec<R>
where
    R: Send,
    F: Fn(usize) -> R + Sync,
{
    if n == 0 {
        return Vec::new();
    }
    // Per-call-site identity from the caller's source location
    // (track_caller chain): heartbeat's serial spans record into
    // this site, and the site's cv^2 + policy arms drive the
    // heartbeat-vs-SLAW choice below.
    let (plan_owned, routing_arm) = plan
        .with_site_if_none(crate::sched::call_site::caller_site())
        .apply_site_class_measured();
    let _routing = RecordRoutingArm::start(plan_owned.site, routing_arm);
    let plan = &plan_owned;
    if capped_to_caller(plan) {
        return run_on_caller(plan, n, || (0..n).map(&f).collect());
    }

    // Plan-estimate gate (entry-only, no rdtsc-polling in hot loop).
    // When the caller supplied an authoritative per-item cost, the
    // dispatcher can compute total estimated cost and make a static
    // decision at entry: serial when below the heartbeat quantum,
    // SLAW-parallel otherwise. Gated on the explicit flag because
    // site-learned profile defaults also populate
    // `estimated_per_item_ns`; those are routing hints and must not
    // preempt the cv^2-driven policy choice below.
    if plan.estimated_per_item_ns_explicit
        && let Some(total_ns) = plan.estimated_total_ns()
    {
        if total_ns < HEARTBEAT_NS {
            // Run fully serial: no scheduler involvement at all.
            return serial_collect_indexed(n, &f);
        }
        // Estimate says heartbeat would fire immediately - skip the
        // serial-prefix-then-handoff dance and go straight to SLAW.
        return collect_indexed(plan, n, 1, f);
    }

    let site = plan.site.expect("attached above");
    let cv2 = site.get().cv2_per_mille();
    let arm = heartbeat_arm(site.get(), cv2, n);

    if arm == crate::sched::call_site::PolicyArm::Default {
        let t0 = std::time::Instant::now();
        let out = collect_indexed(plan, n, 1, f);
        if cv2.is_some() {
            site.get().record_arm(arm, t0.elapsed().as_nanos() as u64);
        }
        return out;
    }

    let t0 = std::time::Instant::now();
    let mut buf: Vec<std::mem::MaybeUninit<R>> = Vec::with_capacity(n);
    // SAFETY: identical contract to `collect_indexed` - every slot
    // is written by `heartbeat_fill` before transmute.
    unsafe { buf.set_len(n); }

    heartbeat_fill(plan, &mut buf[..], 0, &f);

    let mut buf = std::mem::ManuallyDrop::new(buf);
    if cv2.is_some() {
        site.get().record_arm(arm, t0.elapsed().as_nanos() as u64);
    }
    // SAFETY: `heartbeat_fill` writes every slot exactly once;
    // `MaybeUninit<R>` and `R` share layout, so reconstructing
    // the Vec from the raw parts is sound.
    unsafe { Vec::from_raw_parts(buf.as_mut_ptr() as *mut R, n, buf.capacity()) }
}

/// The arm [`collect_indexed_heartbeat`] runs `n` items on at `site`,
/// whose lifetime leaf spread is `cv2`.
///
/// Heartbeat wins on irregular per-item cost (the Acar model's target
/// shape) and loses 3-4x on uniform cost (matmul-shaped work), so the
/// site's observed cv^2 is the routing signal:
///
/// - cv^2 known and low (uniform leaves): SLAW at any n.
/// - cv^2 known and high (irregular leaves): the site's policy arms
///   A/B heartbeat (arm 1) against SLAW (arm 0) from
///   n >= HEARTBEAT_MIN_ITEMS, adopting whichever EWMA wins and
///   re-trialling on a fixed cadence.
/// - cv^2 unreadable because the site's leaves are shorter than the
///   256 ns unit it is kept in: SLAW, the route a low reading takes,
///   decided from the mean leaf the site did measure.
/// - cv^2 unknown (fresh site): the legacy item-count gate.
fn heartbeat_arm(
    site: &crate::sched::call_site::CallSiteState,
    cv2: Option<u64>,
    n: usize,
) -> crate::sched::call_site::PolicyArm {
    use crate::sched::call_site::PolicyArm;
    let cv2_high = crate::sched::adaptive_profile::class_thresholds()
        .cv2_high_per_mille
        .load(std::sync::atomic::Ordering::Relaxed);
    match cv2 {
        Some(c) if c < cv2_high => PolicyArm::Default,
        Some(_) => site.choose_arm(n >= HEARTBEAT_MIN_ITEMS),
        None if site.leaves_under_spread_unit() => PolicyArm::Default,
        None if n < HEARTBEAT_GATE_ITEMS => PolicyArm::Default,
        None => PolicyArm::Alternative,
    }
}

/// Floor for the heartbeat policy arm once a site's cv^2 evidence
/// says the workload is irregular. Well below
/// [`HEARTBEAT_GATE_ITEMS`]: with variance evidence in hand, the
/// per-fork amortization is the only remaining constraint.
const HEARTBEAT_MIN_ITEMS: usize = 4096;

/// Fully-serial collect for small batches where dispatch overhead
/// would exceed total compute cost. Uses the same `MaybeUninit + ptr
/// write` pattern as `collect_indexed` so it shares the no-zero-init
/// path; just runs single-threaded on the calling thread.
fn serial_collect_indexed<R, F>(n: usize, f: &F) -> Vec<R>
where
    R: Send,
    F: Fn(usize) -> R + Sync,
{
    let mut buf: Vec<std::mem::MaybeUninit<R>> = Vec::with_capacity(n);
    // SAFETY: every slot is written by the loop below before transmute.
    unsafe { buf.set_len(n); }
    let base = buf.as_mut_ptr() as *mut R;
    for i in 0..n {
        // SAFETY: 0 <= i < n = capacity; writing into MaybeUninit<R>
        // via ptr.add(i) is the documented pattern.
        unsafe { base.add(i).write(f(i)); }
    }
    let mut buf = std::mem::ManuallyDrop::new(buf);
    // SAFETY: the loop above wrote every slot exactly once.
    // `MaybeUninit<R>` and `R` share layout, so reconstructing
    // the Vec from the raw parts is sound.
    unsafe { Vec::from_raw_parts(buf.as_mut_ptr() as *mut R, n, buf.capacity()) }
}

fn heartbeat_fill<R, F>(
    plan: &JobPlan,
    items: &mut [std::mem::MaybeUninit<R>],
    start: usize,
    f: &F,
)
where
    R: Send,
    F: Fn(usize) -> R + Sync,
{
    let n = items.len();
    if n == 0 {
        return;
    }
    // Per-fork overhead exceeds any promotion benefit below
    // MIN_LEAF_ITEMS; run fully serial in that regime. The fill is
    // one span for the SITE's statistics only (not a chunk leaf,
    // so it stays out of the process-global per-leaf counters).
    if n <= MIN_LEAF_ITEMS {
        let t0 = std::time::Instant::now();
        fill_serial(items, start, f);
        record_leaf_span_ns(plan.site, t0.elapsed().as_nanos() as u64, n as u64);
        return;
    }
    let base = items.as_mut_ptr() as *mut R;
    let quantum = heartbeat_ticks();
    let last_tick = read_tsc();
    // Serial-span start for the site classifier: everything filled
    // between here and a promotion (or loop completion) is one
    // leaf-equivalent span, timed on the counter and recorded in
    // nanoseconds.
    let span_t0 = last_tick;

    // Serial fill until rdtsc tick crosses the quantum; on tick
    // bisect the remaining tail in half and fork the far half via
    // sched::join_context. Both halves recurse through heartbeat_fill
    // so each forked sub-range maintains its own polling counter -
    // promotions form a binary tree over time matching Acar/Rainey
    // PLDI 2018 §4 "Native support for parallel loops".
    let mut i = 0;
    while i < n {
        // SAFETY: i < n, base+i is the i-th slot. Writing R into
        // MaybeUninit<R> via ptr is the documented pattern.
        unsafe { base.add(i).write(f(start + i)); }
        i += 1;

        // Heartbeat check: every POLL_MASK+1 iterations. Require
        // remaining tail >= 2 * MIN_LEAF_ITEMS so the bisect produces
        // halves at or above the serial-only threshold.
        if (i & POLL_MASK) == 0 && (n - i) >= 2 * MIN_LEAF_ITEMS {
            let now = read_tsc();
            if now.wrapping_sub(last_tick) >= quantum {
                record_leaf_span_ticks(plan.site, now.wrapping_sub(span_t0), i as u64);
                let (_filled, tail) = items.split_at_mut(i);
                let mid = tail.len() >> 1;
                let (near, far) = tail.split_at_mut(mid);
                let near_start = start + i;
                let far_start = start + i + mid;
                join_context(
                    plan,
                    |_left_inj| heartbeat_fill(plan, near, near_start, f),
                    |_right_stolen| heartbeat_fill(plan, far, far_start, f),
                );
                return;
            }
        }
    }
    // Loop completed without promoting: the whole run was one
    // serial span.
    record_leaf_span_ticks(plan.site, read_tsc().wrapping_sub(span_t0), n as u64);
}

/// Pure serial fill into a `MaybeUninit<R>` slice. Used by
/// `heartbeat_fill` when the remaining work is below the
/// per-fork-amortization threshold.
#[inline(always)]
fn fill_serial<R, F>(items: &mut [std::mem::MaybeUninit<R>], start: usize, f: &F)
where
    F: Fn(usize) -> R,
{
    let len = items.len();
    let base = items.as_mut_ptr() as *mut R;
    for i in 0..len {
        // SAFETY: 0 <= i < len = capacity; ptr.add(i).write is the
        // documented pattern.
        unsafe { base.add(i).write(f(start + i)); }
    }
}

/// Token-bucket heartbeat: an alternative to the rdtsc-tick
/// heartbeat that accumulates **work tokens** between checks. Each
/// item contributes a caller-supplied token count; promote-to-SLAW
/// fires when accumulated tokens cross [`TOKEN_BUCKET_PROMOTE`] OR
/// when an rdtsc tick crosses the quantum ([`heartbeat_ticks`]). Use for
/// heterogeneous-per-iter workloads where the rdtsc-mask-based
/// `collect_indexed_heartbeat` polls too rarely (heavy items) or
/// too often (light items) for the actual cost distribution.
///
/// `tokens_fn(i)` returns the per-item token count for index `i`.
/// Tokens are an opaque unit: callers can use bytes-of-work, FpN
/// limbs accessed, or coarse 1/10/100 buckets. The threshold is
/// in the same unit; pick `TOKEN_BUCKET_PROMOTE` to match expected
/// "first promote moment" for the workload.
///
#[track_caller]
pub fn collect_indexed_token_bucket<R, F, W>(
    plan: &JobPlan,
    n: usize,
    work_per_item: W,
    f: F,
) -> Vec<R>
where
    R: Send,
    F: Fn(usize) -> R + Sync,
    W: Fn(usize) -> u32 + Sync,
{
    if n == 0 {
        return Vec::new();
    }
    // Per-call-site identity from the caller's source location
    // (track_caller chain); an outer attachment wins.
    let (plan_owned, routing_arm) = plan
        .with_site_if_none(crate::sched::call_site::caller_site())
        .apply_site_class_measured();
    let _routing = RecordRoutingArm::start(plan_owned.site, routing_arm);
    let plan = &plan_owned;
    if capped_to_caller(plan) {
        return run_on_caller(plan, n, || (0..n).map(&f).collect());
    }

    // Entry gate identical to `collect_indexed_heartbeat`: skip the
    // token-bucket machinery if the caller's authoritative estimate
    // says the whole batch fits in one heartbeat tick (explicit-only
    // for the same reason as the heartbeat entry: site-learned
    // profile defaults also populate the estimate field).
    if plan.estimated_per_item_ns_explicit
        && let Some(total_ns) = plan.estimated_total_ns()
    {
        if total_ns < HEARTBEAT_NS {
            return serial_collect_indexed(n, &f);
        }
        return collect_indexed(plan, n, 1, f);
    }

    if n < HEARTBEAT_GATE_ITEMS {
        return collect_indexed(plan, n, 1, f);
    }

    let mut buf: Vec<std::mem::MaybeUninit<R>> = Vec::with_capacity(n);
    // SAFETY: `MaybeUninit<R>` needs no initialization; `set_len`
    // up to capacity is sound. `token_bucket_fill` below writes
    // every slot before we transmute.
    unsafe { buf.set_len(n); }
    token_bucket_fill(plan, &mut buf[..], 0, &f, &work_per_item);
    let mut buf = std::mem::ManuallyDrop::new(buf);
    // SAFETY: `token_bucket_fill` wrote every slot exactly once.
    // `MaybeUninit<R>` and `R` share layout, so reconstructing
    // the Vec from the raw parts is sound.
    unsafe { Vec::from_raw_parts(buf.as_mut_ptr() as *mut R, n, buf.capacity()) }
}

/// Parallel indexed collect with **Tiny-Tasks model** chunk sizing.
/// Consults [`JobPlan::optimal_chunk_count`] to derive `min_leaf =
/// n / optimal_chunks` instead of using the caller-supplied floor.
/// Falls back to plain [`collect_indexed`] with `min_leaf=1` when
/// the plan lacks the required estimates (`estimated_per_item_ns`
/// and `task_overhead_ns`).
#[track_caller]
pub fn collect_indexed_tiny_tasks<R, F>(plan: &JobPlan, n: usize, f: F) -> Vec<R>
where
    R: Send,
    F: Fn(usize) -> R + Sync,
{
    if n == 0 {
        return Vec::new();
    }
    // Before the pool's width is read, since reading it starts the pool.
    if capped_to_caller(plan) {
        return collect_indexed(plan, n, 1, f);
    }
    let workers = plan.effective_workers(global_local_arena().total_workers());
    let chunks = plan.optimal_chunk_count(workers);
    let min_leaf = match chunks {
        Some(c) if c > 0 => (n / c as usize).max(1),
        _ => 1,
    };
    collect_indexed(plan, n, min_leaf, f)
}

// NOTE: no `collect_indexed_idempotent` wire helper exists; the
// idempotent collect is a documented negative result. Bench on
// Zen+ AVX2:
//   n=10k:   standard 30.5us, idempotent 87.5us (2.87x slower)
//   n=100k:  standard 97.7us, idempotent 267us  (2.74x slower)
//   n=1M:    standard 2.06ms, idempotent 2.40ms (1.17x slower)
// The Relaxed-bitmap CAS overhead per slot exceeds any saved
// Acquire-fence benefit on x86-64 (TSO host: Acquire is
// essentially free, so there is no fence to save). The
// IdempotentJob trait stays in src/sched/idempotent.rs as a
// semantically useful marker trait.

/// Token-bucket promotion threshold in caller-defined work units.
/// Default sized so that ~20µs of work at "1 token = 1ns" accumulates
/// before promotion. Tunable per workload; the caller's `tokens_fn`
/// effectively chooses this value's meaning via the unit it picks.
const TOKEN_BUCKET_PROMOTE: u64 = 20_000;

fn token_bucket_fill<R, F, W>(
    plan: &JobPlan,
    items: &mut [std::mem::MaybeUninit<R>],
    start: usize,
    f: &F,
    tokens_fn: &W,
)
where
    R: Send,
    F: Fn(usize) -> R + Sync,
    W: Fn(usize) -> u32 + Sync,
{
    let n = items.len();
    if n == 0 {
        return;
    }
    let base = items.as_mut_ptr() as *mut R;
    let quantum = heartbeat_ticks();
    let mut last_tick = read_tsc();
    // Serial-prefix span start for the site classifier, timed on the
    // counter and recorded in nanoseconds.
    let span_t0 = last_tick;
    let mut tokens_since_check: u64 = 0;

    let workers = plan.effective_workers(global_local_arena().total_workers());
    let max_budget = workers.saturating_mul(plan.effective_leaves_per_worker()).max(1);

    let mut i = 0;
    while i < n {
        let idx = start + i;
        // SAFETY: i < n; base.add(i) is a valid slot.
        unsafe { base.add(i).write(f(idx)); }
        tokens_since_check =
            tokens_since_check.saturating_add(tokens_fn(idx) as u64);
        i += 1;

        // Token threshold OR rdtsc tick: either signal triggers a
        // promote-to-SLAW. Token check is cheap (one branch); rdtsc
        // check fires only every POLL_MASK+1 items to keep cost
        // amortized even when each token is small. The tick counts from
        // the span's start, as the heartbeat filler's does, so a fill
        // whose tokens under-count its items still promotes once a
        // quantum has passed.
        let token_trip = tokens_since_check >= TOKEN_BUCKET_PROMOTE;
        let tick_trip = (i & POLL_MASK) == 0
            && read_tsc().wrapping_sub(last_tick) >= quantum;
        if (token_trip || tick_trip) && i + 1 < n {
            let remaining = n - i;
            if remaining >= 2 {
                record_leaf_span_ticks(plan.site, read_tsc().wrapping_sub(span_t0), i as u64);
                let (_filled, tail) = items.split_at_mut(i);
                collect_inner(
                    plan, tail, start + i, f, 1,
                    max_budget, max_budget, false,
                );
                return;
            }
            tokens_since_check = 0;
            last_tick = read_tsc();
        }
    }
    // Loop completed without promoting: one serial span.
    record_leaf_span_ticks(plan.site, read_tsc().wrapping_sub(span_t0), n as u64);
}

/// Recursive bisection with a fixed `chunk_size` floor + adaptive
/// splitter. Same SLAW pattern as `bisect`.
fn bisect_fixed<T, F>(
    plan: &JobPlan,
    items: &mut [T],
    op: &F,
    splits: usize,
    max_budget: usize,
    chunk_size: usize,
    migrated: bool,
)
where
    T: Send,
    F: Fn(&mut [T]) + Sync,
{
    if items.len() <= chunk_size {
        record_leaf(plan.site, items.len(), || op(items));
        return;
    }
    let cur_splits = if migrated { max_budget } else { splits };
    // Leaf at `<= 1` (see `bisect` for the doubling-bug rationale).
    if cur_splits <= 1 {
        record_leaf(plan.site, items.len(), || op(items));
        return;
    }
    let mid = items.len() / 2;
    let (left, right) = items.split_at_mut(mid);
    let half = cur_splits >> 1;
    join_context(
        plan,
        |left_inj| bisect_fixed(plan, left, op, half, max_budget, chunk_size, left_inj),
        |right_stolen| bisect_fixed(plan, right, op, half, max_budget, chunk_size, right_stolen),
    );
}

/// Parallel reduction: split `items` into chunks, fold each chunk
/// with `fold`, then combine pairs with `reduce` until one value
/// remains.
///
/// `reduce` must be associative. The combine tree is binary and the
/// order of combination is not fixed, so an operator that is not
/// associative returns a value that depends on the shape of the tree.
/// Commutativity is not required.
///
/// The shape is chosen per call from state this call site has learned,
/// so it is not a property of the input. Below four timed samples the
/// reduce cost is unknown and the call takes the bisect tree; from the
/// fourth call an average exists and the flat shape can be selected
/// instead; the average goes on moving until sixteen. The flat shape
/// then splits into `workers * split_multiplier()` chunks, and that
/// multiplier is the process-global split observer's, retuned from the
/// pool's steal rate every 200ms.
///
/// So for a non-associative `reduce`, the same call site over the same
/// input on one host in one process can return one value early and
/// another later. Floating-point addition is the usual case. A caller
/// who needs a reproducible result needs an associative operator, not
/// a quiet host.
///
/// `init` is called once per chunk to seed its accumulator (so
/// each chunk gets a fresh `init` value, allowing thread-local
/// accumulators of types that aren't `Copy`).
///
/// # Strategy selection (adaptive)
///
/// A plan capped at one worker takes neither shape below: `fold` runs
/// once over the whole input on the calling thread, and
/// [`last_reduce_chunks_path`] answers [`ReduceChunksPath::Caller`].
/// Otherwise two dispatch shapes, picked automatically per call:
///
/// - **Flat fan-out**: pre-split into `workers * split_multiplier()`
///   contiguous chunks and dispatch via
///   [`crate::sched::cooperative_join_n_flat`] in one burst, then
///   parallel-tree-reduce the chunk results. Wins on uniform-cost
///   trivial-reduce workloads like histogram (256-bin element-wise
///   add, ~4us per merge) where the bisect's depth-first descent
///   ramp-up cost dominates.
/// - **Recursive bisect** (default): the [`reduce_inner`]
///   tree-bisect path. Wins when chunk costs vary (steal-driven
///   rebalancing absorbs variance) or when `reduce` is heavy
///   (word_count's HashMap merge at ~5ms per merge: the
///   serial-final-reduce cost in the flat path dominates).
///
/// Pick signal: observer-learned **reduce-cost**. The OOB
/// calibration block at the top of every reduce_chunks call
/// times one populated-accumulator `reduce(a, b)` merge on the
/// bench thread (bounded at 16 samples per call site) and
/// publishes the average to the caller's
/// [`crate::sched::call_site::CallSiteState`]. Subsequent calls
/// read it: if observed average is below the calibrated
/// trivial-reduce ceiling and input is large enough for clean
/// chunks above MIN_LEAF_ITEMS, flat-fanout fires. Otherwise
/// bisect.
///
/// The reduce-cost observer lives on the caller's
/// [`crate::sched::call_site::CallSiteState`], so a histogram
/// merge and a HashMap merge observe their own costs even through
/// the same generic entry (see [`crate::sched::call_site`] for why
/// a `static` cannot provide that identity).
#[track_caller]
pub fn reduce_chunks<T, A, F, R, I>(
    plan: &JobPlan,
    items: &[T],
    init: I,
    fold: F,
    reduce: R,
) -> A
where
    T: Sync,
    A: Send + 'static,
    I: Fn() -> A + Sync,
    F: Fn(A, &[T]) -> A + Sync,
    R: Fn(A, A) -> A + Sync,
{
    use std::sync::atomic::Ordering::Relaxed;
    // Per-call-site identity from the caller's source location
    // (track_caller chain): hosts the reduce-cost observer and
    // flows into the inner collect/for_each dispatches.
    let plan_owned = plan.with_site_if_none(crate::sched::call_site::caller_site());
    let plan = &plan_owned;
    if capped_to_caller(plan) {
        record_reduce_chunks_path(ReduceChunksPath::Caller);
        return run_on_caller(plan, items.len(), || fold(init(), items));
    }
    let site = plan.site.expect("attached above").get();
    // Trivial-reduce ceiling from the live calibratable thresholds
    // (default 30k cycles ~= 10us at 3 GHz; separates a 256-bin
    // element-wise add from a HashMap merge). Calibration measures
    // the reference merge in the running binary, so the value is
    // always in this build profile's own cycle units.
    let trivial_reduce_cycles_threshold: u64 =
        crate::sched::adaptive_profile::class_thresholds()
            .trivial_reduce_cycles
            .load(Relaxed);

    let workers = plan.effective_workers(global_local_arena().total_workers());
    let multiplier = plan.effective_leaves_per_worker();
    // Default chunk-count budget: workers * multiplier (multiplier=2
    // by default -> 32 chunks on a 16-worker host).
    //
    // Audit hook: FLYNNEL_REDUCE_CHUNKS_CHUNKS=<N> overrides the
    // default so chunk-count-parity-with-rayon experiments can run
    // without touching the bench code. Production callers never set
    // this; it's strictly for in-source investigation. Reference
    // rayon chunk counts on 16M-byte inputs:
    //   histogram   par_chunks(1MB)   -> 16 chunks
    //   word_count  par_chunks(256KB) -> 64 chunks
    //   kmeans      par_chunks(8KB)   -> 98 chunks
    let max_budget = match std::env::var("FLYNNEL_REDUCE_CHUNKS_CHUNKS")
        .ok()
        .and_then(|s| s.parse::<usize>().ok())
    {
        Some(n) if n >= 1 => n,
        _ => workers.saturating_mul(multiplier).max(1),
    };

    // Out-of-band calibration: time one reduce call on the bench thread
    // (never on workers, to avoid the wait-loop recursion +
    // measure_reduce stack-overflow problem documented in
    // reduce_inner). Bounded at MAX_SAMPLES; after convergence,
    // overhead drops to zero (the if-branch is skipped).
    //
    // Time the reduce over representative folded accumulators,
    // not reduce of init() pairs. Naive reduce(init(), init())
    // measures empty-input cost which is O(0) for HashMap-merge
    // reducers (word_count) and similar collection-shaped
    // accumulators -- it classifies them as trivial and routes
    // them to flat-fanout, which kills their perf because the
    // tree-reduce final stage merges fully-populated maps
    // serially-on-leaves rather than via bisect's natural
    // parallel-merge along the recursion unwind.
    //
    // Folding a small CAL_SLICE_LEN sample produces accumulators
    // with genuinely populated state, then reduce times the actual
    // merge cost. Total bench-thread overhead: 16 samples * (fold
    // cost over CAL_SLICE_LEN items + reduce of two populated
    // values). For histogram: ~16 * 2us = 32us total. For
    // word_count: ~16 * 50us = 0.8ms total. Negligible against a
    // 3-second criterion warm-up.
    if site.reduce_cost_wants_sample() {
        let cal_slice_len = MIN_LEAF_ITEMS.min(items.len());
        if cal_slice_len > 0 {
            let cal_a = fold(init(), &items[..cal_slice_len]);
            let cal_b = fold(init(), &items[..cal_slice_len]);
            let t0 = read_tsc();
            let drained = reduce(cal_a, cal_b);
            let dt = read_tsc().wrapping_sub(t0);
            site.record_reduce_cost_sample(dt);
            drop(drained);
        }
    }

    // Read the observer signal. `None` = cold path: not enough
    // data, default to bisect (the always-correct path).
    let observed_trivial = site
        .reduce_cost_avg_cycles()
        .is_some_and(|avg_cycles| avg_cycles < trivial_reduce_cycles_threshold);

    // Adaptive fast path (gated): trivial reduce + large input
    // -> flat fan-out. Audit found this path consistently slower
    // than bisect for the three characterized reduce_chunks
    // workloads because cooperative_join_n_flat's external_dispatch
    // + LockLatch overhead per tree-reduce round (5+ rounds)
    // dominates the "all workers come online faster" benefit.
    // Bisect's in-worker join_context is cheaper.
    //
    // FLYNNEL_ENABLE_FLAT_FANOUT=1 re-enables the path for in-source
    // experiments (verifying the audit finding hasn't drifted, or
    // benching a future fan_out_in_worker-only refactor). Default
    // off: bisect-only.
    let flat_enabled = std::env::var_os("FLYNNEL_ENABLE_FLAT_FANOUT")
        .is_some_and(|v| v != "0" && v != "false" && !v.is_empty());
    if flat_enabled
        && observed_trivial
        && items.len() >= max_budget.saturating_mul(MIN_LEAF_ITEMS)
    {
        record_reduce_chunks_path(ReduceChunksPath::Flat);
        return reduce_chunks_flat(plan, items, &init, &fold, &reduce, max_budget);
    }
    record_reduce_chunks_path(ReduceChunksPath::Bisect);
    reduce_inner(plan, items, &init, &fold, &reduce, max_budget, max_budget, false)
}

/// Which dispatch path [`reduce_chunks`] last selected. Exposed
/// for the bench-audit test that verifies the observer-learned
/// router actually routes each consumer to the right path
/// (histogram -> Flat, word_count -> Bisect, etc.). Without this
/// surface, "the adaptive routing works" is an unfalsifiable
/// claim instead of a verified one.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ReduceChunksPath {
    /// `reduce_chunks_flat` (`cooperative_join_n_flat` + parallel
    /// tree-reduce). Picked when observer classifies reduce as
    /// trivial and input is large enough for clean chunks.
    Flat,
    /// `reduce_inner` (recursive bisect). Default; picked when
    /// observer hasn't converged yet OR observed reduce is
    /// non-trivial OR input is too small.
    Bisect,
    /// One fold over the whole input on the calling thread, because
    /// the plan capped the dispatch at one worker.
    Caller,
}

thread_local! {
    static LAST_REDUCE_CHUNKS_PATH: std::cell::Cell<Option<ReduceChunksPath>> =
        const { std::cell::Cell::new(None) };
}

#[inline(always)]
fn record_reduce_chunks_path(p: ReduceChunksPath) {
    LAST_REDUCE_CHUNKS_PATH.with(|c| c.set(Some(p)));
}

/// Read the dispatch path that [`reduce_chunks`] selected on its
/// most recent call on the calling thread. Returns `None` if
/// reduce_chunks has not been called on this thread yet.
///
/// Bench-audit hook: a test that calls reduce_chunks with a
/// known-trivial reducer (e.g., sum) and asserts this returns
/// `Some(Flat)` after the observer warm-up converges; and
/// symmetrically for known-heavy reducers (sleep-spinning) ->
/// `Some(Bisect)`. Without this surface, the adaptive router's
/// correctness is unverifiable.
pub fn last_reduce_chunks_path() -> Option<ReduceChunksPath> {
    LAST_REDUCE_CHUNKS_PATH.with(|c| c.get())
}

/// Flat fan-out reduce: split `items` into `n_chunks` contiguous
/// slices, dispatch each as one closure via
/// [`crate::sched::cooperative_join_n_flat`], then parallel-tree-
/// reduce the chunk results. Used by [`reduce_chunks`] when its
/// observer signal classifies the call site as trivial-reduce.
///
/// # Safety
///
/// Uses raw-pointer arithmetic + `slice::from_raw_parts` to
/// reconstruct each chunk inside the dispatched closure, and
/// raw-pointer lifetime-laundering of `&I` and `&F` to fit the
/// `'static` closure requirement of `cooperative_join_n_flat`
/// without forcing reduce_chunks callers to provide `'static`
/// reducer closures.
///
/// Lifetime argument: `cooperative_join_n_flat` blocks the calling
/// thread until every closure has completed before returning, so
/// the caller's `items`, `init`, and `fold` references remain
/// valid for the entire closure lifetime. Pointer transport via
/// `usize` sidesteps the 2021+ disjoint-field-capture interaction
/// with raw pointer types.
///
/// Trait-bound argument: `T: Sync` makes `&[T]: Send` so the
/// reconstructed chunk reference is sound on a peer worker;
/// `I: Sync` and `F: Sync` make `&I` and `&F` `Send` so the
/// address-transport via `usize` is sound.
fn reduce_chunks_flat<T, A, F, R, I>(
    plan: &JobPlan,
    items: &[T],
    init: &I,
    fold: &F,
    reduce: &R,
    n_chunks: usize,
) -> A
where
    T: Sync,
    A: Send + 'static,
    I: Fn() -> A + Sync,
    F: Fn(A, &[T]) -> A + Sync,
    R: Fn(A, A) -> A + Sync,
{
    let len = items.len();
    debug_assert!(
        len >= n_chunks * MIN_LEAF_ITEMS,
        "reduce_chunks_flat called below chunk-size floor: len={len}, n_chunks={n_chunks}, MIN_LEAF_ITEMS={MIN_LEAF_ITEMS}",
    );
    let chunk_size = len.div_ceil(n_chunks);
    let n_real = len.div_ceil(chunk_size);
    let item_size = std::mem::size_of::<T>();
    let base_addr: usize = items.as_ptr() as usize;
    let fold_addr: usize = fold as *const F as usize;
    let init_addr: usize = init as *const I as usize;

    let closures: Vec<Box<dyn FnOnce() -> A + Send>> = (0..n_real)
        .map(|i| {
            let start = i * chunk_size;
            let chunk_len = chunk_size.min(len - start);
            let chunk_addr = base_addr + start * item_size;
            let boxed: Box<dyn FnOnce() -> A + Send> = Box::new(move || {
                // SAFETY: items, init, and fold are alive for the
                // entire cooperative_join_n_flat span (the call
                // blocks until every closure has completed).
                // start + chunk_len <= len by construction above.
                let chunk_ptr = chunk_addr as *const T;
                let chunk: &[T] = unsafe {
                    std::slice::from_raw_parts(chunk_ptr, chunk_len)
                };
                let fold_ref: &F = unsafe { &*(fold_addr as *const F) };
                let init_ref: &I = unsafe { &*(init_addr as *const I) };
                fold_ref(init_ref(), chunk)
            });
            boxed
        })
        .collect();

    let results = crate::sched::cooperative_join_n_flat(plan, closures);
    tree_reduce_in_parallel(plan, results, reduce)
}

/// Pair-merge `results` in parallel via [`crate::sched::
/// cooperative_join_n_flat`] until a single value remains. Used
/// by [`reduce_chunks_flat`] to combine its per-chunk results.
///
/// Matches the recursive bisect's natural tree-reduce shape: the
/// `reduce(la, ra)` call at each `join_context` level. Total work
/// = N-1 reduce calls (same as serial); wall-clock = ceil(log2(N))
/// * reduce_cost. For word_count's ~5ms HashMap merges over 16
///   results: 4 levels * 5ms = 20ms vs 15 * 5ms = 75ms serial.
fn tree_reduce_in_parallel<A, R>(plan: &JobPlan, mut results: Vec<A>, reduce: &R) -> A
where
    A: Send + 'static,
    R: Fn(A, A) -> A + Sync,
{
    while results.len() > 1 {
        let reduce_addr: usize = reduce as *const R as usize;
        // Pop an odd tail off so the remaining count is even.
        let odd_tail = if results.len() % 2 == 1 {
            results.pop()
        } else {
            None
        };
        let mut pairs: Vec<(A, A)> = Vec::with_capacity(results.len() / 2);
        let mut iter = results.into_iter();
        loop {
            let a = match iter.next() {
                Some(x) => x,
                None => break,
            };
            let b = iter
                .next()
                .expect("even results count after odd-tail pop");
            pairs.push((a, b));
        }
        let closures: Vec<Box<dyn FnOnce() -> A + Send>> = pairs
            .into_iter()
            .map(|(a, b)| {
                let boxed: Box<dyn FnOnce() -> A + Send> = Box::new(move || {
                    // SAFETY: reduce is alive for the entire
                    // tree_reduce_in_parallel call;
                    // cooperative_join_n_flat blocks until every
                    // closure completes. R: Sync means &R: Send.
                    let reduce_ref: &R = unsafe { &*(reduce_addr as *const R) };
                    reduce_ref(a, b)
                });
                boxed
            })
            .collect();
        let mut next_round = crate::sched::cooperative_join_n_flat(plan, closures);
        if let Some(extra) = odd_tail {
            next_round.push(extra);
        }
        results = next_round;
    }
    results.into_iter().next().expect("at least 1 result")
}

#[allow(clippy::too_many_arguments)]
fn reduce_inner<T, A, F, R, I>(
    plan: &JobPlan,
    items: &[T],
    init: &I,
    fold: &F,
    reduce: &R,
    splits: usize,
    max_budget: usize,
    migrated: bool,
) -> A
where
    T: Sync,
    A: Send,
    I: Fn() -> A + Sync,
    F: Fn(A, &[T]) -> A + Sync,
    R: Fn(A, A) -> A + Sync,
{
    if items.len() <= MIN_LEAF_ITEMS {
        return fold(init(), items);
    }
    let cur_splits = if migrated { max_budget } else { splits };
    // Leaf at `<= 1` (see `bisect` for the doubling-bug rationale).
    if cur_splits <= 1 {
        return fold(init(), items);
    }
    let mid = items.len() / 2;
    let (left, right) = items.split_at(mid);
    let half = cur_splits >> 1;
    let (la, ra) = join_context(
        plan,
        |left_inj| reduce_inner(plan, left, init, fold, reduce, half, max_budget, left_inj),
        |right_stolen| reduce_inner(plan, right, init, fold, reduce, half, max_budget, right_stolen),
    );
    reduce(la, ra)
}

/// Apply `op` to each element of `items` in parallel, with one
/// dispatched task per element. The leaf chunk size is 1: every
/// element is one unit of work for the scheduler.
///
/// Use this when each element's `op` is large enough (~10us+) to
/// amortize per-task dispatch overhead. For small per-element work
/// use [`for_each_chunk`] instead, which groups multiple elements
/// per leaf via the bisect splitter.
///
/// Examples of one-task-per-element work: per-block high-precision
/// arithmetic, per-row matrix factorization, per-particle PDE
/// step, per-image GPU dispatch coordinator. The common shape is
/// "few large units" rather than "many small units".
#[track_caller]
pub fn par_map_in_place<T, F>(plan: &JobPlan, items: &mut [T], op: F)
where
    T: Send,
    F: Fn(&mut T) + Sync,
{
    for_each_fixed_chunk(plan, items, 1, |slice| {
        for x in slice {
            op(x);
        }
    });
}

/// Parallel zip-apply: for each index `i`, call `op(&mut lhs[i],
/// &rhs[i])`. Mutates `lhs` in place; `rhs` is read-only. Panics
/// if `lhs.len() != rhs.len()`.
///
/// One task per index by default (matches [`par_map_in_place`]'s
/// granule). Useful when paired indices need disjoint mutable
/// access that the standard slice borrow checker cannot prove
/// (e.g., independent per-element ops with no cross-element
/// dependency).
///
/// Implementation routes through `for_each_fixed_chunk` over an
/// index-list and uses raw-pointer arithmetic internally to
/// produce disjoint `&mut T` per index. Each chunk owns a
/// disjoint contiguous index range so the pointer dereferences
/// never alias.
#[track_caller]
pub fn par_zip_apply<T, U, F>(plan: &JobPlan, lhs: &mut [T], rhs: &[U], op: F)
where
    T: Send,
    U: Sync,
    F: Fn(&mut T, &U) + Sync,
{
    assert_eq!(
        lhs.len(),
        rhs.len(),
        "par_zip_apply requires matching slice lengths"
    );
    let n = lhs.len();
    if n == 0 {
        return;
    }
    // Address-as-usize sidesteps the edition-2021 disjoint-capture
    // inspection that sees through *mut T newtypes; routing via
    // primitive `usize` keeps the closure capture trivially Sync.
    let lhs_addr: usize = lhs.as_mut_ptr() as usize;
    let mut indices: Vec<usize> = (0..n).collect();
    for_each_fixed_chunk(plan, &mut indices, 1, move |chunk| {
        let lhs_ptr: *mut T = lhs_addr as *mut T;
        for &i in chunk.iter() {
            // SAFETY: each index `i` belongs to exactly one chunk
            // by construction, so the resulting `&mut T` does not
            // alias any other concurrent `&mut T` on lhs.
            let t: &mut T = unsafe { &mut *lhs_ptr.add(i) };
            op(t, &rhs[i]);
        }
    });
}

// ---------------------------------------------------------------------------
// Tests
// ---------------------------------------------------------------------------

#[cfg(test)]
mod tests {
    use super::*;
    use std::sync::Arc;
    use std::sync::atomic::{AtomicU64, Ordering};

    use crate::sched::plan::JobPlan;

    /// The crossover interpolation survives timings that arrive out of
    /// order, which a contended host produces.
    ///
    /// The sweep times a body at n and then at 2n and expects the
    /// larger count to cost more. Under load it does not always, and
    /// the span between them is a u64 subtraction: unguarded it panics
    /// in debug and wraps in release, and a wrapped span sets the
    /// collapse threshold that governs every dispatch on that host.
    /// This is reachable from `calibrate_inline_collapse_threshold`,
    /// which a consumer calls at startup.
    #[test]
    fn crossover_interpolation_survives_unordered_timings() {
        // Ordered pair: the crossover falls between the two counts.
        let placed = interpolate_crossover(Some((100, 40)), 200, -40);
        assert!(
            (100..=200).contains(&placed),
            "an ordered pair interpolates between its two costs, got {placed}"
        );

        // The larger count timed faster, which the model does not
        // describe. Reporting the measurement is correct; underflowing
        // is not.
        assert_eq!(
            interpolate_crossover(Some((200, 40)), 100, -40),
            100,
            "an unordered pair has no crossover to place and must report the \
             measurement rather than subtract past zero"
        );

        // Equal costs are equally unordered: span zero carries nothing.
        assert_eq!(interpolate_crossover(Some((100, 40)), 100, -40), 100);

        // A gap that did not narrow is not a crossing either.
        assert_eq!(interpolate_crossover(Some((100, -40)), 200, -40), 200);

        // No previous count at all.
        assert_eq!(interpolate_crossover(None, 175, -1), 175);
    }

    #[test]
    fn for_each_chunk_zero_items_is_noop() {
        let mut v: Vec<u32> = Vec::new();
        let plan = JobPlan::new(6, 1);
        for_each_chunk(&plan, &mut v, |slice| {
            for x in slice {
                *x += 1;
            }
        });
        assert!(v.is_empty());
    }

    #[test]
    fn bisect_variants_all_produce_identical_result() {
        // Correctness baseline for the two BisectVariant entries:
        // same input through default (None, the lazy-steal path) and
        // through each variant must produce the same output. Forces n large enough that the probe-and-decide
        // path does not fire (`n >= workers * MIN_LEAF_ITEMS` with
        // workers ~16 and MIN_LEAF_ITEMS=256 -> n >= 4096). At n=20000
        // every variant exercises the bisect path it routes through.
        let n = 20_000usize;
        let template: Vec<u32> = (0..n as u32).collect();

        let run = |variant: Option<crate::sched::plan::BisectVariant>| -> Vec<u32> {
            let mut v = template.clone();
            let mut plan = JobPlan::new(6, n as u32);
            plan.bisect_variant = variant;
            for_each_chunk(&plan, &mut v, |slice| {
                for x in slice {
                    *x = x.wrapping_mul(3).wrapping_add(7);
                }
            });
            v
        };

        let baseline = run(None);
        assert_eq!(run(Some(crate::sched::plan::BisectVariant::ProducerMaxLenWorkers)), baseline);
        assert_eq!(run(Some(crate::sched::plan::BisectVariant::RayonStyleReplenish)), baseline);
    }

    #[test]
    fn for_each_chunk_touches_every_element_exactly_once() {
        let n = 10_000usize;
        let mut v: Vec<u32> = (0..n as u32).collect();
        let plan = JobPlan::new(6, n as u32);
        for_each_chunk(&plan, &mut v, |slice| {
            for x in slice {
                *x = x.wrapping_mul(2);
            }
        });
        for (i, &x) in v.iter().enumerate() {
            assert_eq!(x, (i as u32).wrapping_mul(2),
                "element {i} = {x}, expected {}", (i as u32).wrapping_mul(2));
        }
    }

    #[test]
    fn for_each_chunk_small_input_runs_serial() {
        // An input carrying an authoritative per-item cost whose total
        // falls below this host's collapse threshold runs on the
        // calling thread without entering the pool.
        //
        // The estimate is explicit because that is what the collapse
        // gates on. With a probed cost the decision follows a live
        // measurement of the body, and a loaded host reads a one-add
        // item as expensive enough to dispatch, which is the routing
        // working rather than failing. An assertion over that path
        // states a fact about the host at the moment it runs;
        // `for_each_chunk_small_input_without_an_estimate` covers that
        // path with the invariant that does hold there.
        //
        // The runtime may probe-and-decide (one small probe plus a
        // tail), so op can be called more than once, but every call
        // must come from the calling thread. That is what separates
        // inline-collapse from pool dispatch.
        use std::sync::atomic::{AtomicUsize, Ordering};
        crate::sched::adaptive_profile::migrate_dispatch_profile(
            crate::DispatchProfile::PortBound,
        );
        let n = MIN_LEAF_ITEMS / 4;
        let mut v: Vec<u32> = (0..n as u32).collect();
        // One nanosecond an item puts the total at tens of
        // nanoseconds, below any measured threshold, which is floored
        // at this host's dispatch cost.
        let plan = JobPlan::new(6, n as u32).with_estimated_per_item_ns(1);
        let snap = format!(
            "use_smt={} est_pi={:?} explicit={} oversub={:?} global={:?}",
            plan.use_smt,
            plan.estimated_per_item_ns,
            plan.estimated_per_item_ns_explicit,
            plan.oversubscription_log2,
            crate::sched::adaptive_profile::active_dispatch_profile(),
        );
        let calling_thread = std::thread::current().id();
        let total_processed = AtomicUsize::new(0);
        // Violations are collected and asserted after the dispatch
        // returns: a panic inside a pool-worker leaf is re-raised on
        // the test thread with its message swallowed by the pool's
        // propagation, which hides the diagnostic snapshot.
        let off_thread_calls = AtomicUsize::new(0);
        for_each_chunk(&plan, &mut v, |slice| {
            if std::thread::current().id() != calling_thread {
                off_thread_calls.fetch_add(1, Ordering::Relaxed);
            }
            total_processed.fetch_add(slice.len(), Ordering::Relaxed);
            for x in slice {
                *x += 1000;
            }
        });
        assert_eq!(
            off_thread_calls.load(Ordering::Relaxed),
            0,
            "an explicit estimate under the collapse threshold must not dispatch to the pool [{snap}]"
        );
        assert_eq!(
            total_processed.load(Ordering::Relaxed),
            n,
            "all items must be processed exactly once across probe + tail"
        );
        for (i, &x) in v.iter().enumerate() {
            assert_eq!(x, i as u32 + 1000);
        }
    }

    #[test]
    fn for_each_chunk_small_input_without_an_estimate() {
        // The same small input with no authoritative cost. Here the
        // probe measures the body and routes on what it finds, so
        // whether the work lands on the calling thread or in the pool
        // depends on the host at that moment, and neither outcome is a
        // defect. What must hold either way is that the walk covers
        // the slice exactly once.
        use std::sync::atomic::{AtomicUsize, Ordering};
        crate::sched::adaptive_profile::migrate_dispatch_profile(
            crate::DispatchProfile::PortBound,
        );
        let n = MIN_LEAF_ITEMS / 4;
        let mut v: Vec<u32> = (0..n as u32).collect();
        let plan = JobPlan::new(6, n as u32);
        let total_processed = AtomicUsize::new(0);
        for_each_chunk(&plan, &mut v, |slice| {
            total_processed.fetch_add(slice.len(), Ordering::Relaxed);
            for x in slice {
                *x += 1000;
            }
        });
        assert_eq!(
            total_processed.load(Ordering::Relaxed),
            n,
            "every item is processed exactly once across probe and tail, on whichever thread"
        );
        for (i, &x) in v.iter().enumerate() {
            assert_eq!(x, i as u32 + 1000, "item {i} processed exactly once");
        }
    }

    #[test]
    fn one_outlier_moves_the_range_and_leaves_the_iqr_alone() {
        // Why the two are reported side by side. Eight samples that
        // agree and one that does not is the shape a scheduling hiccup
        // produces, and it is the shape that refuses records whose
        // medians are reproducible to a few percent.
        let tight: Vec<u64> = vec![100, 101, 102, 103, 104, 105, 106, 107, 108];
        assert!(
            sample_spread_per_mille(&tight) < 100,
            "samples that agree report a small range"
        );

        let mut hiccup = tight.clone();
        hiccup[8] = 900;
        assert!(
            sample_spread_per_mille(&hiccup) > 7_000,
            "one sample nine times the rest sets the whole range"
        );
        assert!(
            sample_iqr_per_mille(&hiccup) < 100,
            "the middle half is untouched by it, which is what the median keeps"
        );

        // And the case the range is right about: samples that genuinely
        // disagree move both figures.
        let scattered: Vec<u64> = vec![20, 50, 90, 140, 200, 260, 330, 410, 500];
        assert!(sample_spread_per_mille(&scattered) > 2_000);
        assert!(
            sample_iqr_per_mille(&scattered) > 1_000,
            "a spread that is real reaches the middle half too"
        );
    }

    #[cfg(feature = "persisted-calibration")]
    #[test]
    fn a_record_that_serves_is_one_the_refusal_will_never_be_offered() {
        use crate::sched::calibration_store::CpuCalibration;

        // The decision the caller actually takes, which every other
        // test of this feature skips by building a store and calling
        // the guard directly. A record that serves is returned before
        // the branch that offers one for comparison, so the refusal on
        // the other side of that branch cannot act on it.
        let now = 1_000_000u64;
        let mut passes = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990));
        passes.confirmations = 1;
        assert!(
            stored_record_serves(&passes, None, now),
            "with no age bound a trustworthy record is returned, so nothing offers it"
        );

        let provisional = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990));
        assert!(
            !stored_record_serves(&provisional, None, now),
            "a record nothing has agreed with is measured over, and a record in that \
             state is the only kind the refusal is ever offered"
        );
    }

    #[cfg(feature = "persisted-calibration")]
    #[test]
    fn the_default_bound_is_a_day_and_the_comparison_includes_it() {
        use crate::sched::calibration_store::CpuCalibration;

        assert_eq!(
            DEFAULT_CALIBRATION_MAX_AGE_S,
            24 * 60 * 60,
            "this bound is how long a contended draw can keep routing, so a change \
             to it fails here rather than passing quietly"
        );

        let now = 2_000_000_000u64;
        let bound = Some(DEFAULT_CALIBRATION_MAX_AGE_S);

        let mut at_the_bound = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990));
        at_the_bound.measured_unix_s = now - DEFAULT_CALIBRATION_MAX_AGE_S;
        at_the_bound.confirmations = 1;
        assert!(
            stored_record_serves(&at_the_bound, bound, now),
            "a record exactly the bound's age serves, so the comparison is inclusive"
        );

        let mut past_it = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990));
        past_it.measured_unix_s = now - DEFAULT_CALIBRATION_MAX_AGE_S - 1;
        past_it.confirmations = 1;
        assert!(
            !stored_record_serves(&past_it, bound, now),
            "a second past it is drawn again"
        );
    }

    #[cfg(feature = "persisted-calibration")]
    #[test]
    fn an_age_bound_is_what_lets_a_trustworthy_record_be_drawn_again() {
        use crate::sched::calibration_store::CpuCalibration;

        // A record stamped an hour ago, read under three bounds.
        let drawn_at = 1_000_000u64;
        let now = drawn_at + 3_600;
        // Built and then stamped, rather than through a struct update:
        // the record carries a private padding field, so a functional
        // update from outside its own module cannot name everything it
        // would have to.
        let mut aged = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990));
        aged.measured_unix_s = drawn_at;
        aged.confirmations = 1;

        assert!(
            stored_record_serves(&aged, Some(7_200), now),
            "inside the bound it stands"
        );
        assert!(
            !stored_record_serves(&aged, Some(1_800), now),
            "past the bound it is drawn again, which is the only way a trustworthy \
             record is ever displaced short of a layout change"
        );
        assert!(
            stored_record_serves(&aged, Some(0), now),
            "zero is read as no bound rather than as re-measure always"
        );

        // A clock that moved backward, or a record stamped ahead of
        // this host, must not re-measure at every start until the
        // clock catches up.
        let mut future = CpuCalibration::new(1_000, 70_000, 40_000, 41, 9, Some(990));
        future.measured_unix_s = now + 10_000;
        future.confirmations = 1;
        assert!(
            stored_record_serves(&future, Some(60), now),
            "a record stamped in the future ages to nothing and stands"
        );
    }

    #[test]
    fn a_pinned_profile_parses_all_three_fields_or_none() {
        let p = parse_pinned_profile("1000,5000,20000").expect("three counts parse");
        assert_eq!(p.dispatch_cost_ns, 1000);
        assert_eq!(p.collapse_threshold_ns, 5000);
        assert_eq!(p.jec_wake_threshold_ns, 20000);
        assert_eq!(
            parse_pinned_profile(" 1000 , 5000 , 20000 ").expect("spacing is tolerated"),
            p,
            "a value spaced out for readability pins the same profile"
        );

        // Every rejection falls back to measuring rather than pinning
        // a value the caller did not mean.
        assert!(parse_pinned_profile("1000,5000").is_none(), "a missing field is refused");
        assert!(
            parse_pinned_profile("1000,5000,20000,7").is_none(),
            "a fourth field is refused rather than ignored"
        );
        assert!(
            parse_pinned_profile("0,5000,20000").is_none(),
            "zero marks an uncalibrated profile and cannot be pinned"
        );
        assert!(
            parse_pinned_profile("1000,0,20000").is_none(),
            "zero is refused in the collapse field too"
        );
        assert!(parse_pinned_profile("1000,fast,20000").is_none(), "a non-count is refused");
        assert!(parse_pinned_profile("").is_none(), "an empty value is refused");
    }

    #[test]
    fn the_sampled_path_reaches_the_site_with_an_on_core_reading() {
        // The mechanism is worthless if it never populates, and a run
        // that measured nothing would report as a mechanism that did not
        // help. This asserts the whole chain the sampled path uses:
        // bracket, thread-local buffer, flush, site.
        static SITE: crate::sched::call_site::CallSiteState =
            crate::sched::call_site::CallSiteState::new();
        let site = crate::sched::call_site::SiteRef::new(&SITE);
        assert_eq!(SITE.oncore_items(), 0, "a fresh site has taken no reading");

        // Past FLUSH_THRESHOLD so the buffer reaches the site, with work
        // in each leaf the compiler cannot discard: a body optimized
        // away would leave a clock delta of zero, which is a reading a
        // starved thread can genuinely have.
        let mut sink = 0u64;
        for i in 0..(LocalLeafBuffer::FLUSH_THRESHOLD as usize * 4) {
            sink = sink.wrapping_add(record_leaf_on_core(Some(site), 64, || {
                let mut acc = 0u64;
                for k in 0..4_096u64 {
                    acc = acc.wrapping_add(std::hint::black_box(k ^ i as u64));
                }
                acc
            }));
        }
        std::hint::black_box(sink);

        // A platform without a thread clock takes no readings at all,
        // and that is a correct outcome rather than a failure: the
        // classifier falls back to wall time there. So the assertion is
        // conditional on the platform reporting one, and says which case
        // it took rather than passing silently in both.
        let (thread, _) = crate::sched::occupancy::clock_pair();
        if thread.ticks().is_some() {
            assert!(
                SITE.oncore_items() > 0,
                "this platform has a thread clock, so the sampled path must have \
                 carried a reading to the site"
            );
            assert!(
                SITE.per_item_oncore_cv2_per_mille().is_some(),
                "items were recorded, so a spread is computable from them"
            );
        } else {
            assert_eq!(
                SITE.oncore_items(),
                0,
                "with no thread clock the site must hold no on-core items rather \
                 than a zero that reads as a measurement"
            );
            assert_eq!(SITE.per_item_oncore_cv2_per_mille(), None);
        }
    }

    #[test]
    fn one_call_in_the_stride_takes_the_bracket_and_the_rest_do_not() {
        // Both halves of "one in N" are asserted: how many calls take
        // the bracket, and how far apart they are. A counter that reset
        // on every call would take all of them; one that never reset
        // would take every call after the first N. The count alone
        // separates neither from a correct stride, because a run long
        // enough for the second to be wrong is also long enough for its
        // count to look plausible.
        let rounds = LEAF_SAMPLE_STRIDE as usize * 10;
        let mut tick = 0u32;
        let mut taken_at = Vec::new();
        for call in 0..rounds {
            let (next, take) = advance_bracket_tick(tick);
            tick = next;
            if take {
                taken_at.push(call);
            }
        }

        assert_eq!(
            taken_at.len(),
            10,
            "one call in {LEAF_SAMPLE_STRIDE} over {rounds} calls is 10 brackets, \
             got {} at {taken_at:?}",
            taken_at.len()
        );
        for pair in taken_at.windows(2) {
            assert_eq!(
                pair[1] - pair[0],
                LEAF_SAMPLE_STRIDE as usize,
                "brackets at {taken_at:?} are not one stride apart"
            );
        }
    }

    #[test]
    fn the_indexed_path_times_every_leaf_and_leaves_the_bracket_to_the_switch() {
        // The indexed entry records through `record_leaf_bracket_sampled`,
        // which times every leaf and takes the bracket at the stride only
        // while `levers::oncore_spread` is on. That switch is read once
        // per process, and a test binary runs with it off, so what is
        // assertable here is that the bracket stays behind it.
        //
        // Both halves are asserted. A dispatch that recorded no leaves
        // at all would satisfy the second on its own, and would be a
        // broken recorder rather than the arrangement this pins. The
        // cadence itself is asserted by
        // `one_call_in_the_stride_takes_the_bracket_and_the_rest_do_not`,
        // which can reach it without the switch.
        static SITE: crate::sched::call_site::CallSiteState =
            crate::sched::call_site::CallSiteState::new();
        let site = crate::sched::call_site::SiteRef::new(&SITE);

        // Past the buffer's flush threshold so the counts reach the
        // site, with work in each leaf the compiler cannot discard.
        //
        // The estimate keeps every dispatch on the pool. Without one
        // the entry probes the body, and a probe that reads the whole
        // call under the host's collapse threshold runs it on the
        // caller as one leaf, through a recorder that never brackets.
        // The bracket is taken one leaf in the stride on each thread and
        // a leaf reaches the site when its thread has batched four for
        // the site, so which dispatch first lands a bracketed leaf at
        // the site depends on how the leaves were spread over the
        // workers. The dispatches repeat until one has, up to a bound
        // far past what the cadence allows.
        let n = 4 * MIN_LEAF_ITEMS;
        let mut v: Vec<u64> = (0..n as u64).collect();
        let bound = LocalLeafBuffer::FLUSH_THRESHOLD as usize * 64;
        let mut dispatches = 0usize;
        while dispatches < bound && SITE.oncore_items() == 0 {
            let plan = JobPlan::new(0, n as u32)
                .with_site(site)
                .with_estimated_per_item_ns(5_000);
            for_each_chunk_indexed_min_leaf(&plan, &mut v, MIN_LEAF_ITEMS, |_start, chunk| {
                for slot in chunk.iter_mut() {
                    let mut acc = *slot;
                    for _ in 0..64 {
                        acc = acc.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1);
                    }
                    *slot = std::hint::black_box(acc);
                }
            });
            dispatches += 1;
        }

        assert!(
            SITE.leaf_count() > 0,
            "the indexed entry records every leaf, so the site must have counts"
        );
        assert!(
            SITE.oncore_items() > 0,
            "the indexed entry reaches the bracket, which `oncore_spread` leaves on by \
             default; no bracketed leaf reached the site in {dispatches} dispatches, so \
             the recorder never takes it"
        );
        // That the bracket is gated by the switch rather than taken
        // unconditionally is asserted by
        // `one_call_in_the_stride_takes_the_bracket_and_the_rest_do_not` over
        // `advance_bracket_tick`. The switch is a process-wide OnceLock, so a
        // test binary sees one arm of it and cannot drive both.
        assert!(SITE.per_item_oncore_cv2_per_mille().is_some());
    }

    #[test]
    fn the_triple_path_times_every_leaf_and_leaves_the_bracket_to_the_switch() {
        // The same arrangement as the indexed entry, asserted separately
        // so a failure names which of the two changed. The triple
        // bisect is what the slice ops route through, so this covers
        // the other half of the production call sites that reach the
        // on-core switch.
        static SITE: crate::sched::call_site::CallSiteState =
            crate::sched::call_site::CallSiteState::new();
        let site = crate::sched::call_site::SiteRef::new(&SITE);

        // The estimate keeps every dispatch on the pool, and the
        // dispatches repeat until a bracketed leaf has reached the site,
        // for the reasons the indexed test gives.
        let n = 4 * MIN_LEAF_ITEMS;
        let a: Vec<u64> = (0..n as u64).collect();
        let b: Vec<u64> = (0..n as u64).map(|x| x ^ 0x5DEE_CE66).collect();
        let mut out: Vec<u64> = vec![0; n];
        let bound = LocalLeafBuffer::FLUSH_THRESHOLD as usize * 64;
        let mut dispatches = 0usize;
        while dispatches < bound && SITE.oncore_items() == 0 {
            let plan = JobPlan::new(0, n as u32)
                .with_site(site)
                .with_estimated_per_item_ns(5_000);
            for_each_chunk_triple_min_leaf(
                &plan,
                &mut out,
                &a,
                &b,
                MIN_LEAF_ITEMS,
                |out, a, b| {
                    for i in 0..out.len() {
                        let mut acc = a[i] ^ b[i];
                        for _ in 0..64 {
                            acc = acc.wrapping_mul(6_364_136_223_846_793_005).wrapping_add(1);
                        }
                        out[i] = std::hint::black_box(acc);
                    }
                },
            );
            dispatches += 1;
        }

        assert!(
            SITE.leaf_count() > 0,
            "the triple entry records every leaf, so the site must have counts"
        );
        assert!(
            SITE.oncore_items() > 0,
            "the triple entry reaches the bracket, which `oncore_spread` leaves on by \
             default; no bracketed leaf reached the site in {dispatches} dispatches, so \
             the recorder never takes it"
        );
        assert!(SITE.per_item_oncore_cv2_per_mille().is_some());
    }

    #[test]
    fn host_dispatch_profile_is_measured_and_cached() {
        let t0 = std::time::Instant::now();
        let p = calibrate_host_dispatch();
        let cost = t0.elapsed();
        // Not merely positive: the max(1) floor is positive, and a
        // floored value is a measurement that produced nothing wearing
        // the shape of one that did. A dispatch pushes a job, arms a
        // latch and wakes a worker, so it cannot cost tens of
        // nanoseconds on any host this runs on.
        assert!(
            p.dispatch_cost_ns > 100,
            "a dispatch cost of {} ns is a failed measurement, not a fast pool",
            p.dispatch_cost_ns
        );
        assert!(p.collapse_threshold_ns >= p.dispatch_cost_ns, "the collapse threshold is floored at the dispatch cost");
        assert!(p.jec_wake_threshold_ns > 0, "a measured wake threshold is positive");
        assert_eq!(host_dispatch_profile(), p, "the query hands out the installed profile");
        assert_eq!(inline_collapse_threshold_ns(), p.collapse_threshold_ns);
        eprintln!(
            "host dispatch profile: dispatch cost {} ns, collapse threshold {} ns, wake threshold {} ns (measured in {cost:?})",
            p.dispatch_cost_ns, p.collapse_threshold_ns, p.jec_wake_threshold_ns
        );
    }

    #[test]
    fn triple_min_leaf_small_explicit_estimate_runs_on_the_caller() {
        // 200 items at 1 ns each is 0.2 us of work, under any host's
        // measured dispatch cost: the body runs once, on the calling
        // thread.
        use std::sync::atomic::{AtomicUsize, Ordering};
        let n = 200usize;
        let a: Vec<u32> = (0..n as u32).collect();
        let b: Vec<u32> = vec![1; n];
        let mut out = vec![0u32; n];
        let plan = JobPlan::new(6, n as u32).with_estimated_per_item_ns(1);
        let caller = std::thread::current().id();
        let off_thread = AtomicUsize::new(0);
        let calls = AtomicUsize::new(0);
        for_each_chunk_triple_min_leaf(&plan, &mut out, &a, &b, 64, |o, x, y| {
            if std::thread::current().id() != caller {
                off_thread.fetch_add(1, Ordering::Relaxed);
            }
            calls.fetch_add(1, Ordering::Relaxed);
            for ((o, x), y) in o.iter_mut().zip(x).zip(y) {
                *o = x + y;
            }
        });
        assert_eq!(off_thread.load(Ordering::Relaxed), 0, "must not dispatch to the pool");
        assert_eq!(calls.load(Ordering::Relaxed), 1, "one body call over the whole slice");
        assert!(out.iter().enumerate().all(|(i, &v)| v == i as u32 + 1));
    }

    #[test]
    fn indexed_min_leaf_small_explicit_estimate_runs_on_the_caller() {
        use std::sync::atomic::{AtomicUsize, Ordering};
        let n = 200usize;
        let mut v: Vec<u32> = vec![0; n];
        let plan = JobPlan::new(6, n as u32).with_estimated_per_item_ns(1);
        let caller = std::thread::current().id();
        let off_thread = AtomicUsize::new(0);
        let calls = AtomicUsize::new(0);
        for_each_chunk_indexed_min_leaf(&plan, &mut v, 1, |start, chunk| {
            if std::thread::current().id() != caller {
                off_thread.fetch_add(1, Ordering::Relaxed);
            }
            calls.fetch_add(1, Ordering::Relaxed);
            for (i, x) in chunk.iter_mut().enumerate() {
                *x = (start + i) as u32;
            }
        });
        assert_eq!(off_thread.load(Ordering::Relaxed), 0, "must not dispatch to the pool");
        assert_eq!(calls.load(Ordering::Relaxed), 1, "one body call over the whole slice");
        assert!(v.iter().enumerate().all(|(i, &x)| x == i as u32));
    }

    #[test]
    fn for_each_fixed_chunk_respects_chunk_size() {
        let n = 1000usize;
        let chunk = 64;
        let mut v: Vec<u32> = vec![0; n];
        let plan = JobPlan::new(6, n as u32);
        use std::sync::atomic::AtomicUsize;
        // The assertion is over the largest chunk, so the largest is
        // what the leaves record. Collecting every size to look at the
        // maximum is a Vec and a lock for a number fetch_max already
        // holds.
        let widest = Arc::new(AtomicUsize::new(0));
        let widest_clone = Arc::clone(&widest);
        for_each_fixed_chunk(&plan, &mut v, chunk, |slice| {
            widest_clone.fetch_max(slice.len(), Ordering::Relaxed);
            for x in slice {
                *x = 7;
            }
        });
        let widest = widest.load(Ordering::Relaxed);
        assert!(
            widest <= chunk,
            "chunk size {widest} exceeded the {chunk}-item ceiling"
        );
        // Every element was touched.
        assert!(v.iter().all(|&x| x == 7));
    }

    #[test]
    fn for_each_chunk_parallel_sum_matches_serial() {
        // Use atomic counter to verify total work matches expected
        // count. for_each_chunk must produce the same total work
        // as a serial loop.
        let n = 50_000usize;
        let mut v: Vec<u32> = (0..n as u32).collect();
        let counter = Arc::new(AtomicU64::new(0));
        let plan = JobPlan::new(6, n as u32);
        for_each_chunk(&plan, &mut v, |slice| {
            let local: u64 = slice.iter().map(|&x| x as u64).sum();
            counter.fetch_add(local, Ordering::Relaxed);
        });
        let expected: u64 = (0..n as u64).sum();
        assert_eq!(counter.load(Ordering::Relaxed), expected,
            "parallel sum {} != serial sum {expected}",
            counter.load(Ordering::Relaxed));
    }

    #[test]
    fn reduce_chunks_sum_matches_serial() {
        let n = 100_000usize;
        let v: Vec<u64> = (0..n as u64).collect();
        let plan = JobPlan::new(6, n as u32);
        let parallel = reduce_chunks(
            &plan,
            &v,
            || 0u64,
            |acc, chunk| acc + chunk.iter().sum::<u64>(),
            |a, b| a + b,
        );
        let serial: u64 = v.iter().sum();
        assert_eq!(parallel, serial);
    }

    #[test]
    fn reduce_chunks_large_trivial_reduce_matches_serial() {
        // 64 KiB items: above workers*MIN_LEAF_ITEMS (~4096) so
        // flat-fanout path can activate after observer converges.
        // Smaller than the bench input so debug-profile stack
        // usage stays well under the 8 MiB worker ceiling.
        let n = 64 * 1024usize;
        let v: Vec<u32> = (0..n as u32).collect();
        let plan = JobPlan::new(6, n as u32).with_estimated_per_item_ns(1);
        let mut last = [0u64; 256];
        for _ in 0..5 {
            last = reduce_chunks(
                &plan, &v,
                || [0u64; 256],
                |mut acc, chunk| {
                    for &x in chunk {
                        acc[(x & 0xFF) as usize] += 1;
                    }
                    acc
                },
                |mut a, b| {
                    for i in 0..256 {
                        a[i] += b[i];
                    }
                    a
                },
            );
        }
        let mut serial = [0u64; 256];
        for &x in &v {
            serial[(x & 0xFF) as usize] += 1;
        }
        assert_eq!(last, serial, "flat-fanout path output must match serial reference");
    }

    // Audit test infrastructure that was attempted here exposed
    // two real findings then deleted as unreliable methodology:
    //
    // 1. OOB calibration cycle reading is sensitive to whether
    //    the first sample catches worker-pool spawn-up. Histogram
    //    (trivial reducer) can read as heavy if its first
    //    calibration is the very first reduce_chunks call in the
    //    process. Subsequent runs of the same test see the
    //    cached-warm timing.
    //
    // 2. Debug-profile builds (cargo test default) measure
    //    significantly higher per-reduce cycles than release-
    //    profile builds (cargo bench default) for the same
    //    closure. A histogram-style 256-u64 element-wise add
    //    measures ~200 cycles in release (below the 30k-cycle
    //    threshold -> trivial) but ~30k+ cycles in debug (right
    //    at or above the threshold -> nondeterministic
    //    classification).
    //
    // The chunk-count investigation that produced these findings
    // moved to bench-driven validation (FLYNNEL_REDUCE_CHUNKS_CHUNKS=N
    // + multi-sample sweeps across the reduce_chunks call sites). The
    // [`ReduceChunksPath`] enum and [`last_reduce_chunks_path()`]
    // accessor are kept as infrastructure for any future audit
    // test that doesn't depend on calibration outcome (e.g., one
    // that directly drives the observer state via a test-only
    // setter).

    #[test]
    fn a_measured_fine_grain_site_keeps_its_floor_while_its_class_holds_smt_on() {
        use crate::sched::call_site::{CallSiteState, SiteRef};

        // A site that has measured 5 ns items, which is fine-grain: a
        // leaf of one such item is three orders of magnitude under one
        // dispatch, so the floor must stay where the caller put it.
        static FINE: CallSiteState = CallSiteState::new();
        const PER_ITEM_NS: u64 = 5;
        const ITEMS: u64 = 4_096;
        let leaf_ns = PER_ITEM_NS * ITEMS;
        let scaled = leaf_ns >> 8;
        let sq = scaled.saturating_mul(scaled);
        let per_item_sq = (leaf_ns as u128)
            .saturating_mul(leaf_ns as u128)
            .checked_div((ITEMS as u128) << 16)
            .unwrap_or(0) as u64;
        for _ in 0..4 {
            FINE.record_batch_site_only(leaf_ns * 8, sq * 8, 8, ITEMS * 8, per_item_sq * 8);
        }
        assert_eq!(
            FINE.per_item_ns(),
            Some(PER_ITEM_NS),
            "the site measured its own per-item cost"
        );

        // use_smt is what the floor used to read as "items are heavy".
        // It says nothing about this site's items, and the measurement
        // does.
        let caller_floor = MIN_LEAF_ITEMS;
        let plan = JobPlan::bare(6, 65_536)
            .with_smt()
            .with_site(SiteRef::new(&FINE));
        assert!(plan.use_smt, "the arrangement under test has SMT on");
        assert!(
            !plan.estimated_per_item_ns_explicit,
            "no caller hint, so the floor has only the site to read"
        );
        // The floor is capped by the caller and otherwise sized so one
        // leaf carries about a dispatch of work. Both bounds come from
        // a dispatch cost measured per process, which reads 900 to
        // 1100 ns unloaded and 2100 to 5500 ns loaded, so the contract
        // is what holds on every host and an exact leaf count is not.
        let dispatch_ns = pool_dispatch_cost_ns();
        let measured_floor = adaptive_min_leaf(&plan, caller_floor);
        assert!(
            measured_floor > 1,
            "a site measuring {PER_ITEM_NS} ns an item must not collapse to one item per \
             leaf, which would hand each of them a dispatch; got {measured_floor}"
        );
        assert!(
            measured_floor <= caller_floor,
            "the floor never rises above the caller's {caller_floor}; got {measured_floor}"
        );
        assert!(
            measured_floor == caller_floor
                || (measured_floor as u64 + 1) * PER_ITEM_NS > dispatch_ns,
            "a leaf carries about one dispatch of work unless the caller's floor caps it \
             first, so one item fewer would not cover it: {measured_floor} items at \
             {PER_ITEM_NS} ns against a dispatch measured at {dispatch_ns} ns"
        );

        // A site with no measurement still gets the old signal, so the
        // heavy-item case that floor exists for is unchanged.
        static UNMEASURED: CallSiteState = CallSiteState::new();
        let bare = JobPlan::bare(6, 8).with_smt().with_site(SiteRef::new(&UNMEASURED));
        assert_eq!(
            adaptive_min_leaf(&bare, caller_floor),
            1,
            "with nothing measured, use_smt still stands in for a per-item figure"
        );
    }

    #[test]
    fn bench_audit_explicit_heavy_reduce_routes_to_bisect() {
        // Explicit-cost audit: the reduce closure busy-spins for
        // ~50us, comfortably above the trivial-reduce ceiling
        // regardless of input shape, build profile (debug vs
        // release), or cold-vs-warm worker state. This is the only
        // routing-audit test that's reliable across all build/run
        // contexts. Trivial-reducer routing tests are unreliable
        // methodology here: the calibration cycle reading varies
        // with cold-warm pool state and with build profile, so no
        // such test exists in this module.
        //
        // Enable the flat-fanout gate (production keeps it off by
        // default since the bench audit found flat is structurally
        // slower for the three characterized reduce_chunks
        // consumers; the gate is for in-source experiments).
        // SAFETY: process-global env-var set; tests in this module
        // are expected to run with --test-threads=1 if they touch
        // shared process state.
        unsafe {
            std::env::set_var("FLYNNEL_ENABLE_FLAT_FANOUT", "1");
        }
        LAST_REDUCE_CHUNKS_PATH.with(|c| c.set(None));
        let n = 64 * 1024usize;
        let v: Vec<u32> = (0..n as u32).collect();
        let plan = JobPlan::new(6, n as u32).with_estimated_per_item_ns(1);
        let mut last_path = None;
        for _ in 0..8 {
            let _result = reduce_chunks(
                &plan, &v,
                || 0u64,
                |acc, chunk| acc + chunk.iter().map(|&x| x as u64).sum::<u64>(),
                |a, b| {
                    let t0 = read_tsc();
                    while read_tsc().wrapping_sub(t0) < 150_000 {
                        std::hint::spin_loop();
                    }
                    a.wrapping_add(b)
                },
            );
            last_path = last_reduce_chunks_path();
        }
        assert_eq!(
            last_path,
            Some(ReduceChunksPath::Bisect),
            "explicit-heavy reducer (>30k-cycle spin) must route to Bisect; got {last_path:?}"
        );
    }

    #[test]
    fn reduce_chunks_large_heavy_reduce_matches_serial() {
        // Same input-size reasoning as the trivial-reduce test.
        let n = 64 * 1024usize;
        let v: Vec<u32> = (0..n as u32).collect();
        let plan = JobPlan::new(6, n as u32).with_estimated_per_item_ns(1);
        let mut last_sum: u64 = 0;
        for _ in 0..3 {
            last_sum = reduce_chunks(
                &plan, &v,
                || 0u64,
                |acc, chunk| acc + chunk.iter().map(|&x| x as u64).sum::<u64>(),
                |a, b| {
                    // ~50 us simulated heavy reduce. Observer
                    // should classify above the trivial threshold and
                    // keep call site on bisect path.
                    let t0 = read_tsc();
                    while read_tsc().wrapping_sub(t0) < 150_000 {
                        std::hint::spin_loop();
                    }
                    a.wrapping_add(b)
                },
            );
        }
        let serial: u64 = v.iter().map(|&x| x as u64).sum();
        assert_eq!(last_sum, serial, "bisect path output must match serial reference");
    }

    #[test]
    fn for_each_chunk_records_leaf_time_stats() {
        // `record_leaf` brackets a leaf with the timestamp counter and
        // the flush converts the delta to nanoseconds. Every recorded
        // leaf lands in two places: the plan's site, which only this
        // test's dispatches feed, and the process-global counters,
        // which every dispatch in the suite feeds. The exact checks
        // read the site. The global counters are checked as floors,
        // because a neighbor's leaves can only add to them.
        //
        // One worker, so each dispatch is one leaf run on this thread
        // inside the wall time measured around the loop. The summed
        // leaf time then cannot exceed that wall time by more than the
        // recorder's own cost per leaf, and a total past twice the wall
        // time is the leaf clock and the wall clock not sharing a unit:
        // the timestamp counter runs at over two ticks a nanosecond on
        // every host this runs on, so a delta recorded in ticks lands
        // above the ceiling.
        use crate::sched::split_observer::snapshot_leaf_stats;

        static SITE: crate::sched::call_site::CallSiteState =
            crate::sched::call_site::CallSiteState::new();
        let site = crate::sched::call_site::SiteRef::new(&SITE);
        let n = 8 * MIN_LEAF_ITEMS;
        let mut v: Vec<u32> = (0..n as u32).collect();
        let plan = JobPlan::new(6, n as u32).with_site(site).with_workers(1);
        // Past the buffer's flush threshold, so the counts reach the
        // site and the global counters before the reads below.
        let dispatches = LocalLeafBuffer::FLUSH_THRESHOLD as usize * 4;
        let before = snapshot_leaf_stats();
        let dispatch_start = std::time::Instant::now();
        for _ in 0..dispatches {
            for_each_chunk_indexed_min_leaf(&plan, &mut v, MIN_LEAF_ITEMS, |_start, slice| {
                // Work the compiler cannot discard, heavy enough that a
                // leaf's time is well above the resolution of a pair of
                // counter reads.
                for x in slice.iter_mut() {
                    let mut acc = *x as u64;
                    for _ in 0..64 {
                        acc = acc.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1);
                    }
                    *x = std::hint::black_box(acc) as u32;
                }
            });
        }
        let dispatch_ns = dispatch_start.elapsed().as_nanos() as u64;
        let after = snapshot_leaf_stats();

        let count = SITE.leaf_count();
        assert!(
            count >= LocalLeafBuffer::FLUSH_THRESHOLD,
            "{dispatches} single-leaf dispatches should have flushed at least one batch to \
             the site, which holds {count} leaves"
        );
        let sum_ns = SITE.leaf_sum_ns();
        assert!(
            sum_ns > 0,
            "expected a positive total leaf time in nanoseconds at the site"
        );
        let ceiling = dispatch_ns.saturating_mul(2).max(1);
        assert!(
            sum_ns <= ceiling,
            "{} ns of leaf time over {} leaves ({} ns each) on one thread inside {} ns of wall \
             time, ceiling {}: the leaf clock and the wall clock are not sharing a unit",
            sum_ns,
            count,
            sum_ns / count.max(1),
            dispatch_ns,
            ceiling
        );

        // The global half flushes on its own cadence from whatever
        // residue this thread carried in, so its rise can trail the
        // site's by one batch less a leaf, and neighbors' leaves can
        // only raise it.
        let global_count = after.count.wrapping_sub(before.count);
        assert!(
            global_count + LocalLeafBuffer::FLUSH_THRESHOLD > count,
            "the global counters rose by {global_count} leaves while the site recorded {count}"
        );
        assert!(
            after.sum_ns.wrapping_sub(before.sum_ns) > 0,
            "the global leaf time did not rise while the site recorded {sum_ns} ns"
        );
    }

    /// One item of the span tests' fills: heavy enough that a fill's
    /// span is far above the resolution of a pair of counter reads, and
    /// kept by `black_box` so the fill cannot be discarded.
    fn span_test_item(i: usize) -> u64 {
        let mut acc = i as u64;
        for _ in 0..256 {
            acc = acc.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1);
        }
        std::hint::black_box(acc)
    }

    /// Check that the one span a fill recorded against `site` arrived in
    /// nanoseconds, given the wall time measured around the fill and the
    /// interval from its first item's start to its last item's end,
    /// which the span encloses.
    ///
    /// A span past twice the wall time is counter ticks, which run at
    /// over two a nanosecond on every host this runs on. A span under
    /// half the inner interval is the conversion applied twice. Losing
    /// the core inside the fill lengthens all three alike, and a stall
    /// outside it only widens the ceiling.
    fn assert_span_in_nanoseconds(
        site: &crate::sched::call_site::CallSiteState,
        rate_16: u64,
        wall_ns: u64,
        inner_ns: u64,
    ) {
        assert_eq!(
            site.leaf_count(),
            1,
            "a fill that cannot promote records one span"
        );
        let span_ns = site.leaf_sum_ns();
        assert!(
            span_ns <= wall_ns.saturating_mul(2),
            "a {span_ns} ns span inside {wall_ns} ns of wall time, with the counter at \
             {rate_16} sixteenths of a tick a nanosecond: the span is in counter ticks"
        );
        assert!(
            span_ns >= inner_ns / 2,
            "a {span_ns} ns span around {inner_ns} ns of items, with the counter at \
             {rate_16} sixteenths of a tick a nanosecond: the span was converted twice"
        );
    }

    #[test]
    fn a_heartbeat_span_reaches_its_site_in_nanoseconds() {
        use std::sync::atomic::{AtomicU64, Ordering};
        // Twice MIN_LEAF_ITEMS, so fewer than that remain after the
        // first poll and the fill cannot promote: it records one span,
        // the whole fill, timed on the counter.
        static SITE: crate::sched::call_site::CallSiteState =
            crate::sched::call_site::CallSiteState::new();
        let site = crate::sched::call_site::SiteRef::new(&SITE);
        let plan = JobPlan::new(6, 1).with_site(site);
        // The first read of the rate measures it with a spin, which would
        // otherwise land inside the wall time below.
        let rate_16 = tsc_per_ns_16();
        let n = 2 * MIN_LEAF_ITEMS;
        let mut buf: Vec<std::mem::MaybeUninit<u64>> = Vec::with_capacity(n);
        buf.resize_with(n, std::mem::MaybeUninit::uninit);
        let base = std::time::Instant::now();
        let first_ns = AtomicU64::new(0);
        let last_ns = AtomicU64::new(0);
        let item = |i: usize| {
            if i == 0 {
                first_ns.store(base.elapsed().as_nanos() as u64, Ordering::Relaxed);
            }
            let out = span_test_item(i);
            if i == n - 1 {
                last_ns.store(base.elapsed().as_nanos() as u64, Ordering::Relaxed);
            }
            out
        };
        let t0 = std::time::Instant::now();
        heartbeat_fill(&plan, &mut buf[..], 0, &item);
        let wall_ns = t0.elapsed().as_nanos() as u64;
        let inner_ns = last_ns.load(Ordering::Relaxed) - first_ns.load(Ordering::Relaxed);
        assert_span_in_nanoseconds(&SITE, rate_16, wall_ns, inner_ns);
    }

    #[test]
    fn a_heartbeat_site_of_unreadably_short_leaves_takes_slaw_at_any_size() {
        use crate::sched::call_site::{CallSiteState, PolicyArm};
        // Sixteen leaves of 200 ns: timed, and too short for a spread.
        static SHORT: CallSiteState = CallSiteState::new();
        SHORT.record_batch_site_only(16 * 200, 0, 16, 0, 0);
        assert_eq!(SHORT.cv2_per_mille(), None);
        let big = 4 * HEARTBEAT_GATE_ITEMS;
        assert_eq!(
            heartbeat_arm(&SHORT, SHORT.cv2_per_mille(), big),
            PolicyArm::Default
        );

        // A site with no leaves yet still takes the item-count gate.
        static FRESH: CallSiteState = CallSiteState::new();
        assert_eq!(heartbeat_arm(&FRESH, None, big), PolicyArm::Alternative);
        assert_eq!(
            heartbeat_arm(&FRESH, None, HEARTBEAT_GATE_ITEMS - 1),
            PolicyArm::Default
        );
    }

    #[test]
    fn a_token_bucket_span_reaches_its_site_in_nanoseconds() {
        use std::sync::atomic::{AtomicU64, Ordering};
        // Fewer items than one poll interval and no tokens, so the fill
        // checks neither trip and cannot promote: it records one span,
        // the whole fill, timed on the counter.
        static SITE: crate::sched::call_site::CallSiteState =
            crate::sched::call_site::CallSiteState::new();
        let site = crate::sched::call_site::SiteRef::new(&SITE);
        let plan = JobPlan::new(6, 1).with_site(site);
        let rate_16 = tsc_per_ns_16();
        let n = POLL_MASK;
        let mut buf: Vec<std::mem::MaybeUninit<u64>> = Vec::with_capacity(n);
        buf.resize_with(n, std::mem::MaybeUninit::uninit);
        let base = std::time::Instant::now();
        let first_ns = AtomicU64::new(0);
        let last_ns = AtomicU64::new(0);
        let item = |i: usize| {
            if i == 0 {
                first_ns.store(base.elapsed().as_nanos() as u64, Ordering::Relaxed);
            }
            let out = span_test_item(i);
            if i == n - 1 {
                last_ns.store(base.elapsed().as_nanos() as u64, Ordering::Relaxed);
            }
            out
        };
        let t0 = std::time::Instant::now();
        token_bucket_fill(&plan, &mut buf[..], 0, &item, &|_: usize| 0u32);
        let wall_ns = t0.elapsed().as_nanos() as u64;
        let inner_ns = last_ns.load(Ordering::Relaxed) - first_ns.load(Ordering::Relaxed);
        assert_span_in_nanoseconds(&SITE, rate_16, wall_ns, inner_ns);
    }

    /// Check that `site` holds `spans` spans of `items` items each and
    /// nothing else, and that they reached it with their item counts:
    /// the site has a per-item mean, and it is the site's summed time
    /// over every item the spans covered. Spans recorded without their
    /// items leave a site holding only spans with no item count, and no
    /// per-item mean at all.
    fn assert_spans_carry_items(
        site: &crate::sched::call_site::CallSiteState,
        spans: u64,
        items: u64,
    ) {
        assert_eq!(
            site.leaf_count(),
            spans,
            "each fill that cannot promote records one span"
        );
        let per_item = site
            .per_item_ns()
            .expect("spans that carry their items give the site a per-item mean");
        let covered = spans * items;
        let sum_ns = site.leaf_sum_ns();
        assert!(
            per_item.abs_diff(sum_ns / covered) <= 1,
            "{per_item} ns an item from {sum_ns} ns over {covered} items: the spans' time is \
             not divided by the items they covered"
        );
    }

    #[test]
    fn a_span_carries_the_items_it_covered() {
        use crate::sched::call_site::{CallSiteState, SiteRef};
        // Four fills or passes at each site, none able to promote or
        // split, so each site holds four spans and nothing else.
        const SPANS: u64 = 4;

        // More than MIN_LEAF_ITEMS, so the fill polls, and fewer than
        // twice that remain after its first poll, so it cannot promote.
        static HEARTBEAT: CallSiteState = CallSiteState::new();
        let plan = JobPlan::new(6, 1).with_site(SiteRef::new(&HEARTBEAT));
        let n = 2 * MIN_LEAF_ITEMS;
        for _ in 0..SPANS {
            let mut buf: Vec<std::mem::MaybeUninit<u64>> = Vec::with_capacity(n);
            buf.resize_with(n, std::mem::MaybeUninit::uninit);
            heartbeat_fill(&plan, &mut buf[..], 0, &span_test_item);
        }
        assert_spans_carry_items(&HEARTBEAT, SPANS, n as u64);

        // At most MIN_LEAF_ITEMS, the fill's serial branch.
        static SERIAL: CallSiteState = CallSiteState::new();
        let plan = JobPlan::new(6, 1).with_site(SiteRef::new(&SERIAL));
        let n = MIN_LEAF_ITEMS;
        for _ in 0..SPANS {
            let mut buf: Vec<std::mem::MaybeUninit<u64>> = Vec::with_capacity(n);
            buf.resize_with(n, std::mem::MaybeUninit::uninit);
            heartbeat_fill(&plan, &mut buf[..], 0, &span_test_item);
        }
        assert_spans_carry_items(&SERIAL, SPANS, n as u64);

        // Fewer items than one poll interval and no tokens, so the
        // bucket's fill checks neither trip.
        static BUCKET: CallSiteState = CallSiteState::new();
        let plan = JobPlan::new(6, 1).with_site(SiteRef::new(&BUCKET));
        let n = POLL_MASK;
        for _ in 0..SPANS {
            let mut buf: Vec<std::mem::MaybeUninit<u64>> = Vec::with_capacity(n);
            buf.resize_with(n, std::mem::MaybeUninit::uninit);
            token_bucket_fill(&plan, &mut buf[..], 0, &span_test_item, &|_: usize| 0u32);
        }
        assert_spans_carry_items(&BUCKET, SPANS, n as u64);

        // An input no longer than the chunk, the whole-input pass.
        static FIXED: CallSiteState = CallSiteState::new();
        let plan = JobPlan::new(6, 1).with_site(SiteRef::new(&FIXED));
        let n = 64;
        for _ in 0..SPANS {
            let mut v: Vec<u64> = (0..n as u64).collect();
            for_each_fixed_chunk(&plan, &mut v, n, |slice| {
                for x in slice.iter_mut() {
                    *x = span_test_item(*x as usize);
                }
            });
        }
        assert_spans_carry_items(&FIXED, SPANS, n as u64);
    }

    #[test]
    fn a_token_bucket_fill_with_no_tokens_promotes_once_a_quantum_has_passed() {
        use std::sync::atomic::{AtomicUsize, Ordering};
        // Items light enough that one poll window of POLL_MASK + 1 of
        // them takes a small part of a quantum, and enough of them that
        // the whole fill takes many quanta. With no tokens the time
        // trip is the only way out of the serial prefix.
        let item = |i: usize| {
            let mut acc = i as u64;
            for _ in 0..32 {
                acc = acc.wrapping_mul(0x9E37_79B9_7F4A_7C15).wrapping_add(1);
            }
            std::hint::black_box(acc)
        };
        let n = 1usize << 16;
        // The filler asks for an item's tokens only while it fills
        // serially, so the count of asks is the serial prefix's length.
        let asked = AtomicUsize::new(0);
        let tokens = |_: usize| {
            asked.fetch_add(1, Ordering::Relaxed);
            0u32
        };
        let plan = JobPlan::new(6, 1);
        let mut buf: Vec<std::mem::MaybeUninit<u64>> = Vec::with_capacity(n);
        buf.resize_with(n, std::mem::MaybeUninit::uninit);
        token_bucket_fill(&plan, &mut buf[..], 0, &item, &tokens);
        let serial = asked.load(Ordering::Relaxed);
        assert!(
            serial < n,
            "a fill of {n} items with no tokens stayed serial throughout: the time trip never \
             fired, so it is not counting from the span's start"
        );
        for (i, slot) in buf.iter().enumerate() {
            // SAFETY: the fill writes every slot, the serial prefix
            // directly and the promoted tail through collect_inner.
            assert_eq!(unsafe { slot.assume_init_read() }, item(i));
        }
    }

    #[test]
    fn the_heartbeat_quantum_is_its_stated_nanoseconds_once_the_rate_is_known() {
        // Measures the counter's rate unless this process already has,
        // after which the quantum is taken from it.
        let rate_16 = tsc_per_ns_16();
        let ticks = heartbeat_ticks();
        let (nanos, _) = LocalLeafBuffer::as_nanos(ticks, 0);
        assert!(
            nanos.abs_diff(HEARTBEAT_NS) <= HEARTBEAT_NS / 100,
            "the quantum is {ticks} ticks, {nanos} ns through the leaf path's conversion at \
             {rate_16} sixteenths of a tick a nanosecond, where it is stated as {HEARTBEAT_NS} ns"
        );
    }

    #[test]
    fn for_each_chunk_propagates_panic() {
        let mut v: Vec<u32> = (0..1000).collect();
        let plan = JobPlan::new(6, 1000);
        let r = std::panic::catch_unwind(std::panic::AssertUnwindSafe(|| {
            for_each_chunk(&plan, &mut v, |slice| {
                if slice.first() == Some(&0) {
                    panic!("intentional panic from first chunk");
                }
                for x in slice {
                    *x += 1;
                }
            });
        }));
        assert!(r.is_err(), "panic in any chunk must propagate to caller");
    }

    #[test]
    fn par_map_in_place_touches_every_element() {
        let mut items: Vec<u64> = (0..256).map(|i| i as u64).collect();
        let plan = JobPlan::new(8, 256);
        par_map_in_place(&plan, &mut items, |b| *b = b.wrapping_mul(3));
        for (i, &b) in items.iter().enumerate() {
            assert_eq!(b, (i as u64).wrapping_mul(3));
        }
    }

    #[test]
    fn par_zip_apply_pairs_lhs_and_rhs_by_index() {
        let mut lhs: Vec<u64> = (0..1024).collect();
        let rhs: Vec<u64> = (0..1024).map(|i| i * 7).collect();
        let plan = JobPlan::new(8, 1024);
        par_zip_apply(&plan, &mut lhs, &rhs, |a, b| *a = a.wrapping_add(*b));
        for (i, &value) in lhs.iter().enumerate().take(1024) {
            let expected = (i as u64).wrapping_add((i as u64) * 7);
            assert_eq!(value, expected, "index {i}");
        }
    }

    #[test]
    #[should_panic(expected = "matching slice lengths")]
    fn par_zip_apply_panics_on_mismatched_lengths() {
        let mut lhs: Vec<u64> = vec![0; 10];
        let rhs: Vec<u64> = vec![0; 20];
        let plan = JobPlan::new(8, 10);
        par_zip_apply(&plan, &mut lhs, &rhs, |_a, _b| {});
    }

    #[test]
    fn par_zip_apply_empty_is_noop() {
        let mut lhs: Vec<u64> = Vec::new();
        let rhs: Vec<u64> = Vec::new();
        let plan = JobPlan::new(8, 0);
        par_zip_apply(&plan, &mut lhs, &rhs, |_a, _b| panic!("must not run"));
        assert!(lhs.is_empty());
    }

    #[test]
    fn for_each_indexed_visits_every_index_exactly_once() {
        let n = 10_000usize;
        let seen: Vec<AtomicU64> = (0..n).map(|_| AtomicU64::new(0)).collect();
        let plan = JobPlan::new(6, n as u32);
        for_each_indexed(&plan, n, 1, |i| {
            seen[i].fetch_add(1, Ordering::Relaxed);
        });
        for (i, s) in seen.iter().enumerate() {
            assert_eq!(s.load(Ordering::Relaxed), 1, "index {i} visited {} times", s.load(Ordering::Relaxed));
        }
    }

    #[test]
    fn for_each_indexed_empty_is_noop() {
        let plan = JobPlan::new(6, 0);
        for_each_indexed(&plan, 0, 1, |_| panic!("must not run"));
    }

    #[test]
    fn for_each_chunk_ref_tiles_the_slice_once_at_the_requested_width() {
        let n = 4_099usize;
        let width = 64usize;
        let v: Vec<u32> = (0..n as u32).collect();
        let seen: Vec<AtomicU64> = (0..n).map(|_| AtomicU64::new(0)).collect();
        use std::sync::atomic::AtomicUsize;
        // One slot per tile, each written by the one chunk that covers
        // it, so nothing contends and nothing needs a lock. The length
        // is stored plus one, which leaves zero meaning never covered
        // and lets a tile claimed twice be caught where it happens
        // rather than inferred afterward from a list that still
        // sorts.
        let tiles = n.div_ceil(width);
        let lens: Vec<AtomicUsize> = (0..tiles).map(|_| AtomicUsize::new(0)).collect();
        let plan = JobPlan::new(6, n as u32);
        for_each_chunk_ref(&plan, &v, width, |start, chunk| {
            assert_eq!(start % width, 0, "chunk at {start} does not begin a tile");
            let prior = lens[start / width].swap(chunk.len() + 1, Ordering::Relaxed);
            assert_eq!(prior, 0, "tile {} was covered twice", start / width);
            for (k, &x) in chunk.iter().enumerate() {
                assert_eq!(x as usize, start + k, "chunk at {start} holds the wrong items");
                seen[start + k].fetch_add(1, Ordering::Relaxed);
            }
        });
        for (i, s) in seen.iter().enumerate() {
            assert_eq!(s.load(Ordering::Relaxed), 1, "item {i} covered {} times", s.load(Ordering::Relaxed));
        }
        for (k, slot) in lens.iter().enumerate() {
            let stored = slot.load(Ordering::Relaxed);
            assert_ne!(stored, 0, "tile {k} was never covered");
            let expected = if k + 1 == tiles { n - k * width } else { width };
            assert_eq!(stored - 1, expected, "chunk {k} has the wrong width");
        }
    }

    #[test]
    fn for_each_chunk_ref_empty_is_noop() {
        let v: Vec<u32> = Vec::new();
        let plan = JobPlan::new(6, 0);
        for_each_chunk_ref(&plan, &v, 16, |_, _| panic!("must not run"));
    }

    /// A worker cap of one runs the body on the calling thread at every
    /// data-parallel entry, each named here and each run under a plan
    /// that carries its own per-item estimate and under one that does
    /// not, since several entries branch on the estimate first.
    #[test]
    fn a_worker_cap_of_one_keeps_every_entry_on_the_calling_thread() {
        use std::sync::atomic::AtomicUsize;
        let caller = std::thread::current().id();
        // Counters rather than the thread ids themselves: the assertion
        // is that no body ran anywhere but here, and a count answers it.
        let bodies = AtomicUsize::new(0);
        let elsewhere = AtomicUsize::new(0);
        let note = || {
            bodies.fetch_add(1, Ordering::Relaxed);
            if std::thread::current().id() != caller {
                elsewhere.fetch_add(1, Ordering::Relaxed);
            }
        };
        let n = 4096usize;
        let a: Vec<u32> = vec![1; n];
        let b: Vec<u32> = vec![2; n];
        let estimated = JobPlan::new(6, n as u32).with_workers(1).with_estimated_per_item_ns(5_000);
        let bare = JobPlan::new(6, n as u32).with_workers(1);

        let mut ran = 0usize;
        let mut idle: Vec<String> = Vec::new();
        let mut left: Vec<String> = Vec::new();
        for (label, plan) in [("estimated", &estimated), ("bare", &bare)] {
            let entries: [(&str, &dyn Fn()); 16] = [
                ("for_each_chunk", &|| {
                    let mut v = a.clone();
                    for_each_chunk(plan, &mut v, |c| {
                        note();
                        c.iter_mut().for_each(|x| *x += 1);
                    });
                }),
                ("for_each_chunk_min_leaf", &|| {
                    let mut v = a.clone();
                    for_each_chunk_min_leaf(plan, &mut v, 1, |c| {
                        note();
                        c.iter_mut().for_each(|x| *x += 1);
                    });
                }),
                ("for_each_fixed_chunk", &|| {
                    let mut v = a.clone();
                    for_each_fixed_chunk(plan, &mut v, 64, |c| {
                        note();
                        c.iter_mut().for_each(|x| *x += 1);
                    });
                }),
                ("for_each_chunk_triple", &|| {
                    let mut out = vec![0u32; n];
                    for_each_chunk_triple(plan, &mut out, &a, &b, |o, x, y| {
                        note();
                        for ((o, x), y) in o.iter_mut().zip(x).zip(y) {
                            *o = x + y;
                        }
                    });
                    assert_eq!(out[n - 1], 3, "the triple entry still computed its output");
                }),
                ("for_each_chunk_triple_min_leaf", &|| {
                    let mut out = vec![0u32; n];
                    for_each_chunk_triple_min_leaf(plan, &mut out, &a, &b, 1, |o, x, y| {
                        note();
                        for ((o, x), y) in o.iter_mut().zip(x).zip(y) {
                            *o = x + y;
                        }
                    });
                    assert_eq!(out[0], 3, "the triple entry still computed its output");
                }),
                ("for_each_chunk_indexed", &|| {
                    let mut idx = vec![0u32; n];
                    for_each_chunk_indexed(plan, &mut idx, |start, chunk| {
                        note();
                        for (k, slot) in chunk.iter_mut().enumerate() {
                            *slot = (start + k) as u32;
                        }
                    });
                    assert_eq!(idx[n - 1], (n - 1) as u32, "the indexed entry filled its output");
                }),
                ("for_each_chunk_indexed_min_leaf", &|| {
                    let mut idx = vec![0u32; n];
                    for_each_chunk_indexed_min_leaf(plan, &mut idx, 1, |start, chunk| {
                        note();
                        for (k, slot) in chunk.iter_mut().enumerate() {
                            *slot = (start + k) as u32;
                        }
                    });
                    assert_eq!(idx[n - 1], (n - 1) as u32, "the indexed entry filled its output");
                }),
                ("for_each_indexed", &|| {
                    for_each_indexed(plan, n, 1, |_i| note());
                }),
                ("for_each_chunk_ref", &|| {
                    for_each_chunk_ref(plan, &a, 64, |_start, _chunk| note());
                }),
                ("collect_indexed", &|| {
                    let got: Vec<usize> = collect_indexed(plan, n, 1, |i| {
                        note();
                        i
                    });
                    assert_eq!(got[n - 1], n - 1, "collect_indexed kept its order");
                }),
                ("collect_indexed_heartbeat", &|| {
                    let got: Vec<usize> = collect_indexed_heartbeat(plan, n, |i| {
                        note();
                        i
                    });
                    assert_eq!(got[n - 1], n - 1, "the heartbeat collect kept its order");
                }),
                ("collect_indexed_token_bucket", &|| {
                    let got: Vec<usize> = collect_indexed_token_bucket(
                        plan,
                        n,
                        |_i| 1_000,
                        |i| {
                            note();
                            i
                        },
                    );
                    assert_eq!(got[n - 1], n - 1, "the token-bucket collect kept its order");
                }),
                ("collect_indexed_tiny_tasks", &|| {
                    let got: Vec<usize> = collect_indexed_tiny_tasks(plan, n, |i| {
                        note();
                        i
                    });
                    assert_eq!(got[n - 1], n - 1, "the tiny-tasks collect kept its order");
                }),
                ("reduce_chunks", &|| {
                    let sum = reduce_chunks(
                        plan,
                        &a,
                        || 0u64,
                        |acc, s| {
                            note();
                            acc + s.iter().map(|&x| u64::from(x)).sum::<u64>()
                        },
                        |x, y| x + y,
                    );
                    assert_eq!(sum, n as u64, "reduce_chunks folded every element");
                    assert_eq!(last_reduce_chunks_path(), Some(ReduceChunksPath::Caller));
                }),
                ("par_map_in_place", &|| {
                    let mut v = a.clone();
                    par_map_in_place(plan, &mut v, |x| {
                        note();
                        *x += 1;
                    });
                }),
                ("par_zip_apply", &|| {
                    let mut v = a.clone();
                    par_zip_apply(plan, &mut v, &b, |x, y| {
                        note();
                        *x += y;
                    });
                    assert_eq!(v[n - 1], 3, "par_zip_apply paired every index");
                }),
            ];
            for (name, run) in entries {
                let bodies_before = bodies.load(Ordering::Relaxed);
                let elsewhere_before = elsewhere.load(Ordering::Relaxed);
                run();
                ran += 1;
                if bodies.load(Ordering::Relaxed) == bodies_before {
                    idle.push(format!("{name} ({label})"));
                }
                if elsewhere.load(Ordering::Relaxed) != elsewhere_before {
                    left.push(format!("{name} ({label})"));
                }
            }
        }
        assert_eq!(ran, 32, "every entry ran under both plans");
        assert!(idle.is_empty(), "these entries never ran their body: {idle:?}");
        assert!(
            left.is_empty(),
            "a worker cap of one must not leave the caller, and these did: {left:?}"
        );
    }

    /// A body the worker cap put on the calling thread says nothing
    /// about the caller's estimate, so however long it runs it leaves
    /// the site's collapse latch alone. The estimate here would admit
    /// the collapse on its own, and the cap is what decided.
    #[test]
    fn a_capped_body_past_the_collapse_threshold_leaves_the_latch_alone() {
        static SITE: crate::sched::call_site::CallSiteState =
            crate::sched::call_site::CallSiteState::new();
        let site = crate::sched::call_site::SiteRef::new(&SITE);
        let limit = inline_collapse_threshold_ns();
        let n = 64usize;
        let mut v = vec![0u64; n];
        let plan = JobPlan::new(6, n as u32)
            .with_site(site)
            .with_workers(1)
            .with_estimated_per_item_ns(1);
        for_each_chunk_indexed_min_leaf(&plan, &mut v, 1, |start, chunk| {
            let t0 = std::time::Instant::now();
            while (t0.elapsed().as_nanos() as u64) <= limit {
                std::hint::spin_loop();
            }
            for (k, slot) in chunk.iter_mut().enumerate() {
                *slot = (start + k) as u64;
            }
        });
        assert_eq!(v[n - 1], (n - 1) as u64, "the capped body ran");
        assert!(
            !SITE.collapse_overran(),
            "a body the cap put on the caller ran past {limit} ns and latched the site"
        );
    }

    /// The caller's own oversubscription factor decides the split
    /// budget, in place of the process-global observer multiplier.
    /// The seed-depth route takes the caller's factor and nobody
    /// else's, so a plan carrying one seeds that many more leaves and a
    /// plan whose factor came from a profile seeds what the cost model
    /// chose.
    #[test]
    fn an_explicit_oversubscription_factor_shifts_the_seed_depth() {
        const ITEMS: usize = 65_536;
        const ITEMS_LOG2: usize = 16;
        const WORKERS: usize = 24;

        // No site, so the hysteresis stabilizer is out of the way and
        // the depth is a function of the plan alone.
        let bare = JobPlan::new(6, ITEMS as u32).with_estimated_per_item_ns(500);
        assert!(!bare.oversubscription_log2_explicit);
        let plain = adaptive_seed_depth(&bare, ITEMS, WORKERS);

        for log2 in 0..=3u8 {
            let pinned = JobPlan::new(6, ITEMS as u32)
                .with_estimated_per_item_ns(500)
                .with_oversubscription_log2(log2);
            assert!(pinned.oversubscription_log2_explicit);
            assert_eq!(
                adaptive_seed_depth(&pinned, ITEMS, WORKERS),
                (plain + log2 as usize).min(ITEMS_LOG2),
                "a factor of {log2} shifts the depth by {log2}",
            );
        }

        // A profile carries a factor as a routing default rather than as
        // the caller's opinion, so it leaves the depth where it was.
        let profiled = JobPlan::set_profile(
            6,
            ITEMS as u32,
            crate::dispatch_profile::DispatchProfile::LatencyBound,
        )
        .with_estimated_per_item_ns(500);
        assert!(profiled.oversubscription_log2.is_some());
        assert_eq!(adaptive_seed_depth(&profiled, ITEMS, WORKERS), plain);
    }

    #[test]
    fn an_explicit_oversubscription_factor_sets_the_split_budget() {
        let plan = JobPlan::new(6, 1024);
        assert!(!plan.oversubscription_log2_explicit, "a bare plan defers to the observer");
        let observer = crate::sched::split_observer::split_multiplier() as usize;
        assert_eq!(plan.effective_leaves_per_worker(), observer.max(1));

        for log2 in 0..=3u8 {
            let pinned = JobPlan::new(6, 1024).with_oversubscription_log2(log2);
            assert!(pinned.oversubscription_log2_explicit);
            assert_eq!(pinned.effective_leaves_per_worker(), 1usize << log2,
                "an explicit factor of {log2} means {} leaves per worker", 1usize << log2);
        }

        // A profile sets the factor as a routing default, which still
        // defers to the observer: only the caller's own value wins.
        let profiled = JobPlan::set_profile(
            6, 1024, crate::dispatch_profile::DispatchProfile::LatencyBound);
        assert!(profiled.oversubscription_log2.is_some(), "the profile carries a default");
        assert!(!profiled.oversubscription_log2_explicit, "but the caller set nothing");
        assert_eq!(profiled.effective_leaves_per_worker(), observer.max(1));

        // A declared shape is the caller describing the workload, so
        // a factor the shape carries counts as theirs. The plan's own
        // field cannot answer this: `JobPlan::new` already fills it
        // from the classifier, so the shape's hints are what say
        // whether the shape contributed one.
        use crate::sched::workload_shape::WorkloadShape;
        for shape in [
            WorkloadShape::Streaming,
            WorkloadShape::ProducerFast { burst: 64 },
            WorkloadShape::WorkSteal { n_consumers: 8, batch_size: 1024 },
            WorkloadShape::Cooperative { n_cores: 8 },
            WorkloadShape::VariantRace { n_variants: 3 },
        ] {
            let shaped = JobPlan::new(6, 1024).with_workload_shape(shape);
            match shape.hints().oversubscription_log2 {
                Some(log2) => {
                    assert!(shaped.oversubscription_log2_explicit,
                        "{shape:?} carries a factor, so it is the caller's");
                    assert_eq!(shaped.effective_leaves_per_worker(), 1usize << log2,
                        "{shape:?} sets the split budget");
                }
                None => assert!(!shaped.oversubscription_log2_explicit,
                    "{shape:?} carries no factor, so the observer still decides"),
            }
        }
    }

    /// A body admitted inline by an estimate that reads far too low
    /// latches its site, and later calls dispatch instead.
    #[test]
    fn a_collapsed_body_that_overruns_stops_its_site_collapsing() {
        use crate::sched::call_site::{CallSiteState, SiteRef};
        static SITE: CallSiteState = CallSiteState::new();
        let site = SiteRef::new(&SITE);
        let threshold = host_dispatch_profile().collapse_threshold_ns;
        assert!(threshold > 0, "measured by the query above");
        assert!(!SITE.collapse_overran(), "a fresh site has not overrun");

        // One nanosecond an item puts any size under the threshold,
        // while the body sleeps well past it.
        let n = 64usize;
        let mut items = vec![0u8; n];
        let plan = JobPlan::new(0, n as u32)
            .with_site(site)
            .with_estimated_per_item_ns(1);
        assert!(collapses_inline(&plan, n), "the estimate admits it inline");

        let over = std::time::Duration::from_nanos(threshold.saturating_mul(4));
        for_each_chunk(&plan, &mut items, |chunk| {
            std::thread::sleep(over);
            for x in chunk.iter_mut() {
                *x = x.wrapping_add(1);
            }
        });

        assert!(SITE.collapse_overran(), "the overrun latched the site");
        assert!(!collapses_inline(&plan, n),
            "the same plan no longer collapses at this site");
    }

    #[test]
    fn a_plan_without_both_hints_has_no_model_width_and_keeps_the_caller_floor() {
        let n = 4096usize;
        assert_eq!(model_leaf(&JobPlan::new(6, n as u32), n), None);
        assert_eq!(
            model_leaf(&JobPlan::new(6, n as u32).with_estimated_per_item_ns(50), n),
            None,
            "a per-item cost alone leaves the model without a fixed cost"
        );
        assert_eq!(
            model_leaf(&JobPlan::new(6, n as u32).with_task_overhead_ns(500), n),
            None,
            "an overhead alone leaves the model without work to spread"
        );
    }

    #[test]
    fn an_overhead_alone_is_inert_and_a_per_item_cost_alone_is_not() {
        let n = 4096usize;
        // Both reach no model width, but they are not the same case and
        // the reporting treats them differently.
        let overhead_only = JobPlan::new(6, n as u32).with_task_overhead_ns(500);
        assert_eq!(model_leaf(&overhead_only, n), None);
        assert!(
            !overhead_only.estimated_per_item_ns_explicit,
            "an overhead alone leaves the cost unstated, which is the reported case"
        );

        let cost_only = JobPlan::new(6, n as u32).with_estimated_per_item_ns(50);
        assert_eq!(model_leaf(&cost_only, n), None);
        assert!(
            cost_only.task_overhead_ns.is_none(),
            "a cost alone leaves no overhead, which is not reported: the cost \
             still steers the tier pick, the collapse decision and the leaf floor"
        );
        // The cost alone does steer something, which is why it is not
        // reported as a dropped hint.
        assert!(cost_only.estimated_total_ns().is_some());
    }

    #[test]
    fn a_classifier_default_cost_is_not_a_stated_one_and_yields_no_model_width() {
        let n = 4096usize;
        // set_profile fills estimated_per_item_ns from the profile's
        // default without marking it stated, which is the case that
        // must not reach the model.
        let plan = JobPlan::set_profile(
            6,
            n as u32,
            crate::dispatch_profile::DispatchProfile::MemoryBound,
        )
        .with_task_overhead_ns(500);
        assert!(
            plan.estimated_per_item_ns.is_some(),
            "the profile supplies a default, which is what makes this the interesting case"
        );
        assert!(!plan.estimated_per_item_ns_explicit);
        assert_eq!(model_leaf(&plan, n), None);

        // The same plan with the cost stated does reach the model.
        assert!(model_leaf(&plan.with_estimated_per_item_ns(50), n).is_some());
    }

    #[test]
    fn the_model_width_grows_with_overhead_and_shrinks_with_per_item_cost() {
        let n = 4096usize;
        let width = |per_item: u32, overhead: u32| {
            model_leaf(
                &JobPlan::new(6, n as u32)
                    .with_estimated_per_item_ns(per_item)
                    .with_task_overhead_ns(overhead),
                n,
            )
            .expect("both hints are set")
        };

        // A dearer chunk is worth fewer of, so each carries more items.
        assert!(width(50, 5_000) > width(50, 500));
        // A dearer item pays for a chunk sooner, so each carries fewer.
        assert!(width(500, 500) < width(50, 500));
        // The span is the other half of a chunk's fixed cost and moves
        // the width the same way the overhead does.
        let with_span = model_leaf(
            &JobPlan::new(6, n as u32)
                .with_estimated_per_item_ns(50)
                .with_task_overhead_ns(500)
                .with_task_span_ns(4_500),
            n,
        )
        .expect("both hints are set");
        assert_eq!(with_span, width(50, 5_000));
    }

    #[test]
    fn a_fixed_cost_that_swamps_the_work_collapses_the_width_to_the_whole_run() {
        let n = 64usize;
        let one_chunk = model_leaf(
            &JobPlan::new(6, n as u32)
                .with_estimated_per_item_ns(100_000)
                .with_task_overhead_ns(u32::MAX),
            n,
        )
        .expect("both hints are set");
        assert_eq!(one_chunk, n, "one chunk over n items is n items wide");

        // And the opposite end: a negligible overhead against the same
        // items asks for a chunk per item.
        let per_item = model_leaf(
            &JobPlan::new(6, n as u32)
                .with_estimated_per_item_ns(100_000)
                .with_task_overhead_ns(1),
            n,
        )
        .expect("both hints are set");
        assert_eq!(per_item, 1);
    }
}