shep-daemon 0.25.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
951
952
953
954
955
956
957
958
959
960
961
962
963
964
965
966
967
968
969
970
971
972
973
974
975
976
977
978
979
980
981
982
983
984
985
986
987
988
989
990
991
992
//! Tests for what a load does to armed subsystems and to instance counts.
//!
//! Changing an extras field has to rearm the sheep against the new spec. A
//! plain load never scales and a reset does, and a change that cannot be parked
//! must not be reported as parked.

use super::config_fixtures::{
    APPLY_FIRST_PID, actor_over, apply_config, blue_env_file, declared_app, established,
};
use super::*;

/// Park `parked` on every instance of `name`, so a case parks the flock it
/// has rather than a count spelled out at the call site.
///
/// # Panics
///
/// Panics when the flock holds no instance named `name`.
#[track_caller]
fn park_on_all(actor: &mut Actor<ScriptedRunner>, name: &str, parked: &ResolvedApp) {
    let mut parked_on = 0;
    for slot in actor.sheep.values_mut() {
        if slot.entry.spec.config().name == name {
            slot.entry.pending = Some(parked.clone());
            parked_on += 1;
        }
    }
    assert!(
        parked_on > 0,
        "no instance named {name} to park on, out of {:?}",
        actor
            .sheep
            .values()
            .map(|slot| slot.entry.spec.config().name.as_str())
            .collect::<Vec<_>>(),
    );
}

/// `max_memory`, `watch` and the cron pair are read when a worker is
/// armed, so a spec write alone leaves the old value enforced for as long
/// as that arming lives.
#[tokio::test(start_paused = true)]
async fn a_changed_extras_field_rearms_the_name() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, enforcer) = actor_over(
        &dir,
        &[app_with("web", |app| {
            app.max_memory = Some(MemSize::from_bytes(100 << 20));
        })],
    );

    let mut file = AppConfig::minimal("web", "./srv");
    file.max_memory = Some(MemSize::from_bytes(512 << 20));
    apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "max_memory"])],
        ResetDepth::None,
    )
    .await;

    assert_eq!(
        enforcer
            .arms()
            .last()
            .expect("a changed ceiling re-arms the name")
            .limit,
        MemSize::from_bytes(512 << 20),
        "the registry was re-armed with the old ceiling"
    );
}

/// `PollingEnforcer` computes a breach under its lock and sends it after
/// releasing that lock, so a re-arm landing in between leaves a report in
/// flight speaking for a limit nobody enforces. Reachable only because a
/// load re-arms an id that is already armed.
#[tokio::test(start_paused = true)]
async fn a_breach_measured_under_a_since_raised_ceiling_does_not_restart() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(
        &dir,
        &[app_with("web", |app| {
            app.max_memory = Some(MemSize::from_bytes(100 << 20));
        })],
    );

    let mut file = AppConfig::minimal("web", "./srv");
    file.max_memory = Some(MemSize::from_bytes(512 << 20));
    apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "max_memory"])],
        ResetDepth::None,
    )
    .await;

    // Over the ceiling that was armed when the sample was taken, well
    // under the one the load just put in force.
    actor.handle_extra_restart(
        0,
        APPLY_FIRST_PID,
        None,
        Some(MemSize::from_bytes(200 << 20)),
    );

    let slot = &actor.sheep[&0];
    assert!(
        slot.manual.is_none(),
        "a breach against a ceiling the operator has raised must never claim the manual marker"
    );
    assert_eq!(slot.entry.pid, Some(APPLY_FIRST_PID));
}

/// The merge builds on the app's intended config, the parked one when
/// there is one, not on what the running child was spawned from: on the
/// second load the key is established, a plain load skips it, and a merge
/// based on the running config would carry the old value forward over the
/// parked one.
#[tokio::test(start_paused = true)]
async fn a_second_load_of_the_same_file_keeps_the_first_loads_parked_config() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);

    let first = apply_config(&mut actor, vec![blue_env_file()], ResetDepth::None).await;
    assert_eq!(first[0].pending, vec!["env".to_string()]);

    let second = apply_config(&mut actor, vec![blue_env_file()], ResetDepth::None).await;

    assert_eq!(
        actor.sheep[&0]
            .entry
            .pending
            .as_ref()
            .expect("the parked config survives a second load")
            .config()
            .env
            .get("MODE")
            .map(String::as_str),
        Some("blue"),
        "the second load erased what the first parked"
    );
    assert_eq!(
        second[0]
            .app
            .as_ref()
            .expect("an applied app is recorded")
            .config()
            .env
            .get("MODE")
            .map(String::as_str),
        Some("blue"),
        "the recorded app lost it too, so a reboot would come up without it"
    );
    assert!(
        second[0].pending.is_empty(),
        "the second load changes nothing that was not already coming: {second:?}"
    );
}

/// An instance with no live task is deregistered synchronously inside
/// `handle_scale`, so the id list read before the scale can name a slot
/// that is already gone by the time the spec write walks it.
#[tokio::test(start_paused = true)]
async fn a_scale_down_removing_a_non_running_instance_does_not_panic() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
    // One instance exited and never came back. No `ctl`, so its delete
    // resolves on the spot rather than through a kill ladder.
    let stopped = actor.sheep.get_mut(&1).expect("the fixture registers two");
    stopped.entry.status = ProcStatus::Stopped;
    stopped.entry.pid = None;

    let mut file = AppConfig::minimal("web", "./srv");
    file.instances = 1;
    let reply = apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "instances"])],
        ResetDepth::Policy,
    )
    .await;

    assert_eq!(actor.ids_of_name("web"), vec![0]);
    assert_eq!(reply[0].applied, vec!["instances".to_string()]);
    assert_eq!(actor.sheep[&0].entry.spec.config().instances, 1);
}

/// `Applied::refused` with two empty lists promises nothing happened, so a
/// reply with `app` as `None` leaves the muster roll on the old count while
/// a second instance runs. A file declaring `watch` and `cwd` together
/// merges cleanly and cannot be reached by a running instance, since `cwd`
/// needs a respawn and `watch` does not.
#[tokio::test(start_paused = true)]
async fn a_load_that_scales_and_cannot_reach_the_running_spec_reports_what_landed() {
    let root = tempfile::tempdir().unwrap();
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);

    let mut file = AppConfig::minimal("web", "./srv");
    file.instances = 2;
    file.watch = true;
    file.cwd = Some(root.path().display().to_string());
    let reply = apply_config(
        &mut actor,
        vec![declared_app(
            file,
            &["name", "script", "instances", "watch", "cwd"],
        )],
        ResetDepth::Policy,
    )
    .await;

    assert_eq!(
        actor.ids_of_name("web").len(),
        2,
        "the scale really did happen"
    );
    assert!(
        reply[0].refused.is_none(),
        "a merge that normalizes is not an invalid file: {reply:?}"
    );
    assert!(
        reply[0].app.is_some(),
        "the muster roll must not be left on the pre-load config: {reply:?}"
    );
    assert_eq!(reply[0].applied, vec!["instances".to_string()]);
    assert!(
        reply[0].pending.contains(&"watch".to_string())
            && reply[0].pending.contains(&"cwd".to_string()),
        "a change no running instance can take must park, not vanish: {reply:?}"
    );
    assert!(
        actor.sheep[&0].entry.pending.is_some(),
        "and it must be parked on the entry, not only reported"
    );
}

/// Group membership is decided by the config rather than by what is
/// running, and both group triggers restart every member, so arming a
/// stopped instance lets a cron occurrence or a file save start it again.
/// Nothing heals that: the member is terminal, so no transition calls
/// `disarm` for it.
#[tokio::test(start_paused = true)]
async fn a_load_does_not_arm_a_stopped_instance() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(
        &dir,
        &[app_with("web", |app| {
            app.cron_restart = Some("0 * * * *".to_string());
        })],
    );
    let stopped = actor.sheep.get_mut(&0).expect("the fixture registers one");
    stopped.entry.status = ProcStatus::Stopped;
    stopped.entry.pid = None;

    let mut file = AppConfig::minimal("web", "./srv");
    file.cron_restart = Some("*/5 * * * *".to_string());
    apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "cron_restart"])],
        ResetDepth::None,
    )
    .await;

    assert_eq!(
        actor.registry.group_members("web"),
        None,
        "a load armed a schedule that will start a sheep the operator stopped"
    );
}

/// A plain load skips the count and says why; a `--reset` takes it. The
/// override store cannot tell a stocked count from an untouched one, so a
/// plain load acting on the field would delete instances.
#[tokio::test(start_paused = true)]
async fn a_plain_load_never_scales_and_a_reset_does() {
    let file = || {
        let mut file = AppConfig::minimal("web", "./srv");
        file.instances = 1;
        declared_app(file, &["name", "script", "instances"])
    };

    let plain_dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) =
        actor_over(&plain_dir, &[app_with("web", |app| app.instances = 2)]);
    let reply = apply_config(&mut actor, vec![file()], ResetDepth::None).await;
    assert_eq!(
        actor.ids_of_name("web").len(),
        2,
        "a plain load deleted an instance"
    );
    assert!(!reply[0].applied.contains(&"instances".to_string()));
    assert_eq!(
        reply[0].refused.as_deref(),
        Some(
            "instances: this load never reshapes a flock; no mode scales without also \
             putting back every setting the file declares, and `--reset=file` is the \
             narrowest that does, taking the file's count of 1"
        ),
        "the refusal must name a mode an operator can actually type, \
         not the bare flag `shep start --reset` now refuses on its \
         own, and it must name what following the advice costs: {reply:?}"
    );
    assert_eq!(
        reply[0]
            .app
            .as_ref()
            .expect("an applied app is recorded")
            .config()
            .instances,
        2,
        "the recorded count must be the one really running"
    );

    let reset_dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) =
        actor_over(&reset_dir, &[app_with("web", |app| app.instances = 2)]);
    let reply = apply_config(&mut actor, vec![file()], ResetDepth::Policy).await;
    assert_eq!(actor.ids_of_name("web"), vec![0], "--reset must take it");
    assert_eq!(reply[0].applied, vec!["instances".to_string()]);
}

/// All eight [`EXTRAS_FIELDS`] are read when a worker is armed, so a spec
/// write alone leaves the old value enforced for the life of that worker.
/// The observable is the memory ceiling, unchanged across these apps: a
/// re-arm arms every instance, so any arming recorded is proof of one.
#[tokio::test(start_paused = true)]
async fn every_extras_field_triggers_a_rearm() {
    let root = tempfile::tempdir().unwrap();
    let cwd = root.path().display().to_string();
    let ceiling = MemSize::from_bytes(100 << 20);
    // The "before" values every case edits away from.
    let base = |cwd: &str| {
        let cwd = cwd.to_string();
        move |app: &mut AppConfig| {
            app.max_memory = Some(ceiling);
            app.cwd = Some(cwd.clone());
            app.watch = true;
            app.ignore_watch = vec!["target/**".to_string()];
            app.watch_delay = Some(UpDuration::from_millis(1000));
            app.watch_options = vec!["src/**".to_string()];
            app.cron_restart = Some("0 * * * *".to_string());
            app.cron_timezone = Some("UTC".to_string());
            // An interval no paused-clock case advances to, so the probe
            // never runs and this stays about arming.
            app.liveness_probe = Some(ProbeConfig {
                failure_threshold: 3,
                interval: UpDuration::from_millis(600_000),
                timeout: UpDuration::from_millis(1000),
                ..probe_config(ProbeKind::Tcp, "127.0.0.1:1")
            });
        }
    };
    // One edit per entry in `EXTRAS_FIELDS`, named by the field it moves,
    // so a field dropped from that list goes red under its own name.
    type Edit = (&'static str, fn(&mut AppConfig));
    let edits: Vec<Edit> = vec![
        ("max_memory", |app| {
            app.max_memory = Some(MemSize::from_bytes(512 << 20));
        }),
        ("watch", |app| app.watch = false),
        ("ignore_watch", |app| {
            app.ignore_watch = vec!["dist/**".to_string()];
        }),
        ("watch_delay", |app| {
            app.watch_delay = Some(UpDuration::from_millis(2500));
        }),
        ("watch_options", |app| {
            app.watch_options = vec!["lib/**".to_string()];
        }),
        ("cron_restart", |app| {
            app.cron_restart = Some("*/5 * * * *".to_string());
        }),
        ("cron_timezone", |app| {
            app.cron_timezone = Some("Europe/Berlin".to_string());
        }),
        ("liveness_probe", |app| app.liveness_probe = None),
    ];

    for (field, edit) in edits {
        let dir = tempfile::tempdir().unwrap();
        let (mut actor, enforcer) = actor_over(&dir, &[app_with("web", base(&cwd))]);
        let mut file = AppConfig::minimal("web", "./srv");
        base(&cwd)(&mut file);
        edit(&mut file);
        let reply = apply_config(
            &mut actor,
            vec![declared_app(
                file,
                &[
                    "name",
                    "script",
                    "cwd",
                    "max_memory",
                    "watch",
                    "ignore_watch",
                    "watch_delay",
                    "watch_options",
                    "cron_restart",
                    "cron_timezone",
                    "liveness_probe",
                ],
            )],
            ResetDepth::Policy,
        )
        .await;
        assert!(
            reply[0].applied.contains(&field.to_string()),
            "{field} did not apply at all: {reply:?}"
        );
        assert!(
            !enforcer.arms().is_empty(),
            "changing {field} left the armed worker on the old value"
        );
    }
}

/// A name whose instances are all momentarily non-`Online`, each inside a
/// crash-restart backoff, has nothing to arm, and an early return there
/// would skip the teardown too. `disarm_extras` leaves a `WaitingRestart`
/// sheep armed, so it stays a group member and the group is never torn
/// down. Arms first, which makes this a test of the teardown;
/// `a_load_does_not_arm_a_stopped_instance` arms nothing, so its assertion
/// holds either way.
#[tokio::test(start_paused = true)]
async fn a_load_tears_down_a_group_whose_instances_are_all_down() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(
        &dir,
        &[app_with("web", |app| {
            app.cron_restart = Some("0 * * * *".to_string());
        })],
    );
    actor.arm_extras(0);
    assert_eq!(
        actor.registry.group_members("web"),
        Some(vec![0]),
        "the fixture must really be armed, or this pins nothing"
    );

    // Mid-backoff: no pid, not online, still registered and still a group
    // member.
    let waiting = actor.sheep.get_mut(&0).expect("the fixture registers one");
    waiting.entry.status = ProcStatus::WaitingRestart;
    waiting.entry.pid = None;

    let mut file = AppConfig::minimal("web", "./srv");
    file.cron_restart = Some("*/5 * * * *".to_string());
    let reply = apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "cron_restart"])],
        ResetDepth::None,
    )
    .await;

    assert_eq!(reply[0].applied, vec!["cron_restart".to_string()]);
    assert_eq!(
        actor.registry.group_members("web"),
        None,
        "a worker built from the replaced config survived a load that reported the \
         field as applied"
    );
}

/// The merge normalizes against the count the intended config carries, and
/// the flock can be running a different one, where the same config
/// refuses. The earlier parked config is left alone, still the one a
/// respawn picks up, so the report must not claim this load's fields are
/// in it.
#[tokio::test(start_paused = true)]
async fn a_change_that_cannot_be_parked_is_not_reported_as_parked() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
    // An earlier load parked a one-instance config. Two are running.
    let earlier = app_with("web", |app| {
        app.instances = 1;
        app.env = BTreeMap::from([("MODE".to_string(), "blue".to_string())]);
    });
    park_on_all(&mut actor, "web", &earlier);

    // One explicit log path, no `{{instance}}` and no `merge_logs`: legal
    // for the one instance the parked config declares, refused for the
    // two really running.
    let mut file = AppConfig::minimal("web", "./srv");
    file.out_file = Some("/tmp/web.log".to_string());
    let reply = apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "out_file"])],
        ResetDepth::None,
    )
    .await;

    assert!(
        !reply[0].pending.contains(&"out_file".to_string()),
        "a field that went nowhere must not be reported as coming: {reply:?}"
    );
    assert!(
        reply[0]
            .refused
            .as_deref()
            .is_some_and(|why| why.contains("out_file")),
        "and the operator must be told which field it was: {reply:?}"
    );
    assert_eq!(
        actor.sheep[&0]
            .entry
            .pending
            .as_ref()
            .expect("the earlier load's parked config survives")
            .config()
            .env
            .get("MODE")
            .map(String::as_str),
        Some("blue"),
        "the earlier load's parked config must not be cleared"
    );
    assert!(
        reply[0].app.is_none(),
        "there is no config that holds at the count really running, so nothing may be \
         recorded: {reply:?}"
    );
}

/// A plain load holds `instances` out of the merge, so the sibling case
/// above does not cover the depth every `shep start` uses. Two running
/// instances plus one shared explicit `out_file` is the reachable shape.
#[tokio::test(start_paused = true)]
async fn a_plain_load_whose_merge_cannot_normalize_refuses_and_touches_nothing() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);

    let mut file = AppConfig::minimal("web", "./srv");
    file.out_file = Some("/tmp/web.log".to_string());
    let reply = apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "out_file"])],
        ResetDepth::None,
    )
    .await;

    assert!(reply[0].refused.is_some(), "{reply:?}");
    assert!(reply[0].applied.is_empty() && reply[0].pending.is_empty());
    assert!(reply[0].app.is_none());
    assert!(actor.sheep[&0].entry.spec.config().out_file.is_none());
    assert!(actor.sheep[&0].entry.pending.is_none());
    assert_eq!(actor.ids_of_name("web").len(), 2);
}

/// `parked_wanted` is set by an earlier parked config as well as by this
/// load's own `NeedsRespawn` drift, so a load whose only drift is a Live
/// field can fail the rebuild with no field of its own to name. The stale
/// parked config then puts the old value back at the next respawn.
/// Reachable whenever the instance count moved between two loads.
#[tokio::test(start_paused = true)]
async fn a_parked_config_that_cannot_be_rebuilt_is_reported_even_with_no_field_to_name() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
    // Parked when the app ran one instance: a shared explicit log path is
    // legal for one and refused for the two running now.
    let earlier = app_with("web", |app| {
        app.instances = 1;
        app.max_restarts = 10;
        app.out_file = Some("/tmp/web.log".to_string());
    });
    park_on_all(&mut actor, "web", &earlier);

    let mut file = AppConfig::minimal("web", "./srv");
    file.max_restarts = 99;
    let reply = apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "max_restarts"])],
        ResetDepth::None,
    )
    .await;

    assert_eq!(actor.sheep[&0].entry.spec.config().max_restarts, 99);
    assert!(
        reply[0]
            .refused
            .as_deref()
            .is_some_and(|why| why.contains("could not be rebuilt")),
        "a respawn is going to put max_restarts back to 10 and nobody was told: {reply:?}"
    );
    assert_eq!(
        actor.sheep[&0]
            .entry
            .pending
            .as_ref()
            .expect("the earlier parked config is still there")
            .config()
            .max_restarts,
        10,
        "and that is what makes the refusal true"
    );
}

/// The store's `declared` set is what `ResetDepth::None` skips over, so a
/// refused key entering it makes the refusal's own advice useless: the
/// retry meets silence rather than the same refusal.
#[tokio::test(start_paused = true)]
async fn a_refused_key_is_not_established_so_the_same_file_still_tries() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
    let earlier = app_with("web", |app| app.instances = 1);
    park_on_all(&mut actor, "web", &earlier);
    let file = || {
        let mut file = AppConfig::minimal("web", "./srv");
        file.out_file = Some("/tmp/web.log".to_string());
        declared_app(file, &["name", "script", "out_file"])
    };

    let names_it = |why: &str| why.contains("out_file");
    let first = apply_config(&mut actor, vec![file()], ResetDepth::None).await;
    assert!(
        first[0].refused.as_deref().is_some_and(names_it),
        "{first:?}"
    );

    let second = apply_config(&mut actor, vec![file()], ResetDepth::None).await;
    assert!(
        second[0].refused.as_deref().is_some_and(names_it),
        "a retry of the same file must meet the same refusal, not silence: {second:?}"
    );
    assert!(
        !shep_core::overrides::get(&actor.paths.overrides, "web")
            .unwrap()
            .expect("the load recorded what it established")
            .declared
            .contains("out_file"),
        "a key that went nowhere was established by nobody"
    );
}

/// `ResetDepth::None` skips a key somebody has established, so a load that
/// records nothing leaves every key permanently re-writable. Three loads,
/// because two cannot tell the difference: the first establishes the key,
/// the second drops it from the file, and the third re-adds it with a
/// different value, which only a record written by the first can refuse.
#[tokio::test(start_paused = true)]
async fn a_key_a_load_took_is_established_against_the_next_load() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |_| {})]);
    let with_budget = |budget: u32| {
        let mut file = AppConfig::minimal("web", "./srv");
        file.max_restarts = budget;
        declared_app(file, &["name", "script", "max_restarts"])
    };

    let first = apply_config(&mut actor, vec![with_budget(99)], ResetDepth::None).await;
    assert_eq!(first[0].applied, vec!["max_restarts".to_string()]);
    assert_eq!(actor.sheep[&0].entry.spec.config().max_restarts, 99);

    // The key leaves the file. Nothing happens, and nothing may forget
    // that it was established.
    let dropped = apply_config(
        &mut actor,
        vec![declared_app(
            AppConfig::minimal("web", "./srv"),
            &["name", "script"],
        )],
        ResetDepth::None,
    )
    .await;
    assert!(dropped[0].applied.is_empty(), "{dropped:?}");

    // And comes back with a different value. The app has run on 99 since
    // the first load, so a plain load must not take it.
    let third = apply_config(&mut actor, vec![with_budget(5)], ResetDepth::None).await;

    assert_eq!(
        actor.sheep[&0].entry.spec.config().max_restarts,
        99,
        "a file overwrote a key an earlier load had established"
    );
    assert!(third[0].applied.is_empty(), "{third:?}");
    assert!(
        shep_core::overrides::get(&actor.paths.overrides, "web")
            .unwrap()
            .expect("a load records what it established")
            .declared
            .contains("max_restarts"),
        "and the record is what makes that true"
    );
}

/// A key the file declares gives up its override during the merge, so a
/// load that then fails to park has to hand it back. `env` is the one
/// field that reaches this under the default depth: top-level keys hold
/// their override by being established and never spend anything, while
/// `env` merges one key at a time and spends the whole override table for
/// any key the file declares.
#[tokio::test(start_paused = true)]
async fn a_refused_env_change_gives_the_operators_override_back() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[app_with("web", |app| app.instances = 2)]);
    // Parked when the app ran one instance, with a shared explicit log
    // path: legal for one, refused for the two running now.
    let earlier = app_with("web", |app| {
        app.instances = 1;
        app.out_file = Some("/tmp/web.log".to_string());
    });
    park_on_all(&mut actor, "web", &earlier);
    shep_core::overrides::put(
        &actor.paths.overrides,
        "web",
        &established(
            &["name", "script"],
            vec![("env", serde_json::json!({ "OPERATOR": "1" }))],
        ),
    )
    .unwrap();

    // Two env keys: the one the operator already holds, which spends the
    // override table, and one nobody has, which makes `env` drift.
    let mut file = AppConfig::minimal("web", "./srv");
    file.env = BTreeMap::from([
        ("OPERATOR".to_string(), "2".to_string()),
        ("MODE".to_string(), "blue".to_string()),
    ]);
    let reply = apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "env"])],
        ResetDepth::None,
    )
    .await;
    assert!(
        reply[0]
            .refused
            .as_deref()
            .is_some_and(|why| why.contains("env")),
        "the fixture must really refuse the env change: {reply:?}"
    );

    let record = shep_core::overrides::get(&actor.paths.overrides, "web")
        .unwrap()
        .expect("a load records what it established");
    assert_eq!(
        record
            .fields
            .get("env")
            .and_then(|env| env.get("OPERATOR"))
            .and_then(serde_json::Value::as_str),
        Some("1"),
        "a load that changed nothing spent the operator's override"
    );
    assert!(
        record.declared_env.is_empty(),
        "and it established env keys that never landed: {:?}",
        record.declared_env
    );
}

/// An app `normalize` accepts under `memory_enforce = "kernel"`, which only
/// Linux does.
#[cfg(target_os = "linux")]
fn kernel_capped(limit: u64) -> ResolvedApp {
    app_with("web", |app| {
        app.max_memory = Some(MemSize::from_bytes(limit));
        app.memory_enforce = shep_core::config::MemoryEnforce::Kernel;
    })
}

/// The sampler would restart a sheep the kernel already caps, and count the
/// restart differently: a kernel-enforced sheep is never armed with it.
#[cfg(target_os = "linux")]
#[tokio::test(start_paused = true)]
async fn a_kernel_enforced_sheep_is_not_armed_with_the_polling_enforcer() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, enforcer) = actor_over(&dir, &[kernel_capped(100 << 20)]);

    let mut file = AppConfig::minimal("web", "./srv");
    file.max_memory = Some(MemSize::from_bytes(512 << 20));
    apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "max_memory"])],
        ResetDepth::None,
    )
    .await;

    assert!(
        enforcer.arms().is_empty(),
        "a re-arm reached the sampler: {:?}",
        enforcer.arms()
    );
}

/// A sheep's cgroup under a plain directory, with the `memory.max` the kernel
/// would hold from its spawn.
#[cfg(target_os = "linux")]
fn capped_cgroup(actor: &mut Actor<ScriptedRunner>, held: &str) -> (tempfile::TempDir, PathBuf) {
    let base = tempfile::tempdir().unwrap();
    let cgroup = crate::cgroup::sheep_dir(base.path(), 0, 0);
    std::fs::create_dir(&cgroup).unwrap();
    std::fs::write(cgroup.join("memory.max"), held).unwrap();
    actor.cgroup_base = Some(base.path().to_path_buf());
    (base, cgroup)
}

#[cfg(target_os = "linux")]
#[tokio::test(start_paused = true)]
async fn a_live_max_memory_change_rewrites_a_kernel_enforced_sheeps_memory_max() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[kernel_capped(100 << 20)]);
    let (_base, cgroup) = capped_cgroup(&mut actor, "104857600");

    let mut file = AppConfig::minimal("web", "./srv");
    file.max_memory = Some(MemSize::from_bytes(512 << 20));
    apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "max_memory"])],
        ResetDepth::None,
    )
    .await;

    assert_eq!(
        std::fs::read_to_string(cgroup.join("memory.max")).unwrap(),
        "536870912"
    );
}

#[cfg(target_os = "linux")]
fn max_tasks_file(tasks: u32) -> DeclaredApp {
    let mut file = AppConfig::minimal("web", "./srv");
    file.max_tasks = Some(tasks);
    declared_app(file, &["name", "script", "max_tasks"])
}

#[cfg(target_os = "linux")]
#[tokio::test(start_paused = true)]
async fn a_live_max_tasks_change_rewrites_pids_max_and_reports_it_applied() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) =
        actor_over(&dir, &[app_with("web", |app| app.max_tasks = Some(64))]);
    let (_base, cgroup) = capped_cgroup(&mut actor, "max");

    let applied = apply_config(&mut actor, vec![max_tasks_file(128)], ResetDepth::None).await;

    assert_eq!(
        std::fs::read_to_string(cgroup.join("pids.max")).unwrap(),
        "128"
    );
    assert_eq!(applied[0].applied, ["max_tasks"]);
    assert!(applied[0].pending.is_empty());
}

#[cfg(target_os = "linux")]
#[tokio::test(start_paused = true)]
async fn a_max_tasks_the_kernel_did_not_take_is_reported_pending_not_applied() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) =
        actor_over(&dir, &[app_with("web", |app| app.max_tasks = Some(64))]);
    let (_base, cgroup) = capped_cgroup(&mut actor, "max");
    // A write to a directory fails, as a cgroup without the `pids` controller would.
    std::fs::create_dir(cgroup.join("pids.max")).unwrap();

    let applied = apply_config(&mut actor, vec![max_tasks_file(128)], ResetDepth::None).await;

    assert!(applied[0].applied.is_empty(), "{:?}", applied[0].applied);
    assert_eq!(
        applied[0].pending,
        ["max_tasks"],
        "the next spawn writes it"
    );
    assert!(applied[0].refused.is_none(), "stored, not refused");
}

#[cfg(target_os = "linux")]
#[tokio::test(start_paused = true)]
async fn leaving_and_entering_kernel_enforcement_live_lifts_and_sets_the_cap() {
    let dir = tempfile::tempdir().unwrap();
    let (mut actor, _enforcer) = actor_over(&dir, &[kernel_capped(100 << 20)]);
    let (_base, cgroup) = capped_cgroup(&mut actor, "104857600");
    let memory_enforce = |value| {
        let mut file = AppConfig::minimal("web", "./srv");
        file.memory_enforce = value;
        declared_app(file, &["name", "script", "memory_enforce"])
    };

    // `File`, not `None`: a load at `None` only appends a key nobody has
    // established, so the second one would change nothing.
    apply_config(
        &mut actor,
        vec![memory_enforce(shep_core::config::MemoryEnforce::Poll)],
        ResetDepth::File,
    )
    .await;
    assert_eq!(
        std::fs::read_to_string(cgroup.join("memory.max")).unwrap(),
        "max",
        "polling takes the enforcement back, so the kernel stops capping"
    );

    apply_config(
        &mut actor,
        vec![memory_enforce(shep_core::config::MemoryEnforce::Kernel)],
        ResetDepth::File,
    )
    .await;
    assert_eq!(
        std::fs::read_to_string(cgroup.join("memory.max")).unwrap(),
        "104857600"
    );
}

/// No spawn follows a live switch, so the switch has to take the `oom_kill`
/// reading the exit compares against, or the first kill goes uncharged.
#[cfg(target_os = "linux")]
#[tokio::test(start_paused = true)]
async fn a_live_switch_to_kernel_takes_the_oom_baseline_and_a_switch_back_drops_it() {
    let dir = tempfile::tempdir().unwrap();
    let poll = app_with("web", |app| {
        app.max_memory = Some(MemSize::from_bytes(100 << 20));
    });
    let (mut actor, _enforcer) = actor_over(&dir, &[poll]);
    let (_base, cgroup) = capped_cgroup(&mut actor, "max");
    std::fs::write(
        cgroup.join("memory.events.local"),
        "low 0\nhigh 0\nmax 0\noom 3\noom_kill 3\n",
    )
    .unwrap();
    let memory_enforce = |value| {
        let mut file = AppConfig::minimal("web", "./srv");
        file.memory_enforce = value;
        declared_app(file, &["name", "script", "memory_enforce"])
    };

    apply_config(
        &mut actor,
        vec![memory_enforce(shep_core::config::MemoryEnforce::Kernel)],
        ResetDepth::File,
    )
    .await;
    assert_eq!(actor.oom_baselines.get(&0), Some(&3));

    apply_config(
        &mut actor,
        vec![memory_enforce(shep_core::config::MemoryEnforce::Poll)],
        ResetDepth::File,
    )
    .await;
    assert_eq!(actor.oom_baselines.get(&0), None);
}

#[cfg(target_os = "linux")]
#[tokio::test(start_paused = true)]
async fn a_live_max_cpu_cores_change_rewrites_a_running_sheeps_cpu_max() {
    let dir = tempfile::tempdir().unwrap();
    let throttled = app_with("web", |app| {
        app.max_cpu_cores = Some(shep_core::config::CpuCores::from_cores(0.5));
    });
    let (mut actor, _enforcer) = actor_over(&dir, &[throttled]);
    let (_base, cgroup) = capped_cgroup(&mut actor, "max");
    std::fs::write(cgroup.join("cpu.max"), "50000 100000").unwrap();

    let mut file = AppConfig::minimal("web", "./srv");
    file.max_cpu_cores = Some(shep_core::config::CpuCores::from_cores(2.5));
    apply_config(
        &mut actor,
        vec![declared_app(file, &["name", "script", "max_cpu_cores"])],
        ResetDepth::None,
    )
    .await;

    assert_eq!(
        std::fs::read_to_string(cgroup.join("cpu.max")).unwrap(),
        "250000 100000"
    );
}

#[cfg(target_os = "linux")]
#[tokio::test(start_paused = true)]
async fn a_pane_edit_of_max_cpu_cores_rewrites_a_running_sheeps_cpu_max() {
    let dir = tempfile::tempdir().unwrap();
    let throttled = app_with("web", |app| {
        app.max_cpu_cores = Some(shep_core::config::CpuCores::from_cores(0.5));
    });
    let (mut actor, _enforcer) = actor_over(&dir, &[throttled]);
    let (_base, cgroup) = capped_cgroup(&mut actor, "max");
    std::fs::write(cgroup.join("cpu.max"), "50000 100000").unwrap();

    actor
        .write_sheep_field("web", "max_cpu_cores", &serde_json::json!(2.5))
        .unwrap();
    assert_eq!(
        std::fs::read_to_string(cgroup.join("cpu.max")).unwrap(),
        "250000 100000"
    );

    actor
        .write_sheep_field("web", "max_cpu_cores", &serde_json::Value::Null)
        .unwrap();
    assert_eq!(
        std::fs::read_to_string(cgroup.join("cpu.max")).unwrap(),
        "max 100000"
    );
}