shep-daemon 0.1.2

The supervision engine behind shep: process spawning, restart policy, the log plane, watch and cron restarts, and the RPC server
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
905
906
907
908
909
910
911
912
913
914
915
916
917
918
919
920
921
922
923
924
925
926
927
928
929
930
931
932
933
934
935
936
937
938
939
940
941
942
943
944
945
946
947
948
949
950
//! [`OsProber`] — HTTP, TCP and exec probes over real sockets and processes.
//!
//! The HTTP probe is hand-rolled over `tokio::net::TcpStream` rather than
//! built on a client crate. No TLS and no redirects, and both are visible to
//! the user rather than silent: `https://` targets are rejected at config
//! time
//! ([`ProbeTargetError::HttpsUnsupported`](shep_core::config::ProbeTargetError::HttpsUnsupported)),
//! and a `301` is a [`ProbeFailure::Rejected`], never followed — a probe
//! that follows redirects is a probe that can pass against a completely
//! different service. For the same reason a response only counts as healthy
//! if it *is* an HTTP response — a status line must begin `HTTP/` before its
//! status code is worth reading, or any line-oriented daemon that answers
//! `… 204 …` passes.

// Rejected alternatives, so nobody re-litigates: `reqwest` brings tower and a
// TLS stack into a daemon targeting single-digit-MB idle RSS (spec §14.11);
// `hyper` + `hyper-util` + `http-body-util` is three dependencies and a
// connection-pool abstraction to send one request with `Connection: close`;
// `ureq` and `minreq` are blocking, and a blocking read cannot be cancelled
// by `tokio::time::timeout` — the timeout would return while the thread
// stayed stuck. The `Prober` seam means swapping any of them in later
// touches one file (IR-31).

use core::fmt;
use core::future::Future;
use core::pin::Pin;
use core::time::Duration;

use std::collections::BTreeMap;
use std::path::PathBuf;
use std::process::Stdio;

use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::net::TcpStream;
use tokio::process::Command;

use shep_core::config::ProbeTarget;

use super::{ProbeFailure, Prober};

/// Longest status line `OsProber` will read before giving up on a response.
///
/// An HTTP/1.1 status line is a method-agnostic `HTTP/1.1 200 OK` — tens of
/// bytes. This bound exists so a probe target that is not an HTTP server
/// cannot stream unbounded data into the daemon's heap; nothing legitimate
/// comes close to it.
const HTTP_STATUS_LINE_CAP: u64 = 8 * 1024;

/// The port RFC 7230 §5.4 lets a `Host:` header leave out. `ProbeTarget`
/// already defaults a portless `http://` target to this, so a header built
/// from one carries no port either way.
const HTTP_DEFAULT_PORT: u16 = 80;

/// `Prober` over real sockets and real processes.
pub struct OsProber {
    /// Working directory for exec probes — `None` inherits the daemon's own,
    /// matching `Command::current_dir`'s own default.
    cwd: Option<PathBuf>,
    /// Environment for exec probes — a probe usually needs the same `PORT`
    /// the sheep was given.
    ///
    /// `Debug` does not leak environment values (IR-41).
    env: BTreeMap<String, String>,
}

impl OsProber {
    /// A prober that runs exec probes in `cwd` with `env`.
    #[must_use]
    pub fn new(cwd: Option<PathBuf>, env: BTreeMap<String, String>) -> Self {
        Self { cwd, env }
    }
}

/// Debug implementation does not leak env values (IR-41): mirrors
/// `AppConfig`'s manual `Debug` (`crates/shep-core/src/config/app.rs`) —
/// one redaction spelling in the workspace, not two.
///
/// `finish_non_exhaustive`, not `finish`: this is a redacting impl, and the
/// trailing `..` is the honest rendering of one. `finish` would assert that
/// what it printed is every field there is, which stops being true the day
/// somebody adds a second secret-bearing one and does not think to come back
/// here.
impl fmt::Debug for OsProber {
    fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
        f.debug_struct("OsProber")
            .field("cwd", &self.cwd)
            .field("env", &format_args!("<{} vars>", self.env.len()))
            .finish_non_exhaustive()
    }
}

impl Prober for OsProber {
    fn probe<'a>(
        &'a self,
        target: &'a ProbeTarget,
        timeout: Duration,
    ) -> Pin<Box<dyn Future<Output = Result<(), ProbeFailure>> + Send + 'a>> {
        // The `ProbeTarget` match is the compile-time gate (Global
        // Constraints rule 6): `ProbeTarget` is deliberately not
        // `#[non_exhaustive]`, so a fourth transport fails `cargo check`
        // with E0004 at exactly the place that has to change, and no `_`
        // arm may be added here.
        Box::pin(async move {
            match target {
                ProbeTarget::Http { host, port, path } => {
                    probe_http(host, *port, path, timeout).await
                }
                ProbeTarget::Tcp { host, port } => probe_tcp(host, *port, timeout).await,
                ProbeTarget::Exec { command } => self.probe_exec(command, timeout).await,
            }
        })
    }
}

impl OsProber {
    /// Runs `command` through the platform shell, giving up after `timeout`.
    async fn probe_exec(&self, command: &str, timeout: Duration) -> Result<(), ProbeFailure> {
        #[cfg(unix)]
        let mut cmd = {
            let mut c = Command::new("sh");
            c.arg("-c").arg(command);
            c
        };
        #[cfg(windows)]
        let mut cmd = {
            let mut c = Command::new("cmd");
            c.arg("/C").arg(command);
            c
        };

        // Kills and reaps the shell itself when `child` goes out of scope,
        // so a probe that times out leaves no zombie behind. It is only half
        // the cleanup: `kill_on_drop` reaches the one process it spawned, and
        // the reason a timed-out probe "accumulates processes until the box
        // falls over" is everything that process forked. `kill_probe_group`
        // below is the other half.
        cmd.kill_on_drop(true);
        // Puts the shell in a process group of its own, whose pgid is its own
        // pid — the property `kill(-pid)` needs, and the same one
        // `TokioRunner::spawn` sets for a sheep. Without it, `-pid` names no
        // group at all and the timeout path has nothing group-wide to signal.
        #[cfg(unix)]
        cmd.process_group(0);
        // A probe's output is the probe's business, never the daemon's. The
        // default is inheritance, which puts a `curl`-style probe's entire
        // response body — bearer tokens, session cookies, whatever the
        // endpoint returns — into the daemon's own stdout once per interval,
        // forever. The verdict this prober needs is the exit status alone,
        // so there is nothing to read and no reason to keep a pipe.
        cmd.stdin(Stdio::null());
        cmd.stdout(Stdio::null());
        cmd.stderr(Stdio::null());
        if let Some(cwd) = &self.cwd {
            cmd.current_dir(cwd);
        }
        // The probe sees the sheep's environment, never the daemon's —
        // `SpawnSpec::env`'s "no daemon-env leakage beyond this map"
        // (`crates/shep-daemon/src/runner.rs`) already rules this out for
        // sheep, and a probe is not a special case.
        cmd.env_clear().envs(&self.env);

        let mut child = cmd
            .spawn()
            .map_err(|err| ProbeFailure::Transport(err.to_string()))?;
        // Bound to a `let` rather than matched inline: a `match` keeps its
        // scrutinee's temporaries — here the `child.wait()` future, holding
        // `&mut child` — alive across every arm, and the abandon arms need
        // `child` back to signal its group.
        let waited = tokio::time::timeout(timeout, child.wait()).await;
        match waited {
            Ok(Ok(status)) if status.success() => Ok(()),
            // Load-bearing detail for anyone who later drops the shell: a
            // command naming a binary that does not exist is `Rejected("127")`
            // and NOT a spawn failure, because `sh` itself always spawns and
            // reports "not found" through its own exit code. Exec straight to
            // the program instead and the same 127 becomes a real spawn
            // failure, so this classification flips without a line here
            // changing.
            Ok(Ok(status)) => Err(ProbeFailure::Rejected(exit_code_text(&status))),
            // Both remaining arms abandon a process that may still be
            // running: the timeout by definition, and a `wait` that errored
            // (ECHILD — something outside our control reaped the pid) without
            // ever establishing that the child is gone.
            Ok(Err(err)) => {
                kill_probe_group(&child);
                Err(ProbeFailure::Transport(err.to_string()))
            }
            Err(_elapsed) => {
                kill_probe_group(&child);
                Err(ProbeFailure::Timeout)
            }
        }
    }
}

/// SIGKILLs the whole process group of a probe child being abandoned.
///
/// Reuses the runner's own [`signal_group`](crate::tokio_runner::signal_group)
/// rather than spelling `kill(-pid)` a third time — the two stop rungs are
/// the other callers, and this is the same orphan shape they were built for:
/// `sh -c 'thing & …'` leaves a fork that a leader-only kill never reaches.
///
/// Failure is logged, never returned: the verdict a timed-out probe reports
/// is [`ProbeFailure::Timeout`] whether or not the corpse was cleared, and
/// `ESRCH` here just means the group was already gone.
#[cfg(unix)]
fn kill_probe_group(child: &tokio::process::Child) {
    // `None` once the child has been waited to completion — nothing left to
    // signal, and a pid the OS may since have recycled is the one thing this
    // must never send SIGKILL to.
    let Some(pid) = child.id() else {
        return;
    };
    if let Err(error) = crate::tokio_runner::signal_group(pid, nix::sys::signal::Signal::SIGKILL) {
        tracing::warn!(pid, %error, "probe process group kill failed");
    }
}

/// Windows has no process group this can signal, and `kill_on_drop` reaches
/// the `cmd` alone. That gap is the same one the whole OS tier carries there
/// — `TokioRunner` and its own group signalling are `#[cfg(unix)]`, so a
/// Windows daemon has no supervised processes to leak in the first place.
#[cfg(windows)]
fn kill_probe_group(_child: &tokio::process::Child) {}

/// Renders an `ExitStatus` for [`ProbeFailure::Rejected`]. `code()` is
/// `None` on unix when the child died from a signal rather than exiting —
/// carried distinctly rather than defaulted to a number no real exit code
/// produces.
fn exit_code_text(status: &std::process::ExitStatus) -> String {
    status.code().map_or_else(
        || "terminated by signal".to_string(),
        |code| code.to_string(),
    )
}

/// Probes an HTTP target: connect, write one `GET`, read the status line,
/// pass on `200..=299`.
///
/// Connect, write and read are wrapped in ONE `tokio::time::timeout` rather
/// than one per step — three separate timeouts add up to three times the
/// budget the caller configured.
async fn probe_http(
    host: &str,
    port: u16,
    path: &str,
    timeout: Duration,
) -> Result<(), ProbeFailure> {
    match tokio::time::timeout(timeout, http_roundtrip(host, port, path)).await {
        Ok(result) => result,
        Err(_elapsed) => Err(ProbeFailure::Timeout),
    }
}

async fn http_roundtrip(host: &str, port: u16, path: &str) -> Result<(), ProbeFailure> {
    // `(host, port)`, not a formatted `"{host}:{port}"` parsed into a
    // `SocketAddr`: `ProbeTarget` strips brackets from a bracketed IPv6
    // literal at parse time, and a bracket-stripped host is exactly what
    // this tuple form needs — it fails as a formatted string, which has no
    // brackets left to make it parseable.
    let mut stream = TcpStream::connect((host, port))
        .await
        .map_err(|err| ProbeFailure::Transport(err.to_string()))?;

    let header_host = host_header(host, port);
    let request =
        format!("GET {path} HTTP/1.1\r\nHost: {header_host}\r\nConnection: close\r\n\r\n");
    stream
        .write_all(request.as_bytes())
        .await
        .map_err(|err| ProbeFailure::Transport(err.to_string()))?;

    let status_line = read_status_line(stream).await?;
    evaluate_status_line(&status_line)
}

/// Builds the RFC 7230 `Host:` header value for a target.
///
/// Two things happen here, and both matter on the wire.
///
/// The brackets come back. `ProbeTarget` strips them off `[::1]` at parse
/// time so `TcpStream::connect` can take `(host, port)` directly — a
/// formatted `"{host}:{port}"` does not parse as a socket address for IPv6 —
/// but `Host: ::1` reads as colon-separated fields rather than one address,
/// so the header needs them again.
///
/// The port comes with it, unless it is the scheme default. RFC 7230 §5.4
/// gives `Host = uri-host [ ":" port ]`, and omitting a non-default port is
/// not cosmetic: a name-based virtual host serving several ports routes on
/// this header alone, so `Host: example.com` for a probe of
/// `http://example.com:8080/health` can be answered by an entirely different
/// application than the one under test.
fn host_header(host: &str, port: u16) -> String {
    let host = if host.contains(':') {
        format!("[{host}]")
    } else {
        host.to_string()
    };
    if port == HTTP_DEFAULT_PORT {
        host
    } else {
        format!("{host}:{port}")
    }
}

/// Reads up to the first `\r\n`, or [`HTTP_STATUS_LINE_CAP`] bytes,
/// whichever comes first.
async fn read_status_line(stream: TcpStream) -> Result<String, ProbeFailure> {
    let mut reader = BufReader::new(stream.take(HTTP_STATUS_LINE_CAP));
    let mut buf = Vec::new();
    reader
        .read_until(b'\n', &mut buf)
        .await
        .map_err(|err| ProbeFailure::Transport(err.to_string()))?;
    if buf.is_empty() {
        return Err(ProbeFailure::Transport(
            "connection closed before a response was received".to_string(),
        ));
    }
    Ok(String::from_utf8_lossy(&buf).into_owned())
}

/// Parses the numeric status out of an HTTP status line (`HTTP/1.1 200 OK`
/// -> `200`) and maps it to a pass or a [`ProbeFailure::Rejected`].
///
/// The line must begin `HTTP/`. The spec words this check positionally —
/// "the second space-separated token" — and position alone reads `BANANA 204
/// whatever` as healthy, which is exactly the "a probe that can pass against
/// a completely different service" hazard this module already invokes to ban
/// redirects: a probe port that turns out to be some other line-oriented
/// daemon should fail, not pass. Requiring the prefix cannot cost a real
/// server anything, since RFC 7230 §3.1.2 makes `HTTP/x.y` the mandatory
/// first token of every HTTP/1.x status line — this narrows the accepted
/// grammar, and narrowing is the safe direction to differ from a spec's
/// shorthand in.
///
/// Never indexes a token that might not be there and never panics on a
/// malformed line: `.nth(1)` and `.parse().ok()` both return `None` rather
/// than panicking, and a probe target is arbitrary, untrusted text — this is
/// the one place a probe genuinely sees "what came back."
///
/// A line with no parseable status is `Rejected` rather than `Transport`:
/// the connection succeeded and bytes came back, so a verdict *was*
/// possible — it was just negative, exactly like a non-2xx status.
fn evaluate_status_line(line: &str) -> Result<(), ProbeFailure> {
    let trimmed = line.trim_end_matches(['\r', '\n']);
    let Some(after_version) = trimmed.strip_prefix("HTTP/") else {
        return Err(ProbeFailure::Rejected(format!(
            "not an HTTP status line: {trimmed:?}"
        )));
    };
    // Counted over what follows `HTTP/`, so the token taken is still the one
    // after the version: `1.1 200 OK` -> `200`.
    match after_version
        .split(' ')
        .nth(1)
        .and_then(|token| token.parse::<u16>().ok())
    {
        Some(200..=299) => Ok(()),
        Some(code) => Err(ProbeFailure::Rejected(code.to_string())),
        None => Err(ProbeFailure::Rejected(format!(
            "malformed HTTP status line: {trimmed:?}"
        ))),
    }
}

/// Probes a TCP target: pass on a successful connect, nothing more.
async fn probe_tcp(host: &str, port: u16, timeout: Duration) -> Result<(), ProbeFailure> {
    match tokio::time::timeout(timeout, TcpStream::connect((host, port))).await {
        Ok(Ok(_stream)) => Ok(()),
        Ok(Err(err)) => Err(ProbeFailure::Transport(err.to_string())),
        Err(_elapsed) => Err(ProbeFailure::Timeout),
    }
}

#[cfg(test)]
mod tests {
    // IR-33: real time, not the paused clock. Every test below connects to a
    // real `127.0.0.1` listener, or spawns a real child process. `#[tokio::
    // test(start_paused = true)]` freezes only `tokio::time`; the kernel's
    // TCP stack and process table keep running on the real clock regardless,
    // so a paused test waiting on a real connect, a real accept, or a real
    // child exit would deadlock — the clock inside the test's own task never
    // appears to move, while the other side of the socket or process is
    // unaffected and just sits there. Every `#[tokio::test]` below is a
    // plain one, on purpose.

    use core::time::Duration;

    use std::collections::BTreeMap;

    use tokio::net::TcpListener;

    use super::*;
    use crate::testing::{HttpReply, loopback_http, loopback_http_on};

    /// Every test's probe timeout — generous enough that CI/loaded-machine
    /// scheduling jitter can't turn a real pass into a flaky timeout, small
    /// enough that a mistakenly-hanging probe doesn't stall the suite.
    const PROBE_TIMEOUT: Duration = Duration::from_secs(5);

    fn http_target(port: u16, path: &str) -> ProbeTarget {
        ProbeTarget::Http {
            host: "127.0.0.1".to_string(),
            port,
            path: path.to_string(),
        }
    }

    // fails if the status check is `== 200` instead of the documented
    // `200..=299` range
    #[tokio::test]
    async fn passing_status_codes_are_accepted_across_the_2xx_range() {
        for code in [200u16, 204, 299] {
            let server = loopback_http(vec![HttpReply::Status(code)]).await;
            let prober = OsProber::new(None, BTreeMap::new());
            let result = prober
                .probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
                .await;
            assert_eq!(result, Ok(()), "status {code} should pass");
        }
    }

    // fails if a prober follows the redirect (or otherwise treats 3xx as
    // success) instead of reporting it as a plain rejection — a probe that
    // follows redirects can pass against a completely different service
    #[tokio::test]
    async fn a_301_is_rejected_not_followed() {
        let server = loopback_http(vec![HttpReply::Status(301)]).await;
        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober
            .probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
            .await;
        assert_eq!(result, Err(ProbeFailure::Rejected("301".to_string())));
    }

    // fails if the prober only checks whether ANY bytes arrived rather than
    // parsing the actual status code
    #[tokio::test]
    async fn a_500_is_rejected() {
        let server = loopback_http(vec![HttpReply::Status(500)]).await;
        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober
            .probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
            .await;
        assert_eq!(result, Err(ProbeFailure::Rejected("500".to_string())));
    }

    // fails if connect/write/read each get their own `timeout` instead of
    // sharing one: three separate timeouts of `short` would let this test
    // take up to 3x `short` instead of resting at very roughly `short`
    #[tokio::test]
    async fn a_hanging_response_times_out_within_a_small_multiple_of_the_budget() {
        let server = loopback_http(vec![HttpReply::Hang]).await;
        let short = Duration::from_millis(150);
        let prober = OsProber::new(None, BTreeMap::new());

        let start = std::time::Instant::now();
        let result = prober
            .probe(&http_target(server.addr.port(), "/"), short)
            .await;
        let elapsed = start.elapsed();

        assert_eq!(result, Err(ProbeFailure::Timeout));
        assert!(
            elapsed < short * 3,
            "expected the probe to give up within a small multiple of {short:?}, took {elapsed:?}"
        );
    }

    // Pins the bytes the probe puts on the wire, which nothing else here
    // reads: the fake's reply does not depend on its request, so a prober
    // that ignored the target's path, dropped `Host:`, dropped `Connection:
    // close` (leaving the server holding a connection open for a client that
    // has already stopped reading) or sent no request at all passed every
    // other test in this module.
    //
    // The `Host:` header carries the port whenever it is not 80, per RFC
    // 7230 §5.4. A name-based virtual host serving several ports routes on
    // this header alone, so dropping the port can hand the probe to an
    // entirely different application than the one under test.
    #[tokio::test]
    async fn an_http_probe_sends_one_get_carrying_the_targets_path_and_a_host_header() {
        let mut server = loopback_http(vec![HttpReply::Status(200)]).await;
        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober
            .probe(&http_target(server.addr.port(), "/healthz"), PROBE_TIMEOUT)
            .await;
        assert_eq!(result, Ok(()));
        // The port is ephemeral, so it is formatted in rather than pinned —
        // but that it appears at all is the assertion.
        let port = server.addr.port();
        assert_eq!(
            server.next_request().await,
            format!("GET /healthz HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n")
        );
    }

    // The first of the two IPv6 obligations `ProbeTarget`'s `host` doc
    // records, isolated from the network
    // so it holds on a machine with no IPv6 at all: `ProbeTarget` strips the
    // brackets off `[::1]` at parse time, and `Host: ::1` without them back
    // reads as colon-separated fields rather than one address.
    #[test]
    fn an_ipv6_host_is_re_bracketed_for_the_host_header() {
        // Brackets return for IPv6, and the port rides along unless it is
        // the scheme default the header may omit.
        assert_eq!(host_header("::1", 8080), "[::1]:8080");
        assert_eq!(host_header("2001:db8::1", 443), "[2001:db8::1]:443");
        assert_eq!(host_header("::1", 80), "[::1]");
        assert_eq!(host_header("127.0.0.1", 8080), "127.0.0.1:8080");
        assert_eq!(host_header("127.0.0.1", 80), "127.0.0.1");
        assert_eq!(host_header("localhost", 9000), "localhost:9000");
    }

    // Both IPv6 obligations at once, end to end: the connect uses
    // the bracket-STRIPPED host (formatting `"{host}:{port}"` into a
    // `SocketAddr` parse is what fails, since there are no brackets left to
    // make that string parseable), and the header gets them back.
    //
    // Needs an IPv6 loopback. Every GitHub-hosted runner carries `::1` on
    // `lo`; a container with IPv6 switched off fails the bind, and
    // `loopback_http_on`'s panic names the address so that reads as a missing
    // capability rather than a mystery.
    #[tokio::test]
    async fn an_ipv6_target_connects_unbracketed_and_brackets_the_host_header() {
        let mut server = loopback_http_on("[::1]:0", vec![HttpReply::Status(200)]).await;
        let target = ProbeTarget::Http {
            host: "::1".to_string(),
            port: server.addr.port(),
            path: "/".to_string(),
        };

        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober.probe(&target, PROBE_TIMEOUT).await;

        assert_eq!(result, Ok(()), "an IPv6 target must connect at all");
        let port = server.addr.port();
        assert_eq!(
            server.next_request().await,
            format!("GET / HTTP/1.1\r\nHost: [::1]:{port}\r\nConnection: close\r\n\r\n")
        );
    }

    // fails if a refused connection is collapsed into Timeout instead of
    // Transport — a down service must not look like a slow one
    #[tokio::test]
    async fn a_port_with_nothing_listening_fails_as_transport() {
        // Bind to grab a genuinely free port, then drop the listener so the
        // port refuses rather than being caught mid-handshake by anything
        // still listening.
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let addr = listener.local_addr().unwrap();
        drop(listener);

        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober
            .probe(&http_target(addr.port(), "/"), PROBE_TIMEOUT)
            .await;
        assert!(
            matches!(result, Err(ProbeFailure::Transport(_))),
            "expected Transport, got {result:?}"
        );
    }

    // fails if the status-line parser panics on a malformed line (e.g.
    // unwrapping or indexing a token that is not there) instead of failing
    // it as a value. Asserts Rejected specifically (the judgment call this
    // task made among the brief's two allowed outcomes): the connection
    // succeeded and bytes came back, so a verdict was possible — it was
    // just negative.
    #[tokio::test]
    async fn a_garbage_first_line_is_rejected_not_panicked_on() {
        let server = loopback_http(vec![HttpReply::Raw("not http\r\n".to_string())]).await;
        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober
            .probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
            .await;
        assert!(
            matches!(result, Err(ProbeFailure::Rejected(_))),
            "expected Rejected, got {result:?}"
        );
    }

    // The brief's own suggested fixture ("not http\r\n") has two
    // space-separated tokens, so it only exercises a parser whose second
    // token fails to parse as u16 — never one that indexes past the end of
    // the token list. This one is what catches an implementation reaching for
    // `tokens[1]` instead of `.nth(1)`.
    //
    // It has to be a well-formed VERSION with nothing after it, not arbitrary
    // one-word garbage: since the parser now requires an `HTTP/` prefix, a
    // line that fails that check never reaches the token indexing at all, and
    // a fixture like "garbage\r\n" would quietly stop testing anything here.
    #[tokio::test]
    async fn a_status_line_with_no_code_after_the_version_is_rejected_not_panicked_on() {
        let server = loopback_http(vec![HttpReply::Raw("HTTP/1.1\r\n".to_string())]).await;
        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober
            .probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
            .await;
        assert!(
            matches!(result, Err(ProbeFailure::Rejected(_))),
            "expected Rejected, got {result:?}"
        );
    }

    // fails if the status check is positional only. `BANANA 204 whatever`
    // has a 2xx in the slot the spec's wording points at, so a parser that
    // counts tokens without ever asking what protocol answered calls this
    // healthy — the same "passes against a completely different service"
    // hazard the no-redirects rule exists for, one layer down. Some other
    // line-oriented daemon listening on the port a Flockfile named is a
    // realistic way to meet this, and it must fail.
    #[tokio::test]
    async fn a_2xx_from_a_service_that_is_not_http_is_rejected() {
        let server =
            loopback_http(vec![HttpReply::Raw("BANANA 204 whatever\r\n".to_string())]).await;
        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober
            .probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
            .await;
        assert!(
            matches!(result, Err(ProbeFailure::Rejected(_))),
            "expected Rejected, got {result:?}"
        );
    }

    // fails if a TCP probe reports success against any resolvable address
    // instead of actually attempting (and requiring) a real connect
    #[tokio::test]
    async fn tcp_probe_against_a_bound_listener_passes() {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let addr = listener.local_addr().unwrap();

        let prober = OsProber::new(None, BTreeMap::new());
        let target = ProbeTarget::Tcp {
            host: "127.0.0.1".to_string(),
            port: addr.port(),
        };
        let result = prober.probe(&target, PROBE_TIMEOUT).await;
        assert_eq!(result, Ok(()));
        drop(listener);
    }

    #[tokio::test]
    async fn tcp_probe_against_a_closed_port_fails_as_transport() {
        let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
        let addr = listener.local_addr().unwrap();
        drop(listener);

        let prober = OsProber::new(None, BTreeMap::new());
        let target = ProbeTarget::Tcp {
            host: "127.0.0.1".to_string(),
            port: addr.port(),
        };
        let result = prober.probe(&target, PROBE_TIMEOUT).await;
        assert!(
            matches!(result, Err(ProbeFailure::Transport(_))),
            "expected Transport, got {result:?}"
        );
    }

    // fails if the TCP probe's connect is not itself wrapped in `timeout` —
    // 192.0.2.1 is TEST-NET-1 (RFC 5737): reserved, unroutable, and
    // confirmed empirically on this machine to hang a connect attempt
    // rather than refuse it immediately (`nc -w2 192.0.2.1 1` blocks for the
    // full 2s rather than erroring at once), which is exactly the "down
    // service that looks slow, not refused" shape a bare `connect().await`
    // with no timeout would hang on forever.
    #[tokio::test]
    async fn tcp_probe_against_a_non_routable_address_gives_up_within_the_budget() {
        let short = Duration::from_millis(300);
        let prober = OsProber::new(None, BTreeMap::new());
        let target = ProbeTarget::Tcp {
            host: "192.0.2.1".to_string(),
            port: 1,
        };

        let start = std::time::Instant::now();
        let result = prober.probe(&target, short).await;
        let elapsed = start.elapsed();

        // Either failure is correct, and which one arrives is the network's
        // call, not the prober's: a blackholing path (this machine, and the
        // usual CI shape) leaves the connect hanging until the timeout fires,
        // while a path whose router answers ICMP unreachable refuses it
        // outright and yields `Transport`. Pinning `Timeout` alone would fail
        // on the second kind of network without anything being wrong.
        //
        // The bound below is what this test is really for, and it holds
        // either way: a connect with no `timeout` around it does not come
        // back at all.
        assert!(
            matches!(
                result,
                Err(ProbeFailure::Timeout | ProbeFailure::Transport(_))
            ),
            "expected Timeout or Transport, got {result:?}"
        );
        assert!(
            elapsed < short * 3,
            "expected the probe to give up within a small multiple of {short:?}, took {elapsed:?}"
        );
    }

    #[cfg(unix)]
    fn exec_target(command: &str) -> ProbeTarget {
        ProbeTarget::Exec {
            command: command.to_string(),
        }
    }

    #[cfg(unix)]
    #[tokio::test]
    async fn exec_probe_exit_zero_passes() {
        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober.probe(&exec_target("exit 0"), PROBE_TIMEOUT).await;
        assert_eq!(result, Ok(()));
    }

    // fails if the exit code is dropped or hardcoded instead of carried
    // through to Rejected
    #[cfg(unix)]
    #[tokio::test]
    async fn exec_probe_nonzero_exit_is_rejected_with_the_code() {
        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober.probe(&exec_target("exit 3"), PROBE_TIMEOUT).await;
        assert_eq!(result, Err(ProbeFailure::Rejected("3".to_string())));
    }

    // fails if the exec probe's `cmd.status()` await is not itself wrapped
    // in `timeout` — a bare await on a 5s sleep would make this test take
    // ~5s instead of resting near `short`. Also the test that makes
    // `kill_on_drop(true)` load-bearing rather than decorative: dropping
    // `cmd.status()`'s future when `tokio::time::timeout` gives up is what
    // actually sends the kill, and this is the one test in this module
    // whose child would otherwise keep running for seconds after the probe
    // itself has already reported failure.
    #[cfg(unix)]
    #[tokio::test]
    async fn exec_probe_that_hangs_is_killed_and_times_out() {
        let short = Duration::from_millis(200);
        let prober = OsProber::new(None, BTreeMap::new());

        let start = std::time::Instant::now();
        let result = prober.probe(&exec_target("sleep 5"), short).await;
        let elapsed = start.elapsed();

        assert_eq!(result, Err(ProbeFailure::Timeout));
        assert!(
            elapsed < short * 3,
            "expected the probe to give up within a small multiple of {short:?}, took {elapsed:?}"
        );
    }

    /// How long the forked grandchild in
    /// [`a_timed_out_exec_probe_kills_the_grandchild_too`] sleeps: comfortably
    /// longer than [`REAP_DEADLINE`], so a passing run proves the kill reached
    /// it rather than that it finished on its own; short enough that a run
    /// panicking before [`Reaper`] fires leaves nothing lingering for a whole
    /// CI job.
    #[cfg(unix)]
    const ORPHAN_SLEEP_SECS: u32 = 30;

    /// How long [`assert_reaped`] waits for a pid to leave the process table.
    /// A signal that lands takes milliseconds; this is slack for a loaded
    /// runner, not an expected duration.
    #[cfg(unix)]
    const REAP_DEADLINE: Duration = Duration::from_secs(5);

    /// Last-resort net for a test that PANICS with real processes still
    /// alive, so a failing assertion never leaks a 30-second `sleep` into the
    /// rest of the run.
    ///
    /// Fires ONLY while panicking: on the success path the test has already
    /// proven the pid is gone, and signalling a pid the OS may since have
    /// recycled is a hazard rather than a safety net.
    #[cfg(unix)]
    struct Reaper(i32);

    #[cfg(unix)]
    impl Drop for Reaper {
        fn drop(&mut self) {
            if !std::thread::panicking() {
                return;
            }
            let _ = nix::sys::signal::kill(
                nix::unistd::Pid::from_raw(self.0),
                nix::sys::signal::Signal::SIGKILL,
            );
        }
    }

    /// Polls `kill(pid, None)` for `ESRCH` instead of sleeping a fixed guess.
    /// `kill(pid, None)` still returns `Ok` for a zombie, so only a
    /// transition all the way to `ESRCH` proves the process is really gone
    /// rather than exited-but-unreaped.
    ///
    /// Copied from `tests/real_runner.rs`'s own `assert_reaped` for the same
    /// reason that one is a copy of `daemon_e2e.rs`'s: an integration binary
    /// is a separate crate and cannot share a `#[cfg(test)]` helper with this
    /// module.
    #[cfg(unix)]
    async fn assert_reaped(pid: i32, what: &str) {
        let reaped = tokio::time::timeout(REAP_DEADLINE, async {
            loop {
                match nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid), None) {
                    Err(nix::errno::Errno::ESRCH) => break,
                    _ => tokio::time::sleep(Duration::from_millis(20)).await,
                }
            }
        })
        .await;
        assert!(reaped.is_ok(), "{what} (pid {pid}) is still alive");
    }

    // fails if a timed-out probe kills only the shell it spawned, which is
    // all `kill_on_drop(true)` can reach. The brief's rationale for that flag
    // — probes otherwise "accumulate processes until the box falls over" —
    // is about what the command leaves behind, and a single simple command is
    // the only case a leader-only kill actually covers, because `sh` execs
    // into it. `sleep &` is the same orphan shape
    // `tests/real_runner.rs`'s `a_graceful_stop_reaches_a_forked_grandchild`
    // pins for the stop ladder, pointed at the prober.
    #[cfg(unix)]
    #[tokio::test]
    async fn a_timed_out_exec_probe_kills_the_grandchild_too() {
        let dir = tempfile::tempdir().unwrap();
        let pidfile = dir.path().join("grandchild.pid");
        // The pid comes back through a FILE, not stdout: this probe's stdio
        // is `/dev/null` by design, and `probe()` hands back a verdict rather
        // than a pid. `echo $!` runs only after the fork, so a written file
        // is itself proof the grandchild existed.
        // `wait` rather than a second foreground `sleep`, matching
        // `real_runner.rs`'s wrapper: both hold the shell open past the
        // timeout, but `wait` leaves exactly ONE process for a failing run's
        // `Reaper` to clean up instead of two.
        let command = format!(
            "sleep {ORPHAN_SLEEP_SECS} & echo $! > \"{}\"; wait",
            pidfile.display()
        );
        // Three orders of magnitude more than a `sh` fork-and-write needs, so
        // the file is on disk long before the timeout fires, and still fast.
        let short = Duration::from_millis(500);

        let prober = OsProber::new(None, BTreeMap::new());
        let result = prober.probe(&exec_target(&command), short).await;
        assert_eq!(result, Err(ProbeFailure::Timeout));

        let grandchild: i32 = std::fs::read_to_string(&pidfile)
            .expect("fixture precondition: the shell must record its forked child's pid")
            .trim()
            .parse()
            .expect("`echo $!` prints a pid");
        let _reaper = Reaper(grandchild);

        // The assertion the whole test exists for.
        assert_reaped(grandchild, "the probe command's forked child").await;
    }

    // fails if a spawn-level failure (the probe is misconfigured, not the
    // app unhealthy) is conflated with an ordinary nonzero exit. A command
    // string naming a genuinely nonexistent binary does NOT exercise this:
    // confirmed empirically (`sh -c nonexistent_binary; echo $?` -> 127)
    // that `sh` itself always spawns and reports "not found" via its OWN
    // exit code, which this prober correctly treats as Rejected("127") per
    // the brief's own flat rule ("anything else is Rejected"), not as a
    // spawn failure. A nonexistent `cwd` is what forces spawn itself
    // (`Command::status`, which chdir's before exec) to return an `Err`
    // before any shell ever runs — the actually-misconfigured-probe case.
    #[cfg(unix)]
    #[tokio::test]
    async fn exec_probe_that_cannot_be_spawned_at_all_fails_as_transport() {
        let prober = OsProber::new(
            Some(PathBuf::from("/definitely/does/not/exist/shep-probe-test")),
            BTreeMap::new(),
        );
        let result = prober.probe(&exec_target("exit 0"), PROBE_TIMEOUT).await;
        assert!(
            matches!(result, Err(ProbeFailure::Transport(_))),
            "expected Transport, got {result:?}"
        );
    }

    // fails if `.envs()` is called without a preceding `.env_clear()` (the
    // canary, inherited from this test process's own real environment,
    // would then leak into the child alongside the prober's own var), and
    // fails if the prober ignores its own env entirely (the own var would
    // never appear). Does not mutate the real process environment (no
    // `std::env::set_var`): the canary is read from THIS process's already-
    // set `HOME`, which every dev machine and CI runner sets, rather than
    // written by the test.
    #[cfg(unix)]
    #[tokio::test]
    async fn exec_probe_sees_only_the_env_it_was_constructed_with() {
        assert!(
            std::env::var("HOME").is_ok(),
            "fixture precondition: this test process needs a real HOME to prove it does NOT leak"
        );
        let mut env = BTreeMap::new();
        env.insert("SHEP_PROBE_OWN_VAR".to_string(), "expected".to_string());
        let prober = OsProber::new(None, env);
        let command = "test \"$SHEP_PROBE_OWN_VAR\" = expected && [ -z \"${HOME:-}\" ]";
        let result = prober.probe(&exec_target(command), PROBE_TIMEOUT).await;
        assert_eq!(result, Ok(()));
    }

    // fails if the exec probe leaves stdio inherited, which is `Command`'s
    // default: a `curl`-style probe then writes its whole response body —
    // bearer tokens and all — into the daemon's own stdout once per interval.
    //
    // Asserted from INSIDE the child because nothing in the parent can read a
    // `Command`'s configured stdio back, and because libtest's capture swaps
    // a thread-local rather than fd 1 — an inherited child really does write
    // to the harness's own stdout. `/dev/null` is a character device that is
    // not a terminal; every realistic inherited stdout is a pipe (CI, any
    // captured run), a regular file (`cargo test > log`) or a terminal, and
    // each of those fails one of the two checks.
    #[cfg(unix)]
    #[tokio::test]
    async fn exec_probe_gets_null_stdio_rather_than_the_daemons() {
        let prober = OsProber::new(None, BTreeMap::new());
        let command = "[ -c /dev/fd/0 ] && [ ! -t 0 ] \
                       && [ -c /dev/fd/1 ] && [ ! -t 1 ] \
                       && [ -c /dev/fd/2 ] && [ ! -t 2 ]";
        let result = prober.probe(&exec_target(command), PROBE_TIMEOUT).await;
        assert_eq!(result, Ok(()));
    }

    // IR-10 dyn-compatibility smoke test.
    #[test]
    fn os_prober_is_dyn_compatible() {
        let _: &dyn Prober = &OsProber::new(None, BTreeMap::new());
    }

    #[test]
    fn debug_redacts_env_values_but_shows_the_count() {
        // IR-41: env may carry the sheep's secrets (e.g. DATABASE_URL).
        // Exact string pinned so a lazy derive(Debug) refactor fails here.
        let mut env = BTreeMap::new();
        env.insert("DATABASE_URL".to_string(), "postgres://secret".to_string());
        env.insert("RUST_LOG".to_string(), "info".to_string());
        let prober = OsProber::new(None, env);
        assert_eq!(
            format!("{prober:?}"),
            "OsProber { cwd: None, env: <2 vars>, .. }"
        );
    }
}