use core::fmt;
use core::future::Future;
use core::pin::Pin;
use core::time::Duration;
use std::collections::BTreeMap;
use std::path::PathBuf;
use std::process::Stdio;
use tokio::io::{AsyncBufReadExt, AsyncReadExt, AsyncWriteExt, BufReader};
use tokio::net::TcpStream;
use tokio::process::Command;
use shep_core::config::ProbeTarget;
use super::{ProbeFailure, Prober};
const HTTP_STATUS_LINE_CAP: u64 = 8 * 1024;
const HTTP_DEFAULT_PORT: u16 = 80;
pub struct OsProber {
cwd: Option<PathBuf>,
env: BTreeMap<String, String>,
}
impl OsProber {
#[must_use]
pub fn new(cwd: Option<PathBuf>, env: BTreeMap<String, String>) -> Self {
Self { cwd, env }
}
}
impl fmt::Debug for OsProber {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("OsProber")
.field("cwd", &self.cwd)
.field("env", &format_args!("<{} vars>", self.env.len()))
.finish_non_exhaustive()
}
}
impl Prober for OsProber {
fn probe<'a>(
&'a self,
target: &'a ProbeTarget,
timeout: Duration,
) -> Pin<Box<dyn Future<Output = Result<(), ProbeFailure>> + Send + 'a>> {
Box::pin(async move {
match target {
ProbeTarget::Http { host, port, path } => {
probe_http(host, *port, path, timeout).await
}
ProbeTarget::Tcp { host, port } => probe_tcp(host, *port, timeout).await,
ProbeTarget::Exec { command } => self.probe_exec(command, timeout).await,
}
})
}
}
impl OsProber {
async fn probe_exec(&self, command: &str, timeout: Duration) -> Result<(), ProbeFailure> {
#[cfg(unix)]
let mut cmd = {
let mut c = Command::new("sh");
c.arg("-c").arg(command);
c
};
#[cfg(windows)]
let mut cmd = {
let mut c = Command::new("cmd");
c.arg("/C").arg(command);
c
};
cmd.kill_on_drop(true);
#[cfg(unix)]
cmd.process_group(0);
cmd.stdin(Stdio::null());
cmd.stdout(Stdio::null());
cmd.stderr(Stdio::null());
if let Some(cwd) = &self.cwd {
cmd.current_dir(cwd);
}
cmd.env_clear().envs(&self.env);
let mut child = cmd
.spawn()
.map_err(|err| ProbeFailure::Transport(err.to_string()))?;
let pid = child.id();
let waited = tokio::time::timeout(timeout, child.wait()).await;
kill_probe_group(pid);
match waited {
Ok(Ok(status)) if status.success() => Ok(()),
Ok(Ok(status)) => Err(ProbeFailure::Rejected(exit_code_text(&status))),
Ok(Err(err)) => Err(ProbeFailure::Transport(err.to_string())),
Err(_elapsed) => Err(ProbeFailure::Timeout),
}
}
}
#[cfg(unix)]
fn kill_probe_group(pid: Option<u32>) {
let Some(pid) = pid else {
return;
};
if let Err(error) = crate::tokio_runner::signal_group(pid, nix::sys::signal::Signal::SIGKILL) {
tracing::warn!(pid, %error, "probe process group kill failed");
}
}
#[cfg(windows)]
fn kill_probe_group(_pid: Option<u32>) {}
fn exit_code_text(status: &std::process::ExitStatus) -> String {
status.code().map_or_else(
|| "terminated by signal".to_string(),
|code| code.to_string(),
)
}
async fn probe_http(
host: &str,
port: u16,
path: &str,
timeout: Duration,
) -> Result<(), ProbeFailure> {
match tokio::time::timeout(timeout, http_roundtrip(host, port, path)).await {
Ok(result) => result,
Err(_elapsed) => Err(ProbeFailure::Timeout),
}
}
async fn http_roundtrip(host: &str, port: u16, path: &str) -> Result<(), ProbeFailure> {
let mut stream = TcpStream::connect((host, port))
.await
.map_err(|err| ProbeFailure::Transport(err.to_string()))?;
let header_host = host_header(host, port);
let request =
format!("GET {path} HTTP/1.1\r\nHost: {header_host}\r\nConnection: close\r\n\r\n");
stream
.write_all(request.as_bytes())
.await
.map_err(|err| ProbeFailure::Transport(err.to_string()))?;
let status_line = read_status_line(stream).await?;
evaluate_status_line(&status_line)
}
fn host_header(host: &str, port: u16) -> String {
let host = if host.contains(':') {
format!("[{host}]")
} else {
host.to_string()
};
if port == HTTP_DEFAULT_PORT {
host
} else {
format!("{host}:{port}")
}
}
async fn read_status_line(stream: TcpStream) -> Result<String, ProbeFailure> {
let mut reader = BufReader::new(stream.take(HTTP_STATUS_LINE_CAP));
let mut buf = Vec::new();
reader
.read_until(b'\n', &mut buf)
.await
.map_err(|err| ProbeFailure::Transport(err.to_string()))?;
if buf.is_empty() {
return Err(ProbeFailure::Transport(
"connection closed before a response was received".to_string(),
));
}
Ok(String::from_utf8_lossy(&buf).into_owned())
}
fn evaluate_status_line(line: &str) -> Result<(), ProbeFailure> {
let trimmed = line.trim_end_matches(['\r', '\n']);
let Some(after_version) = trimmed.strip_prefix("HTTP/") else {
return Err(ProbeFailure::Rejected(format!(
"not an HTTP status line: {trimmed:?}"
)));
};
match after_version
.split(' ')
.nth(1)
.and_then(|token| token.parse::<u16>().ok())
{
Some(200..=299) => Ok(()),
Some(code) => Err(ProbeFailure::Rejected(code.to_string())),
None => Err(ProbeFailure::Rejected(format!(
"malformed HTTP status line: {trimmed:?}"
))),
}
}
async fn probe_tcp(host: &str, port: u16, timeout: Duration) -> Result<(), ProbeFailure> {
match tokio::time::timeout(timeout, TcpStream::connect((host, port))).await {
Ok(Ok(_stream)) => Ok(()),
Ok(Err(err)) => Err(ProbeFailure::Transport(err.to_string())),
Err(_elapsed) => Err(ProbeFailure::Timeout),
}
}
#[cfg(test)]
mod tests {
use core::time::Duration;
use std::collections::BTreeMap;
use tokio::net::TcpListener;
use super::*;
use crate::testing_http::{HttpReply, loopback_http, loopback_http_on};
const PROBE_TIMEOUT: Duration = Duration::from_secs(5);
fn http_target(port: u16, path: &str) -> ProbeTarget {
ProbeTarget::Http {
host: "127.0.0.1".to_string(),
port,
path: path.to_string(),
}
}
#[tokio::test]
async fn passing_status_codes_are_accepted_across_the_2xx_range() {
for code in [200u16, 204, 299] {
let server = loopback_http(vec![HttpReply::Status(code)]).await;
let prober = OsProber::new(None, BTreeMap::new());
let result = prober
.probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
.await;
assert_eq!(result, Ok(()), "status {code} should pass");
}
}
#[tokio::test]
async fn a_301_is_rejected_not_followed() {
let server = loopback_http(vec![HttpReply::Status(301)]).await;
let prober = OsProber::new(None, BTreeMap::new());
let result = prober
.probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
.await;
assert_eq!(result, Err(ProbeFailure::Rejected("301".to_string())));
}
#[tokio::test]
async fn a_500_is_rejected() {
let server = loopback_http(vec![HttpReply::Status(500)]).await;
let prober = OsProber::new(None, BTreeMap::new());
let result = prober
.probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
.await;
assert_eq!(result, Err(ProbeFailure::Rejected("500".to_string())));
}
#[tokio::test]
async fn a_hanging_response_times_out_within_a_small_multiple_of_the_budget() {
let server = loopback_http(vec![HttpReply::Hang]).await;
let short = Duration::from_millis(150);
let prober = OsProber::new(None, BTreeMap::new());
let start = std::time::Instant::now();
let result = prober
.probe(&http_target(server.addr.port(), "/"), short)
.await;
let elapsed = start.elapsed();
assert_eq!(result, Err(ProbeFailure::Timeout));
assert!(
elapsed < short * 3,
"expected the probe to give up within a small multiple of {short:?}, took {elapsed:?}"
);
}
#[tokio::test]
async fn an_http_probe_sends_one_get_carrying_the_targets_path_and_a_host_header() {
let mut server = loopback_http(vec![HttpReply::Status(200)]).await;
let prober = OsProber::new(None, BTreeMap::new());
let result = prober
.probe(&http_target(server.addr.port(), "/healthz"), PROBE_TIMEOUT)
.await;
assert_eq!(result, Ok(()));
let port = server.addr.port();
assert_eq!(
server.next_request().await,
format!("GET /healthz HTTP/1.1\r\nHost: 127.0.0.1:{port}\r\nConnection: close\r\n\r\n")
);
}
#[test]
fn an_ipv6_host_is_re_bracketed_for_the_host_header() {
assert_eq!(host_header("::1", 8080), "[::1]:8080");
assert_eq!(host_header("2001:db8::1", 443), "[2001:db8::1]:443");
assert_eq!(host_header("::1", 80), "[::1]");
assert_eq!(host_header("127.0.0.1", 8080), "127.0.0.1:8080");
assert_eq!(host_header("127.0.0.1", 80), "127.0.0.1");
assert_eq!(host_header("localhost", 9000), "localhost:9000");
}
#[tokio::test]
async fn an_ipv6_target_connects_unbracketed_and_brackets_the_host_header() {
let mut server = loopback_http_on("[::1]:0", vec![HttpReply::Status(200)]).await;
let target = ProbeTarget::Http {
host: "::1".to_string(),
port: server.addr.port(),
path: "/".to_string(),
};
let prober = OsProber::new(None, BTreeMap::new());
let result = prober.probe(&target, PROBE_TIMEOUT).await;
assert_eq!(result, Ok(()), "an IPv6 target must connect at all");
let port = server.addr.port();
assert_eq!(
server.next_request().await,
format!("GET / HTTP/1.1\r\nHost: [::1]:{port}\r\nConnection: close\r\n\r\n")
);
}
#[tokio::test]
async fn a_port_with_nothing_listening_fails_as_transport() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
drop(listener);
let prober = OsProber::new(None, BTreeMap::new());
let result = prober
.probe(&http_target(addr.port(), "/"), PROBE_TIMEOUT)
.await;
assert!(
matches!(result, Err(ProbeFailure::Transport(_))),
"expected Transport, got {result:?}"
);
}
#[tokio::test]
async fn a_garbage_first_line_is_rejected_not_panicked_on() {
let server = loopback_http(vec![HttpReply::Raw("not http\r\n".to_string())]).await;
let prober = OsProber::new(None, BTreeMap::new());
let result = prober
.probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
.await;
assert!(
matches!(result, Err(ProbeFailure::Rejected(_))),
"expected Rejected, got {result:?}"
);
}
#[tokio::test]
async fn a_status_line_with_no_code_after_the_version_is_rejected_not_panicked_on() {
let server = loopback_http(vec![HttpReply::Raw("HTTP/1.1\r\n".to_string())]).await;
let prober = OsProber::new(None, BTreeMap::new());
let result = prober
.probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
.await;
assert!(
matches!(result, Err(ProbeFailure::Rejected(_))),
"expected Rejected, got {result:?}"
);
}
#[tokio::test]
async fn a_2xx_from_a_service_that_is_not_http_is_rejected() {
let server =
loopback_http(vec![HttpReply::Raw("BANANA 204 whatever\r\n".to_string())]).await;
let prober = OsProber::new(None, BTreeMap::new());
let result = prober
.probe(&http_target(server.addr.port(), "/"), PROBE_TIMEOUT)
.await;
assert!(
matches!(result, Err(ProbeFailure::Rejected(_))),
"expected Rejected, got {result:?}"
);
}
#[tokio::test]
async fn tcp_probe_against_a_bound_listener_passes() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
let prober = OsProber::new(None, BTreeMap::new());
let target = ProbeTarget::Tcp {
host: "127.0.0.1".to_string(),
port: addr.port(),
};
let result = prober.probe(&target, PROBE_TIMEOUT).await;
assert_eq!(result, Ok(()));
drop(listener);
}
#[tokio::test]
async fn tcp_probe_against_a_closed_port_fails_as_transport() {
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
let addr = listener.local_addr().unwrap();
drop(listener);
let prober = OsProber::new(None, BTreeMap::new());
let target = ProbeTarget::Tcp {
host: "127.0.0.1".to_string(),
port: addr.port(),
};
let result = prober.probe(&target, PROBE_TIMEOUT).await;
assert!(
matches!(result, Err(ProbeFailure::Transport(_))),
"expected Transport, got {result:?}"
);
}
#[tokio::test]
async fn tcp_probe_against_a_non_routable_address_gives_up_within_the_budget() {
let short = Duration::from_millis(300);
let prober = OsProber::new(None, BTreeMap::new());
let target = ProbeTarget::Tcp {
host: "192.0.2.1".to_string(),
port: 1,
};
let start = std::time::Instant::now();
let result = prober.probe(&target, short).await;
let elapsed = start.elapsed();
assert!(
matches!(
result,
Err(ProbeFailure::Timeout | ProbeFailure::Transport(_))
),
"expected Timeout or Transport, got {result:?}"
);
assert!(
elapsed < short * 3,
"expected the probe to give up within a small multiple of {short:?}, took {elapsed:?}"
);
}
#[cfg(unix)]
fn exec_target(command: &str) -> ProbeTarget {
ProbeTarget::Exec {
command: command.to_string(),
}
}
#[cfg(unix)]
#[tokio::test]
async fn exec_probe_exit_zero_passes() {
let prober = OsProber::new(None, BTreeMap::new());
let result = prober.probe(&exec_target("exit 0"), PROBE_TIMEOUT).await;
assert_eq!(result, Ok(()));
}
#[cfg(unix)]
#[tokio::test]
async fn exec_probe_nonzero_exit_is_rejected_with_the_code() {
let prober = OsProber::new(None, BTreeMap::new());
let result = prober.probe(&exec_target("exit 3"), PROBE_TIMEOUT).await;
assert_eq!(result, Err(ProbeFailure::Rejected("3".to_string())));
}
#[cfg(unix)]
#[tokio::test]
async fn exec_probe_that_hangs_is_killed_and_times_out() {
let short = Duration::from_millis(200);
let prober = OsProber::new(None, BTreeMap::new());
let start = std::time::Instant::now();
let result = prober.probe(&exec_target("sleep 5"), short).await;
let elapsed = start.elapsed();
assert_eq!(result, Err(ProbeFailure::Timeout));
assert!(
elapsed < short * 3,
"expected the probe to give up within a small multiple of {short:?}, took {elapsed:?}"
);
}
#[cfg(unix)]
const ORPHAN_SLEEP_SECS: u32 = 30;
#[cfg(unix)]
const REAP_DEADLINE: Duration = Duration::from_secs(5);
#[cfg(unix)]
struct Reaper(i32);
#[cfg(unix)]
impl Drop for Reaper {
fn drop(&mut self) {
if !std::thread::panicking() {
return;
}
let _ = nix::sys::signal::kill(
nix::unistd::Pid::from_raw(self.0),
nix::sys::signal::Signal::SIGKILL,
);
}
}
#[cfg(unix)]
async fn assert_reaped(pid: i32, what: &str) {
let reaped = tokio::time::timeout(REAP_DEADLINE, async {
loop {
match nix::sys::signal::kill(nix::unistd::Pid::from_raw(pid), None) {
Err(nix::errno::Errno::ESRCH) => break,
_ => tokio::time::sleep(Duration::from_millis(20)).await,
}
}
})
.await;
assert!(reaped.is_ok(), "{what} (pid {pid}) is still alive");
}
#[cfg(unix)]
#[tokio::test]
async fn a_timed_out_exec_probe_kills_the_grandchild_too() {
let dir = tempfile::tempdir().unwrap();
let pidfile = dir.path().join("grandchild.pid");
let command = format!(
"sleep {ORPHAN_SLEEP_SECS} & echo $! > \"{}\"; wait",
pidfile.display()
);
let short = Duration::from_millis(500);
let prober = OsProber::new(None, BTreeMap::new());
let result = prober.probe(&exec_target(&command), short).await;
assert_eq!(result, Err(ProbeFailure::Timeout));
let grandchild: i32 = std::fs::read_to_string(&pidfile)
.expect("fixture precondition: the shell must record its forked child's pid")
.trim()
.parse()
.expect("`echo $!` prints a pid");
let _reaper = Reaper(grandchild);
assert_reaped(grandchild, "the probe command's forked child").await;
}
#[cfg(unix)]
#[tokio::test]
async fn a_passing_exec_probe_kills_the_grandchild_too() {
let dir = tempfile::tempdir().unwrap();
let pidfile = dir.path().join("grandchild.pid");
let command = format!(
"sleep {ORPHAN_SLEEP_SECS} & echo $! > \"{}\"",
pidfile.display()
);
let prober = OsProber::new(None, BTreeMap::new());
let result = prober.probe(&exec_target(&command), PROBE_TIMEOUT).await;
assert_eq!(
result,
Ok(()),
"fixture precondition: the probe command must pass"
);
let grandchild: i32 = std::fs::read_to_string(&pidfile)
.expect("fixture precondition: the shell must record its forked child's pid")
.trim()
.parse()
.expect("`echo $!` prints a pid");
let _reaper = Reaper(grandchild);
assert_reaped(grandchild, "a passing probe command's forked child").await;
}
#[cfg(unix)]
#[tokio::test]
async fn exec_probe_that_cannot_be_spawned_at_all_fails_as_transport() {
let prober = OsProber::new(
Some(PathBuf::from("/definitely/does/not/exist/shep-probe-test")),
BTreeMap::new(),
);
let result = prober.probe(&exec_target("exit 0"), PROBE_TIMEOUT).await;
assert!(
matches!(result, Err(ProbeFailure::Transport(_))),
"expected Transport, got {result:?}"
);
}
#[cfg(unix)]
#[tokio::test]
async fn exec_probe_sees_only_the_env_it_was_constructed_with() {
assert!(
std::env::var("HOME").is_ok(),
"fixture precondition: this test process needs a real HOME to prove it does NOT leak"
);
let mut env = BTreeMap::new();
env.insert("SHEP_PROBE_OWN_VAR".to_string(), "expected".to_string());
let prober = OsProber::new(None, env);
let command = "test \"$SHEP_PROBE_OWN_VAR\" = expected && [ -z \"${HOME:-}\" ]";
let result = prober.probe(&exec_target(command), PROBE_TIMEOUT).await;
assert_eq!(result, Ok(()));
}
#[cfg(unix)]
#[tokio::test]
async fn exec_probe_gets_null_stdio_rather_than_the_daemons() {
let prober = OsProber::new(None, BTreeMap::new());
let command = "[ -c /dev/fd/0 ] && [ ! -t 0 ] \
&& [ -c /dev/fd/1 ] && [ ! -t 1 ] \
&& [ -c /dev/fd/2 ] && [ ! -t 2 ]";
let result = prober.probe(&exec_target(command), PROBE_TIMEOUT).await;
assert_eq!(result, Ok(()));
}
#[test]
fn os_prober_is_dyn_compatible() {
let _: &dyn Prober = &OsProber::new(None, BTreeMap::new());
}
#[test]
fn debug_redacts_env_values_but_shows_the_count() {
let mut env = BTreeMap::new();
env.insert("DATABASE_URL".to_string(), "postgres://secret".to_string());
env.insert("RUST_LOG".to_string(), "info".to_string());
let prober = OsProber::new(None, env);
assert_eq!(
format!("{prober:?}"),
"OsProber { cwd: None, env: <2 vars>, .. }"
);
}
}