use core::future::Future;
use core::pin::Pin;
use core::time::Duration;
use std::net::SocketAddr;
use std::path::{Path, PathBuf};
use std::sync::atomic::{AtomicU64, AtomicUsize, Ordering};
use std::sync::{Arc, LazyLock, Mutex, PoisonError};
use chrono::{DateTime, Utc};
use shep_core::config::{AppConfig, ProbeConfig, ProbeKind, ProbeTarget, ResolvedApp, normalize};
use shep_core::paths::ShepPaths;
use shep_core::status::ProcStatus;
use shep_core::values::{MemSize, UpDuration};
use tokio::io::{AsyncReadExt, AsyncWriteExt};
use tokio::net::TcpListener;
use tokio::sync::{broadcast, mpsc, watch};
use crate::assemble::assemble;
use crate::bus::SharedEvent;
use crate::cron::{Clock, DEFAULT_MAX_CRON_SLEEP};
use crate::entry::{ProcessEntry, ReloadState, RestartBudget};
use crate::extras::{Extras, ExtrasReports, LivenessReport};
use crate::fake::{FIRST_SCRIPTED_PID, ProcScript, ScriptedRunner};
use crate::limits::sample::{MemorySampler, ProcessIdentity, ProcessRss};
use crate::limits::stats::StatsState;
use crate::limits::{LimitBreach, LimitEnforcer};
use crate::privilege::SpawnIdentity;
use crate::probes::{ProbeFailure, Prober};
use crate::rpc::RpcContext;
use crate::runner::{ProcIo, ProcessRunner, RunnerError, SpawnSpec};
use crate::snapshot::FlockRegistry;
use crate::supervisor::SupervisorBuilder;
#[derive(Debug, Clone, Default)]
pub(crate) struct LogCapture(Arc<Mutex<Vec<u8>>>);
impl LogCapture {
fn rendered(&self) -> String {
let buffer = self.0.lock().unwrap_or_else(PoisonError::into_inner);
String::from_utf8_lossy(&buffer).into_owned()
}
}
impl std::io::Write for LogCapture {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogCapture {
type Writer = Self;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
static SECOND_DISPATCH: LazyLock<tracing::Dispatch> =
LazyLock::new(|| tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default()));
pub(crate) fn capture_logs(f: impl FnOnce()) -> String {
LazyLock::force(&SECOND_DISPATCH);
let capture = LogCapture::default();
let subscriber = tracing_subscriber::fmt()
.with_writer(capture.clone())
.with_ansi(false)
.with_max_level(tracing::Level::TRACE)
.finish();
tracing::subscriber::with_default(subscriber, f);
capture.rendered()
}
fn racing_warn() {
tracing::warn!("a callsite two threads reach");
}
#[test]
fn a_sibling_thread_reaching_a_callsite_first_cannot_empty_the_capture() {
let (scope_open, await_scope) = std::sync::mpsc::channel();
let (registered, await_registration) = std::sync::mpsc::channel();
let sibling = std::thread::spawn(move || {
await_scope.recv().expect("the capture must open its scope");
racing_warn();
registered
.send(())
.expect("the capture must still be waiting");
});
let rendered = capture_logs(|| {
scope_open
.send(())
.expect("the sibling must still be waiting");
await_registration
.recv()
.expect("the sibling must register before this emit");
racing_warn();
});
sibling.join().expect("the sibling thread must not panic");
assert!(
rendered.contains("a callsite two threads reach"),
"a sibling thread registering first must not disable the callsite for \
this capture: {rendered:?}"
);
}
pub(crate) fn test_paths(dir: &tempfile::TempDir) -> ShepPaths {
let home = dir.path().to_path_buf();
ShepPaths::resolve(
&|key| (key == "SHEP_HOME").then(|| home.display().to_string()),
std::path::Path::new("/nonexistent"),
)
}
#[derive(Debug)]
pub(crate) struct SharedRunner(pub(crate) Arc<ScriptedRunner>);
impl ProcessRunner for SharedRunner {
type Proc = crate::fake::FakeProc;
fn spawn(&self, spec: &SpawnSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
self.0.spawn(spec)
}
}
#[cfg(unix)]
#[derive(Debug)]
pub(crate) struct AnnouncingRunner<R> {
inner: R,
target: PathBuf,
}
#[cfg(unix)]
impl<R> AnnouncingRunner<R> {
pub(crate) fn new(inner: R, target: &Path) -> Self {
Self {
inner,
target: target.to_path_buf(),
}
}
}
#[cfg(unix)]
impl<R: ProcessRunner> ProcessRunner for AnnouncingRunner<R> {
type Proc = R::Proc;
#[track_caller]
fn spawn(&self, spec: &SpawnSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
let socket = std::os::unix::net::UnixDatagram::unbound()
.expect("a test must be able to open an unbound datagram socket");
socket
.send_to(b"SPAWNED\n", &self.target)
.expect("the test's listening socket must be bound before the spawn");
self.inner.spawn(spec)
}
}
pub(crate) fn proptest_config(local_cases: u32) -> proptest::test_runner::Config {
let default = proptest::test_runner::Config::default();
if std::env::var_os("PROPTEST_CASES").is_some() {
default
} else {
proptest::test_runner::Config {
cases: local_cases,
..default
}
}
}
pub(crate) fn touch(root: &Path, rel: &str) -> std::io::Result<PathBuf> {
let path = root.join(rel);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::write(&path, b"x")?;
Ok(path)
}
pub(crate) struct Harness {
pub(crate) ctx: RpcContext,
_dir: tempfile::TempDir,
_events_rx: broadcast::Receiver<SharedEvent>,
pub(crate) shutdown_rx: watch::Receiver<bool>,
pub(crate) breaches: mpsc::Receiver<LimitBreach>,
pub(crate) liveness: mpsc::Receiver<LivenessReport>,
pub(crate) stats: Arc<StatsState>,
}
const HARNESS_REPORT_CAPACITY: usize = 16;
pub(crate) fn harness(scripts: Vec<ProcScript>) -> Harness {
harness_sampling(scripts, vec![vec![]])
}
pub(crate) fn harness_with_stats(scripts: Vec<ProcScript>) -> Harness {
let tree = |cpu_ms| {
vec![ProcessRss {
pid: FIRST_SCRIPTED_PID,
parent: None,
bytes: SCRIPTED_TREE_BYTES,
cpu_ms,
}]
};
harness_sampling(scripts, vec![tree(1_000), tree(2_500), tree(2_501)])
}
pub(crate) const SCRIPTED_TREE_BYTES: u64 = 4096;
pub(crate) fn harness_at_pid(scripts: Vec<ProcScript>, pid: u32) -> Harness {
harness_sampling_with(ScriptedRunner::new(scripts).spawning_at(pid), vec![vec![]])
}
pub(crate) fn harness_refusing(scripts: Vec<ProcScript>, names: &[&str]) -> Harness {
harness_sampling_with(ScriptedRunner::new(scripts).refusing(names), vec![vec![]])
}
pub(crate) fn harness_failing_to_spawn(scripts: Vec<ProcScript>, names: &[&str]) -> Harness {
harness_sampling_with(
ScriptedRunner::new(scripts).failing_to_spawn(names),
vec![vec![]],
)
}
fn harness_sampling(scripts: Vec<ProcScript>, readings: Vec<Vec<ProcessRss>>) -> Harness {
harness_sampling_with(ScriptedRunner::new(scripts), readings)
}
fn harness_sampling_with(runner: ScriptedRunner, readings: Vec<Vec<ProcessRss>>) -> Harness {
harness_with_runner(runner, |reports| {
let sampler: Arc<dyn MemorySampler> = Arc::new(ScriptedSampler::new(readings));
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
Extras {
clock: Arc::new(TestClock::starting_at(
"2026-01-01T00:00:00Z"
.parse()
.expect("a valid RFC3339 timestamp"),
)),
enforcer: Arc::new(crate::limits::PollingEnforcer::start(
sampler,
reports.breaches.clone(),
Arc::clone(&stats),
)),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats,
}
})
}
pub(crate) fn harness_identifying(
scripts: Vec<ProcScript>,
identities: Vec<ProcessIdentity>,
) -> Harness {
harness_with_extras(scripts, |reports| {
let sampler: Arc<dyn MemorySampler> =
Arc::new(ScriptedSampler::identifying(vec![identities]));
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
Extras {
clock: Arc::new(TestClock::starting_at(
"2026-01-01T00:00:00Z"
.parse()
.expect("a valid RFC3339 timestamp"),
)),
enforcer: Arc::new(crate::limits::PollingEnforcer::start(
sampler,
reports.breaches.clone(),
Arc::clone(&stats),
)),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats,
}
})
}
pub(crate) fn identity(pid: u32, parent: Option<u32>, name: &str) -> ProcessIdentity {
ProcessIdentity {
pid,
parent,
name: name.to_string(),
}
}
pub(crate) fn idle_stats() -> Arc<StatsState> {
Arc::new(StatsState::new(Arc::new(ScriptedSampler::new(vec![
vec![],
]))))
}
pub(crate) fn harness_with_extras(
scripts: Vec<ProcScript>,
build_extras: impl FnOnce(ExtrasReports) -> Extras,
) -> Harness {
harness_with_runner(ScriptedRunner::new(scripts), build_extras)
}
pub(crate) fn harness_with_runner(
runner: ScriptedRunner,
build_extras: impl FnOnce(ExtrasReports) -> Extras,
) -> Harness {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (events, events_rx) = crate::bus::test_bus(256);
let (breach_tx, breaches) = mpsc::channel(HARNESS_REPORT_CAPACITY);
let (live_tx, liveness) = mpsc::channel(HARNESS_REPORT_CAPACITY);
let extras = build_extras(ExtrasReports {
breaches: breach_tx,
liveness: live_tx,
});
let stats = Arc::clone(&extras.stats);
let supervisor = SupervisorBuilder::new(runner, paths.clone(), events.clone())
.extras(extras)
.spawn();
let (shutdown, shutdown_rx) = watch::channel(false);
Harness {
ctx: RpcContext {
supervisor,
events,
registry: FlockRegistry::new(),
snapshot_path: paths.snapshot.clone(),
dogs_config: paths.dogs_config.clone(),
known_dogs: crate::rpc::KnownDogs::new(
["metrics".to_string(), "bark".to_string()]
.into_iter()
.collect(),
),
dog_names: Vec::new(),
boot_first_dogs: Vec::new(),
paths: paths.clone(),
daemon_version: "0.1.0".to_string(),
dog_refusals: crate::dogs::DogRefusals::new(),
peer_contacts: crate::dogs::PeerContacts::new(),
pid: 4242,
shutdown: Arc::new(shutdown),
stats: Arc::clone(&stats),
},
_dir: dir,
_events_rx: events_rx,
shutdown_rx,
breaches,
liveness,
stats,
}
}
#[track_caller]
pub(crate) fn app_with(name: &str, mutate: impl FnOnce(&mut AppConfig)) -> ResolvedApp {
let mut app = AppConfig::minimal(name, "./srv");
mutate(&mut app);
normalize(app).expect("the fixture app must normalize")
}
pub(crate) fn armed_entry(
id: u32,
instance: u32,
pid: u32,
app: ResolvedApp,
paths: &ShepPaths,
) -> ProcessEntry {
let spec = assemble(&app, instance, paths, None);
ProcessEntry {
id,
spec: app,
pending: None,
pending_reidentifies: false,
overridden: Vec::new(),
instance,
status: ProcStatus::Online,
pid: Some(pid),
restarts: 0,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: SpawnIdentity::Resolved(None),
out_file: spec.out_file,
err_file: spec.err_file,
dog: None,
last_exit: None,
}
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub(crate) struct ArmCall {
pub(crate) id: u32,
pub(crate) root_pid: u32,
pub(crate) limit: MemSize,
}
#[derive(Debug, Default)]
pub(crate) struct RecordingEnforcer {
arms: Mutex<Vec<ArmCall>>,
disarms: Mutex<Vec<u32>>,
}
impl RecordingEnforcer {
pub(crate) fn arms(&self) -> Vec<ArmCall> {
self.arms
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone()
}
pub(crate) fn disarms(&self) -> Vec<u32> {
self.disarms
.lock()
.unwrap_or_else(PoisonError::into_inner)
.clone()
}
}
impl LimitEnforcer for RecordingEnforcer {
fn arm(&self, id: u32, root_pid: u32, limit: MemSize) {
self.arms
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push(ArmCall {
id,
root_pid,
limit,
});
}
fn disarm(&self, id: u32) {
self.disarms
.lock()
.unwrap_or_else(PoisonError::into_inner)
.push(id);
}
}
pub(crate) struct TestClock {
epoch: DateTime<Utc>,
started: tokio::time::Instant,
reads: AtomicUsize,
}
impl TestClock {
pub(crate) fn starting_at(epoch: DateTime<Utc>) -> Self {
Self {
epoch,
started: tokio::time::Instant::now(),
reads: AtomicUsize::new(0),
}
}
pub(crate) fn reads(&self) -> usize {
self.reads.load(Ordering::Relaxed)
}
}
impl Clock for TestClock {
fn now_utc(&self) -> DateTime<Utc> {
self.reads.fetch_add(1, Ordering::Relaxed);
let elapsed =
chrono::Duration::from_std(self.started.elapsed()).unwrap_or(chrono::Duration::MAX);
self.epoch + elapsed
}
}
pub(crate) struct ScriptedSampler {
readings: Vec<Vec<ProcessRss>>,
calls: AtomicUsize,
identities: Vec<Vec<ProcessIdentity>>,
identify_calls: AtomicUsize,
}
impl ScriptedSampler {
pub(crate) fn new(readings: Vec<Vec<ProcessRss>>) -> Self {
assert!(
!readings.is_empty(),
"ScriptedSampler needs at least one reading to replay"
);
Self {
readings,
calls: AtomicUsize::new(0),
identities: Vec::new(),
identify_calls: AtomicUsize::new(0),
}
}
pub(crate) fn identifying(tables: Vec<Vec<ProcessIdentity>>) -> Self {
assert!(
!tables.is_empty(),
"ScriptedSampler::identifying needs at least one table to replay"
);
Self {
readings: vec![vec![]],
calls: AtomicUsize::new(0),
identities: tables,
identify_calls: AtomicUsize::new(0),
}
}
pub(crate) fn calls(&self) -> usize {
self.calls.load(Ordering::Relaxed)
}
}
impl MemorySampler for ScriptedSampler {
fn sample(&self) -> Vec<ProcessRss> {
let call = self.calls.fetch_add(1, Ordering::Relaxed);
let index = call.min(self.readings.len() - 1);
self.readings[index].clone()
}
fn identify(&self) -> Vec<ProcessIdentity> {
if self.identities.is_empty() {
return Vec::new();
}
let call = self.identify_calls.fetch_add(1, Ordering::Relaxed);
let index = call.min(self.identities.len() - 1);
self.identities[index].clone()
}
}
pub(crate) struct ScriptedProber {
script: Vec<Result<(), ProbeFailure>>,
calls: AtomicUsize,
delay: Duration,
last_timeout_ms: AtomicU64,
}
impl ScriptedProber {
pub(crate) fn new(script: Vec<Result<(), ProbeFailure>>) -> Self {
Self {
script,
calls: AtomicUsize::new(0),
delay: Duration::ZERO,
last_timeout_ms: AtomicU64::new(0),
}
}
pub(crate) fn with_delay(mut self, delay: Duration) -> Self {
self.delay = delay;
self
}
pub(crate) fn calls(&self) -> usize {
self.calls.load(Ordering::Relaxed)
}
pub(crate) fn last_timeout(&self) -> Duration {
Duration::from_millis(self.last_timeout_ms.load(Ordering::Relaxed))
}
}
impl Prober for ScriptedProber {
fn probe<'a>(
&'a self,
_target: &'a ProbeTarget,
timeout: Duration,
) -> Pin<Box<dyn Future<Output = Result<(), ProbeFailure>> + Send + 'a>> {
Box::pin(async move {
let call = self.calls.fetch_add(1, Ordering::Relaxed);
self.last_timeout_ms
.store(timeout.as_millis() as u64, Ordering::Relaxed);
tokio::time::sleep(self.delay).await;
if self.script.is_empty() {
return Ok(());
}
let index = call.min(self.script.len() - 1);
self.script[index].clone()
})
}
}
pub(crate) fn probe_config(kind: ProbeKind, target: &str) -> ProbeConfig {
ProbeConfig {
kind,
target: target.to_string(),
interval: UpDuration::from_millis(10_000),
timeout: UpDuration::from_millis(5_000),
failure_threshold: 3,
}
}
pub(crate) enum HttpReply {
Status(u16),
Raw(String),
Hang,
}
const REQUEST_HEAD_CAP: usize = 8 * 1024;
const REQUEST_DEADLINE: Duration = Duration::from_secs(5);
pub(crate) struct LoopbackHttp {
pub(crate) addr: SocketAddr,
requests: mpsc::UnboundedReceiver<String>,
accept_loop: tokio::task::JoinHandle<()>,
}
impl LoopbackHttp {
pub(crate) async fn next_request(&mut self) -> String {
tokio::time::timeout(REQUEST_DEADLINE, self.requests.recv())
.await
.expect("the fake received no request within the deadline")
.expect("the fake's accept loop ended without sending a request")
}
}
impl Drop for LoopbackHttp {
fn drop(&mut self) {
self.accept_loop.abort();
}
}
pub(crate) async fn loopback_http(script: Vec<HttpReply>) -> LoopbackHttp {
loopback_http_on("127.0.0.1:0", script).await
}
pub(crate) async fn loopback_http_on(bind: &str, script: Vec<HttpReply>) -> LoopbackHttp {
let listener = TcpListener::bind(bind)
.await
.unwrap_or_else(|err| panic!("bind loopback HTTP fake on {bind}: {err}"));
let addr = listener.local_addr().expect("read bound loopback address");
let (requests_tx, requests) = mpsc::unbounded_channel();
let accept_loop = tokio::spawn(async move {
for reply in script {
let Ok((mut stream, _peer)) = listener.accept().await else {
return; };
if requests_tx
.send(read_request_head(&mut stream).await)
.is_err()
{
return; }
match reply {
HttpReply::Status(code) => {
let response = format!("HTTP/1.1 {code} OK\r\n\r\n");
let _ = stream.write_all(response.as_bytes()).await;
}
HttpReply::Raw(raw) => {
let _ = stream.write_all(raw.as_bytes()).await;
}
HttpReply::Hang => {
std::future::pending::<()>().await;
}
}
}
});
LoopbackHttp {
addr,
requests,
accept_loop,
}
}
async fn read_request_head(stream: &mut tokio::net::TcpStream) -> String {
let mut head = Vec::new();
let mut chunk = [0_u8; 256];
while !head.ends_with(b"\r\n\r\n") && head.len() < REQUEST_HEAD_CAP {
match stream.read(&mut chunk).await {
Ok(0) | Err(_) => break,
Ok(n) => head.extend_from_slice(&chunk[..n]),
}
}
String::from_utf8_lossy(&head).into_owned()
}