mod doubles;
use std::path::{Path, PathBuf};
use std::sync::{Arc, LazyLock, Mutex, PoisonError};
use shep_core::config::{AppConfig, ResolvedApp, normalize};
use shep_core::paths::ShepPaths;
use shep_core::secrets::SecretView;
use shep_core::status::ProcStatus;
use tokio::sync::{broadcast, mpsc, watch};
pub(crate) use doubles::{
ArmCall, RecordingEnforcer, RecordingPidSignals, ScriptedProber, ScriptedSampler, TestClock,
probe_config, rss, rss_cpu,
};
use crate::assemble::assemble;
use crate::bus::SharedEvent;
use crate::cron::DEFAULT_MAX_CRON_SLEEP;
use crate::entry::{ProcessEntry, ReloadState, RestartBudget};
use crate::extras::{Extras, ExtrasReports, LivenessReport};
use crate::fake::{FIRST_SCRIPTED_PID, ProcScript, ScriptedRunner, idle_sweep};
use crate::limits::LimitBreach;
use crate::limits::sample::{MemorySampler, ProcessIdentity, ProcessRss};
use crate::limits::stats::StatsState;
use crate::privilege::SpawnIdentity;
use crate::rpc::RpcContext;
use crate::runner::{Preflight, ProcIo, ProcessRunner, RunnerError, SpawnSpec};
use crate::snapshot::{FlockRegistry, FlockSnapshot, SavedApp};
use crate::supervisor::{SupervisorBuilder, SupervisorHandle};
#[derive(Debug, Clone, Default)]
pub(crate) struct LogCapture(Arc<Mutex<Vec<u8>>>);
impl LogCapture {
fn rendered(&self) -> String {
let buffer = self.0.lock().unwrap_or_else(PoisonError::into_inner);
String::from_utf8_lossy(&buffer).into_owned()
}
}
impl std::io::Write for LogCapture {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0
.lock()
.unwrap_or_else(PoisonError::into_inner)
.extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl<'a> tracing_subscriber::fmt::MakeWriter<'a> for LogCapture {
type Writer = Self;
fn make_writer(&'a self) -> Self::Writer {
self.clone()
}
}
static SECOND_DISPATCH: LazyLock<tracing::Dispatch> =
LazyLock::new(|| tracing::Dispatch::new(tracing::subscriber::NoSubscriber::default()));
pub(crate) fn capture_logs(f: impl FnOnce()) -> String {
LazyLock::force(&SECOND_DISPATCH);
let capture = LogCapture::default();
let subscriber = tracing_subscriber::fmt()
.with_writer(capture.clone())
.with_ansi(false)
.with_max_level(tracing::Level::TRACE)
.finish();
tracing::subscriber::with_default(subscriber, f);
capture.rendered()
}
fn racing_warn() {
tracing::warn!("a callsite two threads reach");
}
#[test]
fn a_sibling_thread_reaching_a_callsite_first_cannot_empty_the_capture() {
let (scope_open, await_scope) = std::sync::mpsc::channel();
let (registered, await_registration) = std::sync::mpsc::channel();
let sibling = std::thread::spawn(move || {
await_scope.recv().expect("the capture must open its scope");
racing_warn();
registered
.send(())
.expect("the capture must still be waiting");
});
let rendered = capture_logs(|| {
scope_open
.send(())
.expect("the sibling must still be waiting");
await_registration
.recv()
.expect("the sibling must register before this emit");
racing_warn();
});
sibling.join().expect("the sibling thread must not panic");
assert!(
rendered.contains("a callsite two threads reach"),
"a sibling thread registering first must not disable the callsite for \
this capture: {rendered:?}"
);
}
pub(crate) async fn sorted_status(handle: &SupervisorHandle) -> Vec<(String, ProcStatus)> {
let mut listed = handle.list().await;
listed.sort_by(|left, right| left.name.cmp(&right.name));
listed
.into_iter()
.map(|info| (info.name, info.status))
.collect()
}
pub(crate) fn roll_of(apps: Vec<AppConfig>) -> FlockSnapshot {
let saved = apps.into_iter().map(|app| SavedApp {
app,
instances_running: 1,
disabled: false,
});
FlockSnapshot::with_apps(saved.collect())
}
pub(crate) fn test_paths(dir: &tempfile::TempDir) -> ShepPaths {
let home = dir.path().to_path_buf();
ShepPaths::resolve(
&|key| (key == "SHEP_HOME").then(|| home.display().to_string()),
std::path::Path::new("/nonexistent"),
)
}
#[derive(Debug)]
pub(crate) struct SharedRunner(pub(crate) Arc<ScriptedRunner>);
impl ProcessRunner for SharedRunner {
type Proc = crate::fake::FakeProc;
fn spawn(&self, spec: &SpawnSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
self.0.spawn(spec)
}
fn preflight(&self, spec: &SpawnSpec) -> Preflight {
self.0.preflight(spec)
}
}
#[cfg(unix)]
#[derive(Debug)]
pub(crate) struct AnnouncingRunner<R> {
inner: R,
target: PathBuf,
}
#[cfg(unix)]
impl<R> AnnouncingRunner<R> {
pub(crate) fn new(inner: R, target: &Path) -> Self {
Self {
inner,
target: target.to_path_buf(),
}
}
}
#[cfg(unix)]
impl<R: ProcessRunner> ProcessRunner for AnnouncingRunner<R> {
type Proc = R::Proc;
#[track_caller]
fn spawn(&self, spec: &SpawnSpec) -> Result<(Self::Proc, ProcIo), RunnerError> {
let socket = std::os::unix::net::UnixDatagram::unbound()
.expect("a test must be able to open an unbound datagram socket");
socket
.send_to(b"SPAWNED\n", &self.target)
.expect("the test's listening socket must be bound before the spawn");
self.inner.spawn(spec)
}
}
pub(crate) fn proptest_config(local_cases: u32) -> proptest::test_runner::Config {
let default = proptest::test_runner::Config::default();
if std::env::var_os("PROPTEST_CASES").is_some() {
default
} else {
proptest::test_runner::Config {
cases: local_cases,
..default
}
}
}
pub(crate) fn touch(root: &Path, rel: &str) -> std::io::Result<PathBuf> {
let path = root.join(rel);
if let Some(parent) = path.parent() {
std::fs::create_dir_all(parent)?;
}
std::fs::write(&path, b"x")?;
Ok(path)
}
pub(crate) struct Harness {
pub(crate) ctx: RpcContext,
_dir: tempfile::TempDir,
_events_rx: broadcast::Receiver<SharedEvent>,
pub(crate) shutdown_rx: watch::Receiver<bool>,
pub(crate) breaches: mpsc::Receiver<LimitBreach>,
pub(crate) liveness: mpsc::Receiver<LivenessReport>,
pub(crate) stats: Arc<StatsState>,
pub(crate) runner: Arc<ScriptedRunner>,
pub(crate) pid_signals: Arc<RecordingPidSignals>,
}
const HARNESS_REPORT_CAPACITY: usize = 16;
pub(crate) fn harness(scripts: Vec<ProcScript>) -> Harness {
harness_sampling(scripts, vec![vec![]])
}
pub(crate) fn harness_with_stats(scripts: Vec<ProcScript>) -> Harness {
let tree = |cpu_ms| {
vec![ProcessRss {
pid: FIRST_SCRIPTED_PID,
parent: None,
bytes: SCRIPTED_TREE_BYTES,
cpu_ms,
}]
};
harness_sampling(scripts, vec![tree(1_000), tree(2_500), tree(2_501)])
}
pub(crate) const SCRIPTED_TREE_BYTES: u64 = 4096;
pub(crate) fn harness_at_pid(scripts: Vec<ProcScript>, pid: u32) -> Harness {
harness_sampling_with(ScriptedRunner::new(scripts).spawning_at(pid), vec![vec![]])
}
pub(crate) fn harness_refusing(scripts: Vec<ProcScript>, names: &[&str]) -> Harness {
harness_sampling_with(ScriptedRunner::new(scripts).refusing(names), vec![vec![]])
}
pub(crate) fn harness_failing_to_spawn(scripts: Vec<ProcScript>, names: &[&str]) -> Harness {
harness_sampling_with(
ScriptedRunner::new(scripts).failing_to_spawn(names),
vec![vec![]],
)
}
fn harness_sampling(scripts: Vec<ProcScript>, readings: Vec<Vec<ProcessRss>>) -> Harness {
harness_sampling_with(ScriptedRunner::new(scripts), readings)
}
fn harness_sampling_with(runner: ScriptedRunner, readings: Vec<Vec<ProcessRss>>) -> Harness {
harness_with_runner(runner, |reports| {
standard_extras(Arc::new(ScriptedSampler::new(readings)), reports)
})
}
fn standard_extras(sampler: Arc<dyn MemorySampler>, reports: ExtrasReports) -> Extras {
let stats = Arc::new(StatsState::new(Arc::clone(&sampler)));
Extras {
clock: Arc::new(TestClock::starting_at(
"2026-01-01T00:00:00Z"
.parse()
.expect("a valid RFC3339 timestamp"),
)),
enforcer: Arc::new(crate::limits::PollingEnforcer::start(
sampler,
reports.breaches.clone(),
Arc::clone(&stats),
)),
lamb_sweep: idle_sweep(),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports,
stats,
}
}
pub(crate) fn harness_identifying(
scripts: Vec<ProcScript>,
identities: Vec<ProcessIdentity>,
) -> Harness {
harness_with_extras(scripts, |reports| {
standard_extras(
Arc::new(ScriptedSampler::identifying(vec![identities])),
reports,
)
})
}
pub(crate) fn identity(pid: u32, parent: Option<u32>, name: &str) -> ProcessIdentity {
ProcessIdentity {
pid,
parent,
name: name.to_string(),
}
}
pub(crate) fn idle_stats() -> Arc<StatsState> {
Arc::new(StatsState::new(Arc::new(ScriptedSampler::new(vec![
vec![],
]))))
}
pub(crate) fn harness_with_extras(
scripts: Vec<ProcScript>,
build_extras: impl FnOnce(ExtrasReports) -> Extras,
) -> Harness {
harness_with_runner(ScriptedRunner::new(scripts), build_extras)
}
pub(crate) fn harness_with_runner(
runner: ScriptedRunner,
build_extras: impl FnOnce(ExtrasReports) -> Extras,
) -> Harness {
build_harness(runner, build_extras, crate::supervisor::DEFAULT_ENVIRONMENT)
}
pub(crate) fn harness_in_environment(scripts: Vec<ProcScript>, environment: &str) -> Harness {
build_harness(
ScriptedRunner::new(scripts),
|reports| standard_extras(Arc::new(ScriptedSampler::new(vec![vec![]])), reports),
environment,
)
}
fn build_harness(
runner: ScriptedRunner,
build_extras: impl FnOnce(ExtrasReports) -> Extras,
environment: &str,
) -> Harness {
let dir = tempfile::tempdir().unwrap();
let paths = test_paths(&dir);
let (events, events_rx) = crate::bus::test_bus(256);
let (breach_tx, breaches) = mpsc::channel(HARNESS_REPORT_CAPACITY);
let (live_tx, liveness) = mpsc::channel(HARNESS_REPORT_CAPACITY);
let extras = build_extras(ExtrasReports {
breaches: breach_tx,
liveness: live_tx,
});
let stats = Arc::clone(&extras.stats);
let provider_secrets = Arc::new(crate::secrets::ProviderSecrets::load(&paths.secrets_cache));
let runner = Arc::new(runner);
let supervisor = SupervisorBuilder::new(
SharedRunner(Arc::clone(&runner)),
paths.clone(),
events.clone(),
)
.extras(extras)
.provider_secrets(Arc::clone(&provider_secrets))
.environment(environment.to_string())
.spawn();
let (shutdown, shutdown_rx) = watch::channel(false);
let pid_signals = Arc::new(RecordingPidSignals::default());
Harness {
ctx: RpcContext {
supervisor,
events,
registry: FlockRegistry::new(),
snapshot_path: paths.snapshot.clone(),
dogs_config: paths.dogs_config.clone(),
known_dogs: crate::rpc::KnownDogs::new(
["metrics".to_string(), "bark".to_string()]
.into_iter()
.collect(),
),
dog_names: Vec::new(),
boot_first_dogs: Vec::new(),
paths: paths.clone(),
daemon_version: "0.1.0".to_string(),
dog_refusals: crate::dogs::DogRefusals::new(),
peer_contacts: crate::dogs::PeerContacts::new(),
pid: 4242,
shutdown,
stats: Arc::clone(&stats),
host: crate::host::HostState::fixed(None),
provider_secrets,
host_environment: environment.to_string(),
pid_signals: Arc::clone(&pid_signals) as Arc<dyn crate::rpc::PidSignals>,
},
_dir: dir,
_events_rx: events_rx,
shutdown_rx,
breaches,
liveness,
stats,
runner,
pid_signals,
}
}
#[track_caller]
pub(crate) fn app_with(name: &str, mutate: impl FnOnce(&mut AppConfig)) -> ResolvedApp {
let mut app = AppConfig::minimal(name, "./srv");
mutate(&mut app);
normalize(app).expect("the fixture app must normalize")
}
pub(crate) fn plain_entry(id: u32, spec: ResolvedApp) -> ProcessEntry {
ProcessEntry {
id,
spec,
pending: None,
pending_reidentifies: false,
overridden: Vec::new(),
instance: 0,
status: ProcStatus::Online,
pid: None,
restarts: 0,
started_at: None,
budget: RestartBudget::default(),
reload: ReloadState::None,
credentials: SpawnIdentity::Resolved(None),
out_file: PathBuf::new(),
err_file: PathBuf::new(),
dog: None,
last_exit: None,
spawn_failure: None,
}
}
pub(crate) fn armed_entry(
id: u32,
instance: u32,
pid: u32,
app: ResolvedApp,
paths: &ShepPaths,
) -> ProcessEntry {
let spec = assemble(
&app,
instance,
paths,
None,
&SecretView::empty("production".to_string()),
)
.expect("the fixture app must assemble");
ProcessEntry {
instance,
pid: Some(pid),
out_file: spec.out_file,
err_file: spec.err_file,
..plain_entry(id, app)
}
}