use shep_core::config::{AppConfig, LevelRule, LineLevel, ProbeConfig, ProbeKind, normalize};
use shep_core::protocol::DogSource;
use shep_core::status::ProcStatus;
use shep_core::values::{MemSize, UpDuration};
use super::*;
use crate::cron::{DEFAULT_MAX_CRON_SLEEP, SystemClock};
use crate::extras::{ExtrasReports, spawn_extras_reporter};
use crate::fake::{ProcScript, ScriptedRunner, idle_sweep};
use crate::testing::capture_logs;
use crate::testing::{
Harness, RecordingEnforcer, ScriptedProber, SharedRunner, app_with, armed_entry, harness,
idle_stats, plain_entry, probe_config, test_paths,
};
#[cfg(unix)]
use crate::tokio_runner::TokioRunner;
use std::sync::atomic::{AtomicUsize, Ordering as AtomicOrdering};
mod action_fixtures;
mod actions;
#[cfg(unix)]
mod adopt;
#[cfg(unix)]
mod adopt_fixtures;
#[cfg(unix)]
mod adopt_restart;
#[cfg(unix)]
mod adopt_swap;
mod config_fixtures;
mod config_load;
mod config_promote;
mod config_rearm;
mod config_report;
mod config_reset;
mod credentials;
mod disable_in_flight;
mod disabled_respawn;
mod dogs;
mod dogs_at_rest;
mod env_batch;
mod flush;
#[cfg(unix)]
mod handover;
mod interleaving;
mod lamb_sweep;
mod listen;
#[cfg(unix)]
mod listen_handover;
mod readiness;
mod reload_bus;
mod reload_drain;
mod reload_drainee;
mod reload_swap;
mod reopen;
mod reset;
mod restart_races;
mod sandbox_cause;
mod scale;
mod secrets;
mod shutdown;
mod signals;
mod start_stop;
mod triggers;
fn drained_process_kinds(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
) -> Vec<ProcessEventKind> {
let mut kinds = Vec::new();
while let Ok(BusEvent::Process { event, .. }) = rx.try_recv().map(|event| event.to_event()) {
kinds.push(event);
}
kinds
}
async fn await_event(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
) -> bool {
loop {
match rx.recv().await.map(|event| event.to_event()) {
Ok(BusEvent::Process {
event,
info,
manually,
..
}) if info.id == id && event == kind => {
return manually;
}
Ok(_) => continue,
Err(tokio::sync::broadcast::error::RecvError::Lagged(_)) => continue,
Err(e) => panic!("event stream closed before {kind:?} for id {id}: {e}"),
}
}
}
async fn assert_no_event_within(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
window: Duration,
) {
match tokio::time::timeout(window, await_event(rx, id, kind)).await {
Err(_elapsed) => {} Ok(_manually) => panic!("unexpected {kind:?} for id {id} within {window:?}"),
}
}
fn test_actor(
paths: ShepPaths,
scripts: Vec<ProcScript>,
sheep: HashMap<u32, SheepSlot>,
tx: mpsc::Sender<Msg>,
) -> Actor<ScriptedRunner> {
let (events, _events_rx) = crate::bus::test_bus(64);
let provider_secrets = Arc::new(ProviderSecrets::load(&paths.secrets_cache));
Actor {
runner: ScriptedRunner::new(scripts),
next_id: sheep.len() as u32,
paths,
events,
host_environment: DEFAULT_ENVIRONMENT.to_string(),
dog_binaries: BTreeMap::new(),
provider_secrets,
tx,
sheep,
next_deadline: 0,
next_action_stamp: 0,
pending: Vec::new(),
shutting_down: false,
extras: None,
registry: ExtrasRegistry::default(),
reloads: HashMap::new(),
smits: Smits::new(),
disabled: BTreeSet::new(),
listeners: BTreeMap::new(),
listeners_before: BTreeMap::new(),
cgroup_base: None,
cgroup_unavailable: None,
oom_baselines: HashMap::new(),
}
}
fn actor_with_stopping_drainee(
dir: &tempfile::TempDir,
pid: u32,
epoch: u64,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<SheepCtl>) {
let paths = test_paths(dir);
let app = normalize(AppConfig::minimal("web", "./srv")).unwrap();
let mut entry = armed_entry(0, 0, pid, app, &paths);
entry.status = ProcStatus::Stopping;
let (ctl_tx, ctl_rx) = mpsc::channel(1);
let slot = SheepSlot {
ctl: Some(ctl_tx),
epoch,
..SheepSlot::new(entry)
};
let mut sheep = HashMap::new();
sheep.insert(0, slot);
let (tx, _rx) = mpsc::channel(16);
let actor = test_actor(paths, Vec::new(), sheep, tx);
(actor, ctl_rx)
}
fn actor_with_a_drainee_holding_a_signal_mailbox(
dir: &tempfile::TempDir,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<SignalRequest>) {
let (mut actor, _mailbox) = actor_with_one_online_sheep(dir, vec![]);
let slot = actor.sheep.get_mut(&0).expect("the fixture registers id 0");
slot.entry.status = ProcStatus::Stopping;
slot.entry.reload = ReloadState::Drainee { new_id: Some(1) };
let (signals, signal_rx) = mpsc::channel(16);
slot.signals = Some(signals);
(actor, signal_rx)
}
const SWAP_WINDOW: Duration = Duration::from_secs(30);
async fn expect_event(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
) {
assert!(
tokio::time::timeout(SWAP_WINDOW, await_event(rx, id, kind))
.await
.is_ok(),
"no {kind:?} for id {id} within {SWAP_WINDOW:?}"
);
}
type Started = (
SupervisorHandle,
Arc<ScriptedRunner>,
tokio::sync::broadcast::Receiver<SharedEvent>,
);
async fn started_apps(
dir: &tempfile::TempDir,
apps: Vec<AppConfig>,
scripts: Vec<ProcScript>,
bus_capacity: usize,
) -> Started {
let (events, rx) = crate::bus::test_bus(bus_capacity);
let runner = Arc::new(ScriptedRunner::new(scripts));
let handle = spawn_supervisor(SharedRunner(Arc::clone(&runner)), test_paths(dir), events);
let apps = apps
.into_iter()
.map(|app| normalize(app).unwrap())
.collect::<Vec<_>>();
handle.start(apps).await.unwrap();
(handle, runner, rx)
}
async fn started(dir: &tempfile::TempDir, app: AppConfig, scripts: Vec<ProcScript>) -> Started {
started_apps(dir, vec![app], scripts, 256).await
}
fn actor_with_one_online_sheep(
dir: &tempfile::TempDir,
scripts: Vec<ProcScript>,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
actor_with_one_online_sheep_of(dir, AppConfig::minimal("web", "./srv"), scripts)
}
fn actor_with_one_online_sheep_of(
dir: &tempfile::TempDir,
app: AppConfig,
scripts: Vec<ProcScript>,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
let paths = test_paths(dir);
let app = normalize(app).unwrap();
let mut sheep = HashMap::new();
sheep.insert(0, SheepSlot::new(armed_entry(0, 0, 1111, app, &paths)));
let (tx, rx) = mpsc::channel(MAILBOX_CAPACITY);
let actor = test_actor(paths, scripts, sheep, tx);
(actor, rx)
}
fn attach_ctl(actor: &mut Actor<ScriptedRunner>) -> mpsc::Receiver<SheepCtl> {
let (ctl_tx, ctl_rx) = mpsc::channel(SHEEP_CTL_CAPACITY);
actor.sheep.get_mut(&0).expect("the fixture's sheep").ctl = Some(ctl_tx);
ctl_rx
}
const SHEEP_ID: u32 = 0;
const DOG_ID: u32 = 1;
fn actor_with_a_sheep_and_a_dog(
dir: &tempfile::TempDir,
) -> (Actor<ScriptedRunner>, mpsc::Receiver<Msg>) {
let paths = test_paths(dir);
let mut sheep = HashMap::new();
for (id, name, dog) in [
(SHEEP_ID, "web", None),
(
DOG_ID,
"bark",
Some(DogSource::BuiltIn {
grants: None,
events: Vec::new(),
}),
),
] {
let app = app_with(name, |config| config.fold = Some("svc".to_string()));
let mut entry = armed_entry(id, 0, 1111 + id, app, &paths);
entry.dog = dog;
sheep.insert(id, SheepSlot::new(entry));
}
let (tx, rx) = mpsc::channel(MAILBOX_CAPACITY);
let actor = test_actor(paths, Vec::new(), sheep, tx);
(actor, rx)
}
#[test]
fn a_wildcard_passes_a_dog_by_and_its_own_name_still_reaches_it() {
let dir = tempfile::tempdir().unwrap();
let (actor, _mailbox) = actor_with_a_sheep_and_a_dog(&dir);
assert_eq!(
actor.matching_ids(&ProcessSelector::All),
vec![SHEEP_ID],
"`all` is the flock, not the kennel"
);
assert_eq!(
actor.matching_ids(&ProcessSelector::parse("/^(web|bark)$/").unwrap()),
vec![SHEEP_ID],
"a sweep that spells both names out is still a sweep"
);
assert_eq!(
actor.matching_ids(&ProcessSelector::Fold("svc".into())),
vec![SHEEP_ID],
"a dog shares its fold with the flock and is still not swept by it"
);
assert_eq!(
actor.matching_ids(&ProcessSelector::Name("bark".into())),
vec![DOG_ID]
);
assert_eq!(
actor.matching_ids(&ProcessSelector::Id(DOG_ID)),
vec![DOG_ID]
);
}
#[test]
fn a_listing_reports_where_a_dog_came_from() {
let dir = tempfile::tempdir().unwrap();
let (actor, _mailbox) = actor_with_a_sheep_and_a_dog(&dir);
assert_eq!(
to_info(&actor.sheep[&DOG_ID].entry, &actor.smits).dog,
Some(DogSource::BuiltIn {
grants: None,
events: Vec::new()
})
);
assert_eq!(
to_info(&actor.sheep[&SHEEP_ID].entry, &actor.smits).dog,
None
);
}
async fn start_app(h: &Harness, app: AppConfig) -> Vec<ProcessInfo> {
h.ctx
.supervisor
.start(vec![normalize(app).unwrap()])
.await
.unwrap()
}
fn dog_app(name: &str) -> ResolvedApp {
normalize(AppConfig::minimal(name, "/nonexistent/shep")).unwrap()
}
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
struct Seen {
id: u32,
kind: ProcessEventKind,
status: ProcStatus,
manually: bool,
}
async fn events_through(
rx: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
id: u32,
kind: ProcessEventKind,
) -> Vec<Seen> {
let collect = async {
let mut seen = Vec::new();
loop {
match rx.recv().await.map(|event| event.to_event()) {
Ok(BusEvent::Process {
event,
info,
manually,
..
}) => {
seen.push(Seen {
id: info.id,
kind: event,
status: info.status,
manually,
});
if info.id == id && event == kind {
return seen;
}
}
Ok(_) => continue,
Err(tokio::sync::broadcast::error::RecvError::Lagged(n)) => {
panic!("the event stream lagged by {n}; no ordering claim survives that")
}
Err(e) => panic!("event stream closed before {kind:?} for id {id}: {e}"),
}
}
};
tokio::time::timeout(SWAP_WINDOW, collect)
.await
.unwrap_or_else(|_| panic!("no {kind:?} for id {id} within {SWAP_WINDOW:?}"))
}
fn at(seen: &[Seen], id: u32, kind: ProcessEventKind) -> usize {
seen.iter()
.position(|e| e.id == id && e.kind == kind)
.unwrap_or_else(|| panic!("no {kind:?} for id {id} in {seen:?}"))
}
fn actor_with_an_empty_flock(
dir: &tempfile::TempDir,
scripts: Vec<ProcScript>,
) -> Actor<ScriptedRunner> {
let (tx, _rx) = mpsc::channel(MAILBOX_CAPACITY);
let paths = test_paths(dir);
test_actor(paths, scripts, HashMap::new(), tx)
}
#[cfg(unix)]
fn own_user_name() -> String {
nix::unistd::User::from_uid(nix::unistd::geteuid())
.unwrap()
.expect("this process has a passwd entry")
.name
}
const NO_SUCH_USER: &str = "definitely-not-a-real-shep-user";