use super::action_fixtures::{
ACTION_WINDOW, actor_with_an_open_channel, sent_action, settle_action, trigger_action,
};
use super::*;
async fn next_channel_event(
events: &mut tokio::sync::broadcast::Receiver<SharedEvent>,
) -> BusEvent {
tokio::time::timeout(ACTION_WINDOW, async {
loop {
let event = events.recv().await.unwrap().to_event();
if matches!(event, BusEvent::Channel { .. }) {
break event;
}
}
})
.await
.expect("no channel event within the window")
}
#[tokio::test(start_paused = true)]
async fn a_triggered_action_answers_with_the_apps_reply() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let mut io = runner.io_handles(0);
let triggered = tokio::spawn(async move {
handle
.trigger(
ProcessSelector::Name("web".to_string()),
"gc".to_string(),
Some("--full".to_string()),
None,
)
.await
});
assert_eq!(
sent_action(&mut io.to_child_rx).await,
ShepherdMessage::Action {
name: "gc".to_string(),
params: Some("--full".to_string()),
caller: None,
id: 0,
},
"the action reaches the child's end of the channel as it was asked for"
);
io.from_child_tx
.send(ChildMessage::ActionReply {
action: "gc".to_string(),
body: "swept 3".to_string(),
id: Some(0),
})
.await
.unwrap();
assert_eq!(
triggered.await.unwrap(),
Ok(vec![ActionReply {
id: 0,
name: "web".to_string(),
outcome: ActionOutcome::Replied {
body: "swept 3".to_string()
},
}]),
"the app's reply body is what the caller is answered with"
);
}
#[tokio::test(start_paused = true)]
async fn a_ready_on_the_channel_reaches_both_the_bus_and_the_readiness_wait() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
app.wait_ready = true;
let (handle, runner, mut events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
io.from_child_tx.send(ChildMessage::Ready).await.unwrap();
let BusEvent::Channel { id, message } = next_channel_event(&mut events).await else {
unreachable!("`next_channel_event` returns only channel events");
};
assert_eq!((id, message), (0, ChildMessage::Ready));
let listed = handle.list().await;
assert_eq!(listed[0].status, ProcStatus::Online);
}
#[tokio::test(start_paused = true)]
async fn a_metric_on_the_channel_reaches_the_bus_with_its_name_and_value() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (_handle, runner, mut events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
io.from_child_tx
.send(ChildMessage::Metric {
name: "rps".to_string(),
value: 42.0,
metric_type: None,
labels: std::collections::BTreeMap::new(),
help: None,
})
.await
.unwrap();
let BusEvent::Channel { id, message } = next_channel_event(&mut events).await else {
unreachable!("`next_channel_event` returns only channel events");
};
assert_eq!(
(id, message),
(
0,
ChildMessage::Metric {
name: "rps".to_string(),
value: 42.0,
metric_type: None,
labels: std::collections::BTreeMap::new(),
help: None,
}
)
);
}
#[tokio::test(start_paused = true)]
async fn a_note_on_the_channel_reaches_the_bus_as_it_was_sent() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (_handle, runner, mut events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
let note = ChildMessage::Note {
name: "lease.web.want.1760500000".to_string(),
value: 1.0,
};
io.from_child_tx.send(note.clone()).await.unwrap();
let event = next_channel_event(&mut events).await;
let BusEvent::Channel { id, message } = &event else {
unreachable!("`next_channel_event` returns only channel events");
};
assert_eq!(
(*id, message.clone(), event.topic().into_owned()),
(0, note, "channel.note".to_string())
);
}
#[tokio::test(start_paused = true)]
async fn an_action_reply_no_trigger_is_waiting_for_still_reaches_the_bus() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let (_handle, runner, mut events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let io = runner.io_handles(0);
io.from_child_tx
.send(ChildMessage::ActionReply {
action: "gc".to_string(),
body: "unprompted".to_string(),
id: None,
})
.await
.unwrap();
let BusEvent::Channel { message: seen, .. } = next_channel_event(&mut events).await else {
unreachable!("`next_channel_event` returns only channel events");
};
let ChildMessage::ActionReply { body, .. } = seen else {
panic!("expected an action reply, got {seen:?}");
};
assert_eq!(body, "unprompted");
}
#[tokio::test(start_paused = true)]
async fn a_triggered_action_times_out_when_the_app_never_answers() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("api", "./api");
app.channel = true;
let (handle, runner, _events) = started(&dir, app, vec![ProcScript::never_exits()]).await;
let mut io = runner.io_handles(0);
assert_eq!(
handle
.trigger(
ProcessSelector::Name("api".to_string()),
"stats".to_string(),
None,
None,
)
.await,
Ok(vec![ActionReply {
id: 0,
name: "api".to_string(),
outcome: ActionOutcome::TimedOut,
}]),
"an app that says nothing is reported as saying nothing, not waited on forever"
);
assert_eq!(
sent_action(&mut io.to_child_rx).await,
ShepherdMessage::Action {
name: "stats".to_string(),
params: None,
caller: None,
id: 0,
},
"the timeout is an app that did not answer, not an action that was never sent"
);
}
#[test]
fn a_stamped_reply_wakes_its_own_wait_even_with_a_debt_outstanding() {
let mut waits = ActionWaits::default();
let (t1_reply, _t1_out) = oneshot::channel();
let (t1_waiter, _t1_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 1,
action: "gc".to_string(),
waiter: Some(t1_waiter),
reply: t1_reply,
});
assert!(waits.resolve(1).is_some(), "T1 must have been live");
let (t2_reply, _t2_out) = oneshot::channel();
let (t2_waiter, t2_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 2,
action: "gc".to_string(),
waiter: Some(t2_waiter),
reply: t2_reply,
});
let woken = waits
.answer("gc", Some(2))
.expect("a reply stamped with the live wait's own stamp must reach it");
woken.send("collected".to_string()).unwrap();
assert_eq!(t2_body.blocking_recv().unwrap(), "collected");
}
#[test]
fn an_unstamped_reply_still_settles_the_oldest_debt_first() {
let mut waits = ActionWaits::default();
let (t1_reply, _t1_out) = oneshot::channel();
let (t1_waiter, _t1_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 1,
action: "gc".to_string(),
waiter: Some(t1_waiter),
reply: t1_reply,
});
waits.resolve(1);
let (t2_reply, _t2_out) = oneshot::channel();
let (t2_waiter, _t2_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 2,
action: "gc".to_string(),
waiter: Some(t2_waiter),
reply: t2_reply,
});
assert!(
waits.answer("gc", None).is_none(),
"an unstamped reply pays the debt, exactly as it did before stamping"
);
assert!(
waits.answer("gc", None).is_some(),
"and the next one reaches the live wait, exactly as it did before"
);
}
#[test]
fn a_stamped_reply_for_a_dead_wait_does_not_reach_a_live_one() {
let mut waits = ActionWaits::default();
let (t1_reply, _t1_out) = oneshot::channel();
let (t1_waiter, _t1_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 1,
action: "gc".to_string(),
waiter: Some(t1_waiter),
reply: t1_reply,
});
waits.resolve(1);
let (t2_reply, _t2_out) = oneshot::channel();
let (t2_waiter, _t2_body) = oneshot::channel();
waits.arm(PendingAction {
stamp: 2,
action: "gc".to_string(),
waiter: Some(t2_waiter),
reply: t2_reply,
});
assert!(
waits.answer("gc", Some(1)).is_none(),
"T1's own late reply belongs to T1's debt, not to T2"
);
assert!(
waits.answer("gc", Some(2)).is_some(),
"and T2 is still waiting for its own"
);
}
#[tokio::test(start_paused = true)]
async fn a_reply_owed_by_a_timed_out_action_never_answers_the_next_one() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let first = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut
);
assert_eq!(first.await.unwrap(), ActionOutcome::TimedOut);
let second = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut,
"a reply the first `gc` was owed was handed to the second one"
);
assert_eq!(second.await.unwrap(), ActionOutcome::TimedOut);
let third = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 7".to_string(), None);
actor.handle_action_reply(0, "gc", "swept 11".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::Replied {
body: "swept 11".to_string()
},
"the debts outlived the replies that settled them"
);
assert_eq!(
third.await.unwrap(),
ActionOutcome::Replied {
body: "swept 11".to_string()
}
);
}
#[tokio::test(start_paused = true)]
async fn a_second_reply_to_an_answered_action_is_dropped() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let answered = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::Replied {
body: "swept 3".to_string()
}
);
assert_eq!(
answered.await.unwrap(),
ActionOutcome::Replied {
body: "swept 3".to_string()
}
);
actor.handle_action_reply(0, "gc", "spent".to_string(), None);
let next = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut,
"a spare reply was kept and used to answer a wait that came after it"
);
assert_eq!(next.await.unwrap(), ActionOutcome::TimedOut);
}
#[tokio::test(start_paused = true)]
async fn two_waits_for_one_action_are_answered_in_the_order_they_were_asked() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox, mut child_rx) = actor_with_an_open_channel(&dir);
let first = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
let second = trigger_action(&mut actor, "gc");
sent_action(&mut child_rx).await;
actor.handle_action_reply(0, "gc", "swept 3".to_string(), None);
actor.handle_action_reply(0, "gc", "swept 7".to_string(), None);
settle_action(&mut actor, &mut mailbox).await;
settle_action(&mut actor, &mut mailbox).await;
assert_eq!(
first.await.unwrap(),
ActionOutcome::Replied {
body: "swept 3".to_string()
},
"the earlier trigger was answered with the later reply"
);
assert_eq!(
second.await.unwrap(),
ActionOutcome::Replied {
body: "swept 7".to_string()
},
"the second reply was dropped and its wait left waiting"
);
}
#[tokio::test(start_paused = true)]
async fn an_action_that_cannot_even_be_delivered_still_ends() {
let dir = tempfile::tempdir().unwrap();
let (mut actor, mut mailbox) = actor_with_one_online_sheep(&dir, vec![]);
let (to_child, _wedged) = mpsc::channel(1);
to_child.try_send(ShepherdMessage::Shutdown).unwrap();
actor
.sheep
.get_mut(&0)
.expect("the fixture registers id 0")
.to_child = Some(to_child);
let answer = trigger_action(&mut actor, "gc");
assert_eq!(
settle_action(&mut actor, &mut mailbox).await,
ActionOutcome::TimedOut,
"an action that never got onto the channel left its wait parked"
);
assert_eq!(answer.await.unwrap(), ActionOutcome::TimedOut);
}
#[tokio::test(start_paused = true)]
async fn a_lamb_label_on_the_channel_reaches_the_describe_walk() {
let dir = tempfile::tempdir().unwrap();
let mut app = AppConfig::minimal("web", "./srv");
app.channel = true;
let root = crate::fake::FIRST_SCRIPTED_PID;
let sampler = crate::testing::ScriptedSampler::identifying(vec![vec![
crate::testing::identity(root, None, "srv"),
crate::testing::identity(root + 1, Some(root), "python"),
crate::testing::identity(root + 2, Some(root), "python"),
]]);
let stats = Arc::new(crate::limits::stats::StatsState::new(Arc::new(sampler)));
let (breaches, _breaches_rx) = mpsc::channel(1);
let (liveness, _liveness_rx) = mpsc::channel(1);
let (events, _events_rx) = crate::bus::test_bus(256);
let runner = Arc::new(ScriptedRunner::new(vec![ProcScript::never_exits()]));
let handle =
SupervisorBuilder::new(SharedRunner(Arc::clone(&runner)), test_paths(&dir), events)
.extras(Extras {
clock: Arc::new(SystemClock),
enforcer: Arc::new(RecordingEnforcer::default()),
lamb_sweep: idle_sweep(),
max_cron_sleep: DEFAULT_MAX_CRON_SLEEP,
reports: ExtrasReports { breaches, liveness },
stats: Arc::clone(&stats),
})
.spawn();
handle.start(vec![normalize(app).unwrap()]).await.unwrap();
let io = runner.io_handles(0);
io.from_child_tx
.send(ChildMessage::LambLabel {
pid: root + 2,
label: shep_core::protocol::LambLabel::new("worker 2").unwrap(),
})
.await
.unwrap();
let lambs = tokio::time::timeout(ACTION_WINDOW, async {
loop {
let lambs = stats.lambs_of(&stats.lamb_index(), root);
if lambs.iter().any(|lamb| lamb.label.is_some()) {
break lambs;
}
tokio::time::sleep(Duration::from_millis(10)).await;
}
})
.await
.expect("the label never reached the walk");
assert_eq!(
lambs,
vec![
shep_core::protocol::Lamb::new(root + 1, "python"),
shep_core::protocol::Lamb::new(root + 2, "python").with_label("worker 2"),
]
);
}