use core::fmt;
use core::time::Duration;
use bytes::Bytes;
use futures_util::{SinkExt, StreamExt};
use serde::Serialize;
use tokio::sync::{mpsc, watch};
use tokio::task::JoinHandle;
use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec};
use shep_core::protocol::{
Envelope, Hello, HelloAck, HelloReply, MIN_SUPPORTED, PROTOCOL_VERSION, RpcError, RpcErrorCode,
WireError, codec, decode_frame, encode_frame,
};
use shep_core::transport::{Listener, ServerReadHalf, ServerStream, ServerWriteHalf};
use crate::bus::spawn_forwarder;
use crate::rpc::{Outcome, RpcContext, dispatch};
use crate::supervisor::ConnId;
pub const CONN_QUEUE: usize = 64;
pub const HANDSHAKE_TIMEOUT_MS: u64 = 5_000;
type Frames = FramedRead<ServerReadHalf, LengthDelimitedCodec>;
#[derive(Debug)]
pub struct RpcServer {
listener: Listener,
ctx: RpcContext,
}
impl RpcServer {
#[must_use]
pub fn new(listener: Listener, ctx: RpcContext) -> Self {
Self { listener, ctx }
}
pub async fn serve(self, mut shutdown: watch::Receiver<bool>) {
let Self { mut listener, ctx } = self;
if *shutdown.borrow() {
return;
}
loop {
tokio::select! {
accepted = listener.accept() => {
match accepted {
Ok(stream) => {
let ctx = ctx.clone();
tokio::spawn(async move {
if let Err(err) = handle_conn(stream, ctx).await {
tracing::debug!(%err, "connection ended");
}
});
}
Err(err) => tracing::warn!(%err, "accept failed; continuing"),
}
}
changed = shutdown.changed() => {
if changed.is_err() || *shutdown.borrow() {
break;
}
}
}
}
}
}
#[cfg(unix)]
pub fn check_peer(stream: &tokio::net::UnixStream, daemon_uid: u32) -> Result<u32, AuthError> {
let cred = stream
.peer_cred()
.map_err(|err| AuthError::NoCredentials(err.to_string()))?;
let peer = cred.uid();
if peer == daemon_uid {
Ok(peer)
} else {
Err(AuthError::ForeignUid {
peer,
daemon: daemon_uid,
})
}
}
#[cfg(unix)]
#[must_use]
pub fn peer_pid(stream: &tokio::net::UnixStream) -> Option<u32> {
u32::try_from(stream.peer_cred().ok()?.pid()?).ok()
}
#[cfg(unix)]
#[must_use]
pub fn daemon_uid() -> u32 {
nix::unistd::geteuid().as_raw()
}
#[non_exhaustive]
#[cfg_attr(windows, allow(dead_code))]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AuthError {
NoCredentials(String),
ForeignUid {
peer: u32,
daemon: u32,
},
}
impl fmt::Display for AuthError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NoCredentials(msg) => write!(f, "could not read peer credentials: {msg}"),
Self::ForeignUid { peer, daemon } => {
write!(f, "peer uid {peer} does not match daemon uid {daemon}")
}
}
}
}
impl core::error::Error for AuthError {}
#[non_exhaustive]
#[derive(Debug)]
pub enum ConnError {
Auth(AuthError),
Frame(std::io::Error),
Decode(WireError),
Encode(WireError),
ProtocolMismatch {
client: u32,
},
HandshakeTimeout,
NoHandshake,
PeerGone,
}
impl fmt::Display for ConnError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Auth(err) => write!(f, "peer-credential check failed: {err}"),
Self::Frame(err) => write!(f, "frame transport error: {err}"),
Self::Decode(err) => write!(f, "frame decode error: {err}"),
Self::Encode(err) => write!(f, "frame encode error: {err}"),
Self::ProtocolMismatch { client } => write!(
f,
"client sent protocol {client}, daemon speaks {PROTOCOL_VERSION}"
),
Self::HandshakeTimeout => write!(
f,
"peer did not send Hello within {HANDSHAKE_TIMEOUT_MS} ms"
),
Self::NoHandshake => f.write_str("peer closed before sending Hello"),
Self::PeerGone => f.write_str("connection's write queue is gone"),
}
}
}
impl core::error::Error for ConnError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Auth(err) => Some(err),
Self::Frame(err) => Some(err),
Self::Decode(err) | Self::Encode(err) => Some(err),
Self::ProtocolMismatch { .. }
| Self::HandshakeTimeout
| Self::NoHandshake
| Self::PeerGone => None,
}
}
}
impl From<AuthError> for ConnError {
fn from(source: AuthError) -> Self {
Self::Auth(source)
}
}
impl From<std::io::Error> for ConnError {
fn from(source: std::io::Error) -> Self {
Self::Frame(source)
}
}
async fn handle_conn(stream: ServerStream, ctx: RpcContext) -> Result<(), ConnError> {
#[cfg(unix)]
check_peer(&stream, daemon_uid())?;
#[cfg(unix)]
let peer = peer_pid(&stream);
#[cfg(not(unix))]
let peer: Option<u32> = None;
if let Some(pid) = peer {
ctx.peer_contacts.connected(pid);
}
let conn = ConnId::next();
let (read_half, write_half) = shep_core::transport::split(stream);
let mut frames = FramedRead::new(read_half, codec());
let (out_tx, out_rx) = mpsc::channel::<Bytes>(CONN_QUEUE);
let writer = tokio::spawn(write_loop(FramedWrite::new(write_half, codec()), out_rx));
let outcome = converse(&mut frames, &out_tx, conn, peer, &ctx).await;
drop(out_tx);
let _ = writer.await;
ctx.supervisor.forget_smits(conn).await;
outcome
}
async fn converse(
frames: &mut Frames,
out: &mpsc::Sender<Bytes>,
conn: ConnId,
peer: Option<u32>,
ctx: &RpcContext,
) -> Result<(), ConnError> {
handshake(frames, out, peer, ctx).await?;
let mut forwarder: Option<JoinHandle<()>> = None;
let outcome = read_loop(frames, out, conn, ctx, &mut forwarder).await;
if let Some(forwarder) = forwarder {
forwarder.abort();
}
outcome
}
async fn read_loop(
frames: &mut Frames,
out: &mpsc::Sender<Bytes>,
conn: ConnId,
ctx: &RpcContext,
forwarder: &mut Option<JoinHandle<()>>,
) -> Result<(), ConnError> {
while let Some(frame) = frames.next().await {
let frame = frame?; let envelope: Envelope = decode_frame(&frame).map_err(ConnError::Decode)?;
match dispatch(envelope, conn, ctx).await {
Outcome::Reply(reply) => send(out, &reply).await?,
Outcome::Subscribe { reply, filter } => {
send(out, &reply).await?; if let Some(old) =
forwarder.replace(spawn_forwarder(&ctx.events, filter, out.clone()))
{
old.abort();
}
}
Outcome::Shutdown(reply) => {
send(out, &reply).await?;
ctx.shutdown();
break;
}
}
}
Ok(())
}
async fn handshake(
frames: &mut Frames,
out: &mpsc::Sender<Bytes>,
peer: Option<u32>,
ctx: &RpcContext,
) -> Result<(), ConnError> {
let frame = tokio::time::timeout(Duration::from_millis(HANDSHAKE_TIMEOUT_MS), frames.next())
.await
.map_err(|_| ConnError::HandshakeTimeout)?
.ok_or(ConnError::NoHandshake)??;
let hello: Hello = decode_frame(&frame).map_err(ConnError::Decode)?;
if hello.dog_name.is_some()
&& let Some(pid) = peer
{
ctx.peer_contacts.named_a_dog(pid);
}
if hello.protocol < MIN_SUPPORTED {
let refusal: HelloReply = Err(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: format!(
"client sent protocol {}, daemon speaks {PROTOCOL_VERSION} and accepts {MIN_SUPPORTED} and above",
hello.protocol
),
daemon_version: Some(ctx.daemon_version.clone()),
});
send(out, &refusal).await?;
match &hello.dog_name {
Some(dog) => {
crate::dogs::record_refused_dog(
dog,
&hello.client_version,
&ctx.dog_refusals,
&ctx.supervisor,
);
crate::dogs::narrate_by_name(
&ctx.supervisor,
&ctx.events,
dog,
format!(
"shep REFUSED this dog's handshake: this shepherd speaks protocol {PROTOCOL_VERSION} and the dog sent {}. Its own build is shep-client {}. Rebuild or reinstall it against this shep and run `shep restart {dog}`",
hello.protocol, hello.client_version
),
);
}
None => tracing::debug!(
client_protocol = hello.protocol,
client_version = %hello.client_version,
"refused a client on protocol skew"
),
}
return Err(ConnError::ProtocolMismatch {
client: hello.protocol,
});
}
if let Some(dog) = &hello.dog_name {
if ctx.dog_refusals.handshook(dog) {
crate::dogs::narrate_by_name(
&ctx.supervisor,
&ctx.events,
dog,
format!(
"shep accepted this dog's handshake; it is registered with this shepherd as `{dog}`, on protocol {PROTOCOL_VERSION}"
),
);
}
}
let ack: HelloReply = Ok(HelloAck {
daemon_version: ctx.daemon_version.clone(),
protocol: PROTOCOL_VERSION,
pid: ctx.pid,
min_supported: Some(MIN_SUPPORTED),
});
send(out, &ack).await
}
async fn write_loop(
mut sink: FramedWrite<ServerWriteHalf, LengthDelimitedCodec>,
mut rx: mpsc::Receiver<Bytes>,
) {
while let Some(bytes) = rx.recv().await {
if sink.send(bytes).await.is_err() {
break; }
}
}
async fn send<T: Serialize>(out: &mpsc::Sender<Bytes>, value: &T) -> Result<(), ConnError> {
let bytes = encode_frame(value).map_err(ConnError::Encode)?;
out.send(bytes).await.map_err(|_| ConnError::PeerGone)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::bus::SharedEvent;
use crate::fake::{FIRST_SCRIPTED_PID, ProcScript};
use crate::testing::harness;
use futures_util::{SinkExt, StreamExt};
use serde::Serialize;
use serde::de::DeserializeOwned;
use shep_core::protocol::{
BusEvent, DogSource, Envelope, Hello, HelloReply, PROTOCOL_VERSION, ProcessEventKind,
ProcessInfo, Request, Response, RpcErrorCode, ServerFrame, codec, decode_frame,
encode_frame,
};
use shep_core::status::ProcStatus;
use tokio_util::codec::Framed;
const RECV_TIMEOUT: Duration = Duration::from_secs(5);
struct Client {
frames: Framed<shep_core::transport::ClientStream, tokio_util::codec::LengthDelimitedCodec>,
}
impl Client {
async fn send<T: Serialize>(&mut self, value: &T) {
self.frames
.send(encode_frame(value).unwrap())
.await
.unwrap();
}
async fn recv<T: DeserializeOwned>(&mut self) -> T {
let frame = tokio::time::timeout(RECV_TIMEOUT, self.frames.next())
.await
.expect("timed out waiting for a frame")
.expect("connection closed early")
.unwrap();
decode_frame(&frame).unwrap()
}
async fn closed(&mut self) -> bool {
tokio::time::timeout(RECV_TIMEOUT, self.frames.next())
.await
.expect("timed out waiting for close")
.is_none()
}
}
async fn connected(ctx: RpcContext) -> Client {
let (server, client) = shep_core::transport::connected_pair().await.unwrap();
tokio::spawn(async move {
let _ = handle_conn(server, ctx).await;
});
Client {
frames: Framed::new(client, codec()),
}
}
#[tokio::test]
async fn handshake_acks_a_matching_protocol() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let ack: HelloReply = client.recv().await;
let ack = ack.expect("a matching protocol must be acked");
assert_eq!(ack.protocol, PROTOCOL_VERSION);
assert_eq!(ack.pid, h.ctx.pid);
assert_eq!(ack.daemon_version, h.ctx.daemon_version);
assert_eq!(ack.min_supported, Some(MIN_SUPPORTED));
}
#[tokio::test]
async fn handshake_accepts_a_peer_newer_than_the_daemon() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "9.9.9".to_string(),
protocol: PROTOCOL_VERSION + 1,
dog_name: None,
})
.await;
let ack: HelloReply = client.recv().await;
let ack = ack.expect("a peer above the daemon's own version must be accepted");
assert_eq!(ack.protocol, PROTOCOL_VERSION);
}
#[tokio::test]
async fn handshake_refuses_protocol_skew_before_closing() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "9.9.9".to_string(),
protocol: MIN_SUPPORTED - 1,
dog_name: None,
})
.await;
let refusal: HelloReply = client.recv().await;
let err = refusal.expect_err("skew below the floor must be refused");
assert_eq!(err.code, RpcErrorCode::ProtocolMismatch);
assert!(
client.closed().await,
"the daemon must close after refusing"
);
}
#[tokio::test]
async fn a_protocol_refusal_carries_the_daemon_version() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "9.9.9".to_string(),
protocol: MIN_SUPPORTED - 1,
dog_name: None,
})
.await;
let refusal: HelloReply = client.recv().await;
let err = refusal.expect_err("skew below the floor must be refused");
assert_eq!(err.code, RpcErrorCode::ProtocolMismatch);
assert_eq!(err.daemon_version.as_deref(), Some(&*h.ctx.daemon_version));
}
#[tokio::test]
async fn a_request_before_the_handshake_ends_the_connection() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Ping,
})
.await;
assert!(client.closed().await);
}
#[tokio::test]
async fn ping_round_trips_over_the_socket() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 11,
deadline_ms: Some(1000),
body: Request::Ping,
})
.await;
let frame: ServerFrame = client.recv().await;
let ServerFrame::Reply(reply) = frame else {
panic!("expected a reply frame")
};
assert_eq!(reply.id, 11);
assert_eq!(reply.result.unwrap(), Response::Pong);
}
#[tokio::test]
async fn subscribe_streams_only_matching_events() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Subscribe {
topics: vec!["process.*".to_string()],
},
})
.await;
let frame: ServerFrame = client.recv().await;
assert!(matches!(frame, ServerFrame::Reply(ref r) if r.result == Ok(Response::Subscribed)));
let event = |kind| -> SharedEvent {
BusEvent::Process {
event: kind,
info: ProcessInfo::builder(0, "web", shep_core::status::ProcStatus::Online)
.pid(Some(1000))
.out_file(Some("/logs/web-0-out.log".to_string()))
.err_file(Some("/logs/web-0-err.log".to_string()))
.build(),
manually: false,
at_ms: 0,
}
.into()
};
h.ctx.events.send(event(ProcessEventKind::Start)).unwrap();
h.ctx
.events
.send(
BusEvent::LogOut {
id: 0,
line: "filtered".to_string(),
}
.into(),
)
.unwrap();
h.ctx.events.send(event(ProcessEventKind::Online)).unwrap();
let first: ServerFrame = client.recv().await;
let second: ServerFrame = client.recv().await;
assert!(matches!(
first,
ServerFrame::Event(BusEvent::Process {
event: ProcessEventKind::Start,
..
})
));
assert!(matches!(
second,
ServerFrame::Event(BusEvent::Process {
event: ProcessEventKind::Online,
..
})
));
}
#[tokio::test]
async fn a_garbage_frame_ends_the_connection_without_panicking() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.frames
.send(bytes::Bytes::from_static(b"not json"))
.await
.unwrap();
assert!(client.closed().await);
}
#[tokio::test]
async fn a_garbage_frame_after_subscribing_still_closes_the_connection() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Subscribe {
topics: vec!["*".to_string()],
},
})
.await;
let _: ServerFrame = client.recv().await; client
.frames
.send(bytes::Bytes::from_static(b"not json"))
.await
.unwrap();
assert!(
client.closed().await,
"a live forwarder must not keep the connection open past a decode error"
);
}
#[cfg(unix)]
#[tokio::test]
async fn peer_credentials_gate_on_uid() {
let (a, _b) = tokio::net::UnixStream::pair().unwrap();
let me = daemon_uid();
assert_eq!(check_peer(&a, me).unwrap(), me);
assert_eq!(
check_peer(&a, me + 1).unwrap_err(),
AuthError::ForeignUid {
peer: me,
daemon: me + 1
}
);
}
#[tokio::test]
async fn a_slow_subscriber_gets_a_dropped_notice_instead_of_hanging_the_bus() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Subscribe {
topics: vec!["log.*".to_string()],
},
})
.await;
let _: ServerFrame = client.recv().await;
let flood = crate::bus::BUS_CAPACITY + CONN_QUEUE + 16;
for i in 0..flood {
h.ctx
.events
.send(
BusEvent::LogOut {
id: 0,
line: format!("line-{i}"),
}
.into(),
)
.unwrap();
}
let dropped = loop {
match client.recv::<ServerFrame>().await {
ServerFrame::Event(BusEvent::Dropped { count }) => break count,
ServerFrame::Event(_) => continue,
other => panic!("expected eventually a Dropped notice, got {other:?}"),
}
};
assert!(
dropped > 0,
"a flood past CONN_QUEUE + BUS_CAPACITY must report a real lag"
);
}
async fn start_dog(ctx: &RpcContext, name: &str) -> ProcessInfo {
let spec = crate::dogs::DogSpec {
name: name.to_string(),
source: DogSource::BuiltIn,
};
let app = crate::dogs::dog_app(&spec, &ctx.paths).expect("the dog fixture must assemble");
ctx.supervisor
.start_dog(app, DogSource::BuiltIn)
.await
.expect("the dog fixture must start")
}
async fn refuse_as(ctx: &RpcContext, dog: Option<&str>) {
let mut client = connected(ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.14".to_string(),
protocol: MIN_SUPPORTED - 1,
dog_name: dog.map(str::to_owned),
})
.await;
let refusal: HelloReply = client.recv().await;
refusal.expect_err("a protocol below the floor must be refused");
assert!(
client.closed().await,
"the daemon must close after refusing"
);
}
async fn dog_row(ctx: &RpcContext, name: &str) -> ProcessInfo {
ctx.supervisor
.list()
.await
.into_iter()
.find(|info| info.name == name)
.unwrap_or_else(|| panic!("no row named {name}"))
}
async fn await_dog(ctx: &RpcContext, name: &str, pid: u32) -> Duration {
let began = tokio::time::Instant::now();
let seen = tokio::time::timeout(RECV_TIMEOUT, async {
loop {
if dog_row(ctx, name).await.pid == Some(pid) {
return;
}
tokio::task::yield_now().await;
}
})
.await;
assert!(
seen.is_ok(),
"{name} never reached pid {pid} within {RECV_TIMEOUT:?}: {:?}",
ctx.supervisor.list().await
);
began.elapsed()
}
#[tokio::test]
async fn a_refused_dog_is_restarted_once_from_the_binary_on_disk() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let started = start_dog(&h.ctx, "metrics").await;
assert_eq!(started.pid, Some(FIRST_SCRIPTED_PID));
refuse_as(&h.ctx, Some("metrics")).await;
await_dog(&h.ctx, "metrics", FIRST_SCRIPTED_PID + 1).await;
assert!(
h.ctx.dog_refusals.stale().is_empty(),
"one refusal buys a restart; it does not condemn the dog"
);
}
#[tokio::test]
async fn a_twice_refused_dog_is_reported_stale_and_never_restarted_again() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
refuse_as(&h.ctx, Some("metrics")).await;
let restart_took = await_dog(&h.ctx, "metrics", FIRST_SCRIPTED_PID + 1).await;
refuse_as(&h.ctx, Some("metrics")).await;
assert_eq!(
h.ctx.dog_refusals.stale(),
vec!["metrics".to_string()],
"the second refusal must be reported, not swallowed"
);
let window = (restart_took * 10).max(Duration::from_millis(200));
tokio::time::sleep(window).await;
let after = dog_row(&h.ctx, "metrics").await;
assert_eq!(
after.pid,
Some(FIRST_SCRIPTED_PID + 1),
"a second restart within {window:?} is the spin G8 forbids"
);
assert_eq!(
after.status,
ProcStatus::Online,
"a third spawn would exhaust the script and error the dog"
);
}
#[tokio::test]
async fn a_dog_that_handshakes_is_no_longer_stale() {
let h = harness(vec![]);
refuse_as(&h.ctx, Some("metrics")).await;
refuse_as(&h.ctx, Some("metrics")).await;
assert_eq!(h.ctx.dog_refusals.stale(), vec!["metrics".to_string()]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.22".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: Some("metrics".to_string()),
})
.await;
let ack: HelloReply = client.recv().await;
ack.expect("a matching protocol must be acked");
assert!(
h.ctx.dog_refusals.stale().is_empty(),
"a dog talking to this daemon is not stale by any definition it can apply"
);
}
#[tokio::test]
async fn a_refused_client_that_is_not_a_dog_touches_nothing() {
let h = harness(vec![ProcScript::never_exits()]);
let started = start_dog(&h.ctx, "metrics").await;
for _ in 0..3 {
refuse_as(&h.ctx, None).await;
}
assert!(h.ctx.dog_refusals.stale().is_empty());
let after = dog_row(&h.ctx, "metrics").await;
assert_eq!(
after.pid, started.pid,
"a nameless refusal must not restart anything"
);
assert_eq!(after.status, ProcStatus::Online);
}
#[cfg(unix)]
#[tokio::test]
async fn a_dog_that_connects_without_naming_itself_is_not_called_a_stale_binary() {
let h = crate::testing::harness_at_pid(
vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
std::process::id(),
);
h.ctx.peer_contacts.force_warm();
let dog = start_dog(&h.ctx, "log-rotate").await;
assert_eq!(
dog.pid,
Some(std::process::id()),
"the fixture only means anything if the dog and this test are one process"
);
let err_log = dog
.err_file
.clone()
.expect("a dog's listing resolves its log paths");
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
protocol: PROTOCOL_VERSION,
client_version: "0.1.22".to_string(),
dog_name: None,
})
.await;
let ack: HelloReply = client.recv().await;
ack.expect("an anonymous handshake on the current protocol is ACCEPTED, not refused");
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::ListFlock,
})
.await;
match client.recv::<ServerFrame>().await {
ServerFrame::Reply(reply) => match reply.result {
Ok(Response::Flock(flock)) => assert!(
flock.iter().any(|info| info.name == "log-rotate"),
"the connection this daemon is about to call stale is serving requests"
),
other => panic!("ListFlock must answer with the flock, got {other:?}"),
},
other => panic!("an accepted connection must serve ListFlock, got {other:?}"),
}
assert_eq!(
h.ctx.peer_contacts.from_pid(Some(std::process::id())),
crate::dogs::Contact::Anonymous,
"the handshake path must record what actually arrived"
);
assert!(
!h.ctx.dog_refusals.has_handshook("log-rotate"),
"no `dog_name` means no handshake was recorded, which is the whole trap"
);
let mut seen = crate::dogs::SilentDogs::default();
let t0 = tokio::time::Instant::now();
let ladder = async |seen: &mut crate::dogs::SilentDogs, at| {
crate::dogs::check_silent_dogs(
&h.ctx.supervisor,
&h.ctx.dog_refusals,
&h.ctx.peer_contacts,
&h.ctx.events,
seen,
at,
)
.await
};
assert!(ladder(&mut seen, t0).await.is_empty());
assert_eq!(
ladder(&mut seen, t0 + crate::dogs::DOG_SILENCE_BUDGET).await,
vec![("log-rotate".to_string(), crate::dogs::Refusal::Restart)]
);
assert_eq!(
ladder(&mut seen, t0 + 2 * crate::dogs::DOG_SILENCE_BUDGET).await,
vec![("log-rotate".to_string(), crate::dogs::Refusal::Stale)],
"the ladder's verdict is unchanged; what changes is what it SAYS"
);
let written = std::fs::read_to_string(&err_log).expect("the narration must reach the log");
assert!(
written.contains("[shep]"),
"shep's voice in a dog's log has to be marked as shep's: {written}"
);
assert!(
written.contains("HAS connected to this shepherd"),
"the verdict must say what this shepherd watched arrive: {written}"
);
assert!(
written.contains("reinstalling the same build will NOT"),
"the two days were spent on advice this line has to refuse: {written}"
);
assert!(
!written.contains("cannot talk to this shep either"),
"the sentence that cost two days must not appear on this path: {written}"
);
assert!(
!written.contains("cannot reach this shep"),
"this dog reached shep; nothing here may claim otherwise: {written}"
);
}
}