use super::peer_auth::ConnError;
#[cfg(unix)]
use super::peer_auth::{check_peer, daemon_uid, peer_pid};
use super::server_lifecycle::{CONN_QUEUE, HANDSHAKE_TIMEOUT_MS};
use crate::bus::spawn_forwarder;
use crate::rpc::{Outcome, RpcContext, dispatch};
use crate::supervisor::ConnId;
use bytes::Bytes;
use core::time::Duration;
use futures_util::{SinkExt, StreamExt};
use serde::Serialize;
use shep_core::protocol::{
Envelope, Hello, HelloAck, HelloReply, MIN_SUPPORTED, PROTOCOL_VERSION, RpcError, RpcErrorCode,
codec, decode_frame, encode_frame,
};
use shep_core::transport::{ServerReadHalf, ServerStream};
use tokio::io::AsyncWrite;
use tokio::sync::mpsc;
use tokio::task::JoinHandle;
use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec};
type Frames = FramedRead<ServerReadHalf, LengthDelimitedCodec>;
pub(super) async fn handle_conn(stream: ServerStream, ctx: RpcContext) -> Result<(), ConnError> {
#[cfg(unix)]
check_peer(&stream, daemon_uid())?;
#[cfg(unix)]
let peer = peer_pid(&stream);
#[cfg(not(unix))]
let peer: Option<u32> = None;
if let Some(pid) = peer {
ctx.peer_contacts.connected(pid);
}
let conn = ConnId::next();
let (read_half, write_half) = shep_core::transport::split(stream);
let mut frames = FramedRead::new(read_half, codec());
let (out_tx, out_rx) = mpsc::channel::<Bytes>(CONN_QUEUE);
let writer = tokio::spawn(write_loop(FramedWrite::new(write_half, codec()), out_rx));
let outcome = converse(&mut frames, &out_tx, conn, peer, &ctx).await;
drop(out_tx);
let _ = writer.await;
ctx.supervisor.forget_smits(conn).await;
outcome
}
async fn converse(
frames: &mut Frames,
out: &mpsc::Sender<Bytes>,
conn: ConnId,
peer: Option<u32>,
ctx: &RpcContext,
) -> Result<(), ConnError> {
handshake(frames, out, peer, ctx).await?;
let mut forwarder: Option<JoinHandle<()>> = None;
let outcome = read_loop(frames, out, conn, ctx, &mut forwarder).await;
if let Some(forwarder) = forwarder {
forwarder.abort();
}
outcome
}
async fn read_loop(
frames: &mut Frames,
out: &mpsc::Sender<Bytes>,
conn: ConnId,
ctx: &RpcContext,
forwarder: &mut Option<JoinHandle<()>>,
) -> Result<(), ConnError> {
while let Some(frame) = frames.next().await {
let frame = frame?; let envelope: Envelope = decode_frame(&frame).map_err(ConnError::Decode)?;
match dispatch(envelope, conn, ctx).await {
Outcome::Reply(reply) => send(out, &reply).await?,
Outcome::Subscribe { reply, filter } => {
send(out, &reply).await?; if let Some(old) =
forwarder.replace(spawn_forwarder(&ctx.events, filter, out.clone()))
{
old.abort();
}
}
Outcome::Shutdown(reply) => {
send(out, &reply).await?;
ctx.shutdown();
break;
}
}
}
Ok(())
}
async fn handshake(
frames: &mut Frames,
out: &mpsc::Sender<Bytes>,
peer: Option<u32>,
ctx: &RpcContext,
) -> Result<(), ConnError> {
let frame = tokio::time::timeout(Duration::from_millis(HANDSHAKE_TIMEOUT_MS), frames.next())
.await
.map_err(|_| ConnError::HandshakeTimeout)?
.ok_or(ConnError::NoHandshake)??;
let hello: Hello = decode_frame(&frame).map_err(ConnError::Decode)?;
if hello.dog_name.is_some()
&& let Some(pid) = peer
{
ctx.peer_contacts.named_a_dog(pid);
}
if hello.protocol < MIN_SUPPORTED {
let refusal: HelloReply = Err(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: format!(
"client sent protocol {}, daemon speaks {PROTOCOL_VERSION} and accepts {MIN_SUPPORTED} and above",
hello.protocol
),
daemon_version: Some(ctx.daemon_version.clone()),
});
send(out, &refusal).await?;
match &hello.dog_name {
Some(dog) => {
crate::dogs::record_refused_dog(
dog,
&hello.client_version,
&ctx.dog_refusals,
&ctx.supervisor,
);
crate::dogs::narrate_by_name(
&ctx.supervisor,
&ctx.events,
dog,
format!(
"shep REFUSED this dog's handshake: this shepherd speaks protocol {PROTOCOL_VERSION} and the dog sent {}. Its own build is shep-client {}. Rebuild or reinstall it against this shep and run `shep restart {dog}`",
hello.protocol, hello.client_version
),
);
}
None => tracing::debug!(
client_protocol = hello.protocol,
client_version = %hello.client_version,
"refused a client on protocol skew"
),
}
return Err(ConnError::ProtocolMismatch {
client: hello.protocol,
});
}
if let Some(dog) = &hello.dog_name {
if ctx.dog_refusals.handshook(dog) {
crate::dogs::narrate_by_name(
&ctx.supervisor,
&ctx.events,
dog,
format!(
"shep accepted this dog's handshake; it is registered with this shepherd as `{dog}`, on protocol {PROTOCOL_VERSION}"
),
);
}
}
let ack: HelloReply = Ok(HelloAck {
daemon_version: ctx.daemon_version.clone(),
protocol: PROTOCOL_VERSION,
pid: ctx.pid,
min_supported: Some(MIN_SUPPORTED),
});
send(out, &ack).await
}
async fn write_loop<W: AsyncWrite + Unpin>(
mut sink: FramedWrite<W, LengthDelimitedCodec>,
mut rx: mpsc::Receiver<Bytes>,
) {
while let Some(first) = rx.recv().await {
let mut next = Some(first);
let mut fed = 0;
while let Some(bytes) = next {
if sink.feed(bytes).await.is_err() {
return; }
fed += 1;
next = if fed < CONN_QUEUE {
rx.try_recv().ok()
} else {
None
};
}
if SinkExt::<Bytes>::flush(&mut sink).await.is_err() {
return; }
}
}
async fn send<T: Serialize>(out: &mpsc::Sender<Bytes>, value: &T) -> Result<(), ConnError> {
let bytes = encode_frame(value).map_err(ConnError::Encode)?;
out.send(bytes).await.map_err(|_| ConnError::PeerGone)
}
#[cfg(test)]
mod tests {
use super::super::server_lifecycle::CONN_QUEUE;
use futures_util::SinkExt;
use shep_core::protocol::{
Envelope, Hello, HelloReply, MIN_SUPPORTED, PROTOCOL_VERSION, RpcErrorCode,
};
use crate::bus::SharedEvent;
use crate::fake::ProcScript;
use crate::testing::harness;
use shep_core::protocol::{
BusEvent, ProcessEventKind, ProcessInfo, Request, Response, ServerFrame,
};
use super::super::testing::*;
use crate::fake::FIRST_SCRIPTED_PID;
use core::time::Duration;
use shep_core::status::ProcStatus;
#[tokio::test]
async fn handshake_acks_a_matching_protocol() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let ack: HelloReply = client.recv().await;
let ack = ack.expect("a matching protocol must be acked");
assert_eq!(ack.protocol, PROTOCOL_VERSION);
assert_eq!(ack.pid, h.ctx.pid);
assert_eq!(ack.daemon_version, h.ctx.daemon_version);
assert_eq!(ack.min_supported, Some(MIN_SUPPORTED));
}
#[tokio::test]
async fn handshake_accepts_a_peer_newer_than_the_daemon() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "9.9.9".to_string(),
protocol: PROTOCOL_VERSION + 1,
dog_name: None,
})
.await;
let ack: HelloReply = client.recv().await;
let ack = ack.expect("a peer above the daemon's own version must be accepted");
assert_eq!(ack.protocol, PROTOCOL_VERSION);
}
#[tokio::test]
async fn handshake_refuses_protocol_skew_before_closing() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "9.9.9".to_string(),
protocol: MIN_SUPPORTED - 1,
dog_name: None,
})
.await;
let refusal: HelloReply = client.recv().await;
let err = refusal.expect_err("skew below the floor must be refused");
assert_eq!(err.code, RpcErrorCode::ProtocolMismatch);
assert!(
client.closed().await,
"the daemon must close after refusing"
);
}
#[tokio::test]
async fn a_protocol_refusal_carries_the_daemon_version() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "9.9.9".to_string(),
protocol: MIN_SUPPORTED - 1,
dog_name: None,
})
.await;
let refusal: HelloReply = client.recv().await;
let err = refusal.expect_err("skew below the floor must be refused");
assert_eq!(err.code, RpcErrorCode::ProtocolMismatch);
assert_eq!(err.daemon_version.as_deref(), Some(&*h.ctx.daemon_version));
}
#[tokio::test]
async fn a_request_before_the_handshake_ends_the_connection() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Ping,
})
.await;
assert!(client.closed().await);
}
#[tokio::test]
async fn ping_round_trips_over_the_socket() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 11,
deadline_ms: Some(1000),
body: Request::Ping,
})
.await;
let frame: ServerFrame = client.recv().await;
let ServerFrame::Reply(reply) = frame else {
panic!("expected a reply frame")
};
assert_eq!(reply.id, 11);
assert_eq!(reply.result.unwrap(), Response::Pong);
}
#[tokio::test]
async fn subscribe_streams_only_matching_events() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Subscribe {
topics: vec!["process.*".to_string()],
},
})
.await;
let frame: ServerFrame = client.recv().await;
assert!(matches!(frame, ServerFrame::Reply(ref r) if r.result == Ok(Response::Subscribed)));
let event = |kind| -> SharedEvent {
BusEvent::Process {
event: kind,
info: ProcessInfo::builder(0, "web", shep_core::status::ProcStatus::Online)
.pid(Some(1000))
.out_file(Some("/logs/web-0-out.log".to_string()))
.err_file(Some("/logs/web-0-err.log".to_string()))
.build(),
manually: false,
at_ms: 0,
}
.into()
};
h.ctx.events.send(event(ProcessEventKind::Start)).unwrap();
h.ctx
.events
.send(
BusEvent::LogOut {
id: 0,
line: "filtered".to_string(),
}
.into(),
)
.unwrap();
h.ctx.events.send(event(ProcessEventKind::Online)).unwrap();
let first: ServerFrame = client.recv().await;
let second: ServerFrame = client.recv().await;
assert!(matches!(
first,
ServerFrame::Event(BusEvent::Process {
event: ProcessEventKind::Start,
..
})
));
assert!(matches!(
second,
ServerFrame::Event(BusEvent::Process {
event: ProcessEventKind::Online,
..
})
));
}
#[tokio::test]
async fn a_garbage_frame_ends_the_connection_without_panicking() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.frames
.send(bytes::Bytes::from_static(b"not json"))
.await
.unwrap();
assert!(client.closed().await);
}
#[tokio::test]
async fn a_garbage_frame_after_subscribing_still_closes_the_connection() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Subscribe {
topics: vec!["*".to_string()],
},
})
.await;
let _: ServerFrame = client.recv().await; client
.frames
.send(bytes::Bytes::from_static(b"not json"))
.await
.unwrap();
assert!(
client.closed().await,
"a live forwarder must not keep the connection open past a decode error"
);
}
#[tokio::test]
async fn a_slow_subscriber_gets_a_dropped_notice_instead_of_hanging_the_bus() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: None,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Subscribe {
topics: vec!["log.*".to_string()],
},
})
.await;
let _: ServerFrame = client.recv().await;
let flood = crate::bus::BUS_CAPACITY + CONN_QUEUE + 16;
for i in 0..flood {
h.ctx
.events
.send(
BusEvent::LogOut {
id: 0,
line: format!("line-{i}"),
}
.into(),
)
.unwrap();
}
let dropped = loop {
match client.recv::<ServerFrame>().await {
ServerFrame::Event(BusEvent::Dropped { count }) => break count,
ServerFrame::Event(_) => continue,
other => panic!("expected eventually a Dropped notice, got {other:?}"),
}
};
assert!(
dropped > 0,
"a flood past CONN_QUEUE + BUS_CAPACITY must report a real lag"
);
}
#[tokio::test]
async fn a_dog_that_handshakes_is_no_longer_stale() {
let h = harness(vec![]);
refuse_as(&h.ctx, Some("metrics")).await;
refuse_as(&h.ctx, Some("metrics")).await;
assert_eq!(h.ctx.dog_refusals.stale(), vec!["metrics".to_string()]);
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
client_version: "0.1.22".to_string(),
protocol: PROTOCOL_VERSION,
dog_name: Some("metrics".to_string()),
})
.await;
let ack: HelloReply = client.recv().await;
ack.expect("a matching protocol must be acked");
assert!(
h.ctx.dog_refusals.stale().is_empty(),
"a dog talking to this daemon is not stale by any definition it can apply"
);
}
#[cfg(unix)]
#[tokio::test]
async fn a_dog_that_connects_without_naming_itself_is_not_called_a_stale_binary() {
let h = crate::testing::harness_at_pid(
vec![
ProcScript::never_exits(),
ProcScript::never_exits(),
ProcScript::never_exits(),
],
std::process::id(),
);
h.ctx.peer_contacts.force_warm();
let dog = start_dog(&h.ctx, "log-rotate").await;
assert_eq!(
dog.pid,
Some(std::process::id()),
"the fixture only means anything if the dog and this test are one process"
);
let err_log = dog
.err_file
.clone()
.expect("a dog's listing resolves its log paths");
let mut client = connected(h.ctx.clone()).await;
client
.send(&Hello {
protocol: PROTOCOL_VERSION,
client_version: "0.1.22".to_string(),
dog_name: None,
})
.await;
let ack: HelloReply = client.recv().await;
ack.expect("an anonymous handshake on the current protocol is ACCEPTED, not refused");
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::ListFlock,
})
.await;
match client.recv::<ServerFrame>().await {
ServerFrame::Reply(reply) => match reply.result {
Ok(Response::Flock(flock)) => assert!(
flock.iter().any(|info| info.name == "log-rotate"),
"the connection this daemon is about to call stale is serving requests"
),
other => panic!("ListFlock must answer with the flock, got {other:?}"),
},
other => panic!("an accepted connection must serve ListFlock, got {other:?}"),
}
assert_eq!(
h.ctx.peer_contacts.from_pid(Some(std::process::id())),
crate::dogs::Contact::Anonymous,
"the handshake path must record what actually arrived"
);
assert!(
!h.ctx.dog_refusals.has_handshook("log-rotate"),
"no `dog_name` means no handshake was recorded, which is the whole trap"
);
let mut seen = crate::dogs::SilentDogs::default();
let t0 = tokio::time::Instant::now();
let ladder = async |seen: &mut crate::dogs::SilentDogs, at| {
crate::dogs::check_silent_dogs(
&h.ctx.supervisor,
&h.ctx.dog_refusals,
&h.ctx.peer_contacts,
&h.ctx.events,
seen,
at,
)
.await
};
assert!(ladder(&mut seen, t0).await.is_empty());
assert_eq!(
ladder(&mut seen, t0 + crate::dogs::DOG_SILENCE_BUDGET).await,
vec![("log-rotate".to_string(), crate::dogs::Refusal::Restart)]
);
assert_eq!(
ladder(&mut seen, t0 + 2 * crate::dogs::DOG_SILENCE_BUDGET).await,
vec![("log-rotate".to_string(), crate::dogs::Refusal::Stale)],
"the ladder's verdict is unchanged; what changes is what it SAYS"
);
let written = std::fs::read_to_string(&err_log).expect("the narration must reach the log");
assert!(
written.contains("[shep]"),
"shep's voice in a dog's log has to be marked as shep's: {written}"
);
assert!(
written.contains("HAS connected to this shepherd"),
"the verdict must say what this shepherd watched arrive: {written}"
);
assert!(
written.contains("reinstalling the same build will NOT"),
"the two days were spent on advice this line has to refuse: {written}"
);
assert!(
!written.contains("cannot talk to this shep either"),
"the sentence that cost two days must not appear on this path: {written}"
);
assert!(
!written.contains("cannot reach this shep"),
"this dog reached shep; nothing here may claim otherwise: {written}"
);
}
#[tokio::test]
async fn a_refused_dog_is_restarted_once_from_the_binary_on_disk() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
let started = start_dog(&h.ctx, "metrics").await;
assert_eq!(started.pid, Some(FIRST_SCRIPTED_PID));
refuse_as(&h.ctx, Some("metrics")).await;
await_dog(&h.ctx, "metrics", FIRST_SCRIPTED_PID + 1).await;
assert!(
h.ctx.dog_refusals.stale().is_empty(),
"one refusal buys a restart; it does not condemn the dog"
);
}
#[tokio::test]
async fn a_twice_refused_dog_is_reported_stale_and_never_restarted_again() {
let h = harness(vec![ProcScript::never_exits(), ProcScript::never_exits()]);
start_dog(&h.ctx, "metrics").await;
refuse_as(&h.ctx, Some("metrics")).await;
let restart_took = await_dog(&h.ctx, "metrics", FIRST_SCRIPTED_PID + 1).await;
refuse_as(&h.ctx, Some("metrics")).await;
assert_eq!(
h.ctx.dog_refusals.stale(),
vec!["metrics".to_string()],
"the second refusal must be reported, not swallowed"
);
let window = (restart_took * 10).max(Duration::from_millis(200));
tokio::time::sleep(window).await;
let after = dog_row(&h.ctx, "metrics").await;
assert_eq!(
after.pid,
Some(FIRST_SCRIPTED_PID + 1),
"a second restart within {window:?} is the spin G8 forbids"
);
assert_eq!(
after.status,
ProcStatus::Online,
"a third spawn would exhaust the script and error the dog"
);
}
#[tokio::test]
async fn a_refused_client_that_is_not_a_dog_touches_nothing() {
let h = harness(vec![ProcScript::never_exits()]);
let started = start_dog(&h.ctx, "metrics").await;
for _ in 0..3 {
refuse_as(&h.ctx, None).await;
}
assert!(h.ctx.dog_refusals.stale().is_empty());
let after = dog_row(&h.ctx, "metrics").await;
assert_eq!(
after.pid, started.pid,
"a nameless refusal must not restart anything"
);
assert_eq!(after.status, ProcStatus::Online);
}
}
#[cfg(test)]
mod write_loop_tests;