use core::fmt;
use core::time::Duration;
use bytes::Bytes;
use futures_util::{SinkExt, StreamExt};
use serde::Serialize;
use tokio::net::unix::{OwnedReadHalf, OwnedWriteHalf};
use tokio::net::{UnixListener, UnixStream};
use tokio::sync::{mpsc, watch};
use tokio::task::JoinHandle;
use tokio_util::codec::{FramedRead, FramedWrite, LengthDelimitedCodec};
use shep_core::protocol::{
Envelope, Hello, HelloAck, HelloReply, PROTOCOL_VERSION, RpcError, RpcErrorCode, WireError,
codec, decode_frame, encode_frame,
};
use crate::bus::spawn_forwarder;
use crate::rpc::{Outcome, RpcContext, dispatch};
use crate::supervisor::ConnId;
pub const CONN_QUEUE: usize = 64;
pub const HANDSHAKE_TIMEOUT_MS: u64 = 5_000;
type Frames = FramedRead<OwnedReadHalf, LengthDelimitedCodec>;
#[derive(Debug)]
pub struct RpcServer {
listener: UnixListener,
ctx: RpcContext,
}
impl RpcServer {
#[must_use]
pub fn new(listener: UnixListener, ctx: RpcContext) -> Self {
Self { listener, ctx }
}
pub async fn serve(self, mut shutdown: watch::Receiver<bool>) {
let Self { listener, ctx } = self;
if *shutdown.borrow() {
return;
}
loop {
tokio::select! {
accepted = listener.accept() => {
match accepted {
Ok((stream, _addr)) => {
let ctx = ctx.clone();
tokio::spawn(async move {
if let Err(err) = handle_conn(stream, ctx).await {
tracing::debug!(%err, "connection ended");
}
});
}
Err(err) => tracing::warn!(%err, "accept failed; continuing"),
}
}
changed = shutdown.changed() => {
if changed.is_err() || *shutdown.borrow() {
break;
}
}
}
}
}
}
pub fn check_peer(stream: &UnixStream, daemon_uid: u32) -> Result<u32, AuthError> {
let cred = stream
.peer_cred()
.map_err(|err| AuthError::NoCredentials(err.to_string()))?;
let peer = cred.uid();
if peer == daemon_uid {
Ok(peer)
} else {
Err(AuthError::ForeignUid {
peer,
daemon: daemon_uid,
})
}
}
#[must_use]
pub fn daemon_uid() -> u32 {
nix::unistd::geteuid().as_raw()
}
#[non_exhaustive]
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum AuthError {
NoCredentials(String),
ForeignUid {
peer: u32,
daemon: u32,
},
}
impl fmt::Display for AuthError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::NoCredentials(msg) => write!(f, "could not read peer credentials: {msg}"),
Self::ForeignUid { peer, daemon } => {
write!(f, "peer uid {peer} does not match daemon uid {daemon}")
}
}
}
}
impl core::error::Error for AuthError {}
#[non_exhaustive]
#[derive(Debug)]
pub enum ConnError {
Auth(AuthError),
Frame(std::io::Error),
Decode(WireError),
Encode(WireError),
ProtocolMismatch {
client: u32,
},
HandshakeTimeout,
NoHandshake,
PeerGone,
}
impl fmt::Display for ConnError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Auth(err) => write!(f, "peer-credential check failed: {err}"),
Self::Frame(err) => write!(f, "frame transport error: {err}"),
Self::Decode(err) => write!(f, "frame decode error: {err}"),
Self::Encode(err) => write!(f, "frame encode error: {err}"),
Self::ProtocolMismatch { client } => write!(
f,
"client sent protocol {client}, daemon speaks {PROTOCOL_VERSION}"
),
Self::HandshakeTimeout => write!(
f,
"peer did not send Hello within {HANDSHAKE_TIMEOUT_MS} ms"
),
Self::NoHandshake => f.write_str("peer closed before sending Hello"),
Self::PeerGone => f.write_str("connection's write queue is gone"),
}
}
}
impl core::error::Error for ConnError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Auth(err) => Some(err),
Self::Frame(err) => Some(err),
Self::Decode(err) | Self::Encode(err) => Some(err),
Self::ProtocolMismatch { .. }
| Self::HandshakeTimeout
| Self::NoHandshake
| Self::PeerGone => None,
}
}
}
impl From<AuthError> for ConnError {
fn from(source: AuthError) -> Self {
Self::Auth(source)
}
}
impl From<std::io::Error> for ConnError {
fn from(source: std::io::Error) -> Self {
Self::Frame(source)
}
}
async fn handle_conn(stream: UnixStream, ctx: RpcContext) -> Result<(), ConnError> {
check_peer(&stream, daemon_uid())?;
let conn = ConnId::next();
let (read_half, write_half) = stream.into_split();
let mut frames = FramedRead::new(read_half, codec());
let (out_tx, out_rx) = mpsc::channel::<Bytes>(CONN_QUEUE);
let writer = tokio::spawn(write_loop(FramedWrite::new(write_half, codec()), out_rx));
let outcome = converse(&mut frames, &out_tx, conn, &ctx).await;
drop(out_tx);
let _ = writer.await;
ctx.supervisor.forget_smits(conn).await;
outcome
}
async fn converse(
frames: &mut Frames,
out: &mpsc::Sender<Bytes>,
conn: ConnId,
ctx: &RpcContext,
) -> Result<(), ConnError> {
handshake(frames, out, ctx).await?;
let mut forwarder: Option<JoinHandle<()>> = None;
let outcome = read_loop(frames, out, conn, ctx, &mut forwarder).await;
if let Some(forwarder) = forwarder {
forwarder.abort();
}
outcome
}
async fn read_loop(
frames: &mut Frames,
out: &mpsc::Sender<Bytes>,
conn: ConnId,
ctx: &RpcContext,
forwarder: &mut Option<JoinHandle<()>>,
) -> Result<(), ConnError> {
while let Some(frame) = frames.next().await {
let frame = frame?; let envelope: Envelope = decode_frame(&frame).map_err(ConnError::Decode)?;
match dispatch(envelope, conn, ctx).await {
Outcome::Reply(reply) => send(out, &reply).await?,
Outcome::Subscribe { reply, filter } => {
send(out, &reply).await?; if let Some(old) =
forwarder.replace(spawn_forwarder(ctx.events.subscribe(), filter, out.clone()))
{
old.abort();
}
}
Outcome::Shutdown(reply) => {
send(out, &reply).await?;
ctx.shutdown();
break;
}
}
}
Ok(())
}
async fn handshake(
frames: &mut Frames,
out: &mpsc::Sender<Bytes>,
ctx: &RpcContext,
) -> Result<(), ConnError> {
let frame = tokio::time::timeout(Duration::from_millis(HANDSHAKE_TIMEOUT_MS), frames.next())
.await
.map_err(|_| ConnError::HandshakeTimeout)?
.ok_or(ConnError::NoHandshake)??;
let hello: Hello = decode_frame(&frame).map_err(ConnError::Decode)?;
if hello.protocol != PROTOCOL_VERSION {
let refusal: HelloReply = Err(RpcError {
code: RpcErrorCode::ProtocolMismatch,
message: format!(
"daemon speaks protocol {PROTOCOL_VERSION}, client sent {}",
hello.protocol
),
});
send(out, &refusal).await?;
return Err(ConnError::ProtocolMismatch {
client: hello.protocol,
});
}
let ack: HelloReply = Ok(HelloAck {
daemon_version: ctx.daemon_version.clone(),
protocol: PROTOCOL_VERSION,
pid: ctx.pid,
});
send(out, &ack).await
}
async fn write_loop(
mut sink: FramedWrite<OwnedWriteHalf, LengthDelimitedCodec>,
mut rx: mpsc::Receiver<Bytes>,
) {
while let Some(bytes) = rx.recv().await {
if sink.send(bytes).await.is_err() {
break; }
}
}
async fn send<T: Serialize>(out: &mpsc::Sender<Bytes>, value: &T) -> Result<(), ConnError> {
let bytes = encode_frame(value).map_err(ConnError::Encode)?;
out.send(bytes).await.map_err(|_| ConnError::PeerGone)
}
#[cfg(test)]
mod tests {
use super::*;
use crate::testing::harness;
use futures_util::{SinkExt, StreamExt};
use serde::Serialize;
use serde::de::DeserializeOwned;
use shep_core::protocol::{
BusEvent, Envelope, Hello, HelloReply, PROTOCOL_VERSION, ProcessEventKind, ProcessInfo,
Request, Response, RpcErrorCode, ServerFrame, codec, decode_frame, encode_frame,
};
use tokio_util::codec::Framed;
const RECV_TIMEOUT: Duration = Duration::from_secs(5);
struct Client {
frames: Framed<UnixStream, tokio_util::codec::LengthDelimitedCodec>,
}
impl Client {
async fn send<T: Serialize>(&mut self, value: &T) {
self.frames
.send(encode_frame(value).unwrap())
.await
.unwrap();
}
async fn recv<T: DeserializeOwned>(&mut self) -> T {
let frame = tokio::time::timeout(RECV_TIMEOUT, self.frames.next())
.await
.expect("timed out waiting for a frame")
.expect("connection closed early")
.unwrap();
decode_frame(&frame).unwrap()
}
async fn closed(&mut self) -> bool {
tokio::time::timeout(RECV_TIMEOUT, self.frames.next())
.await
.expect("timed out waiting for close")
.is_none()
}
}
fn connected(ctx: RpcContext) -> Client {
let (server, client) = UnixStream::pair().unwrap();
tokio::spawn(async move {
let _ = handle_conn(server, ctx).await;
});
Client {
frames: Framed::new(client, codec()),
}
}
#[tokio::test]
async fn handshake_acks_a_matching_protocol() {
let h = harness(vec![]); let mut client = connected(h.ctx.clone());
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
})
.await;
let ack: HelloReply = client.recv().await;
let ack = ack.expect("a matching protocol must be acked");
assert_eq!(ack.protocol, PROTOCOL_VERSION);
assert_eq!(ack.pid, h.ctx.pid);
assert_eq!(ack.daemon_version, h.ctx.daemon_version);
}
#[tokio::test]
async fn handshake_refuses_protocol_skew_before_closing() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone());
client
.send(&Hello {
client_version: "9.9.9".to_string(),
protocol: PROTOCOL_VERSION + 1,
})
.await;
let refusal: HelloReply = client.recv().await;
let err = refusal.expect_err("skew must be refused");
assert_eq!(err.code, RpcErrorCode::ProtocolMismatch);
assert!(
client.closed().await,
"the daemon must close after refusing"
);
}
#[tokio::test]
async fn a_request_before_the_handshake_ends_the_connection() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone());
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Ping,
})
.await;
assert!(client.closed().await);
}
#[tokio::test]
async fn ping_round_trips_over_the_socket() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone());
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 11,
deadline_ms: Some(1000),
body: Request::Ping,
})
.await;
let frame: ServerFrame = client.recv().await;
let ServerFrame::Reply(reply) = frame else {
panic!("expected a reply frame")
};
assert_eq!(reply.id, 11);
assert_eq!(reply.result.unwrap(), Response::Pong);
}
#[tokio::test]
async fn subscribe_streams_only_matching_events() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone());
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Subscribe {
topics: vec!["process.*".to_string()],
},
})
.await;
let frame: ServerFrame = client.recv().await;
assert!(matches!(frame, ServerFrame::Reply(ref r) if r.result == Ok(Response::Subscribed)));
let event = |kind| BusEvent::Process {
event: kind,
info: ProcessInfo::builder(0, "web", shep_core::status::ProcStatus::Online)
.pid(Some(1000))
.out_file(Some("/logs/web-0-out.log".to_string()))
.err_file(Some("/logs/web-0-err.log".to_string()))
.build(),
manually: false,
at_ms: 0,
};
h.ctx.events.send(event(ProcessEventKind::Start)).unwrap();
h.ctx
.events
.send(BusEvent::LogOut {
id: 0,
line: "filtered".to_string(),
})
.unwrap();
h.ctx.events.send(event(ProcessEventKind::Online)).unwrap();
let first: ServerFrame = client.recv().await;
let second: ServerFrame = client.recv().await;
assert!(matches!(
first,
ServerFrame::Event(BusEvent::Process {
event: ProcessEventKind::Start,
..
})
));
assert!(matches!(
second,
ServerFrame::Event(BusEvent::Process {
event: ProcessEventKind::Online,
..
})
));
}
#[tokio::test]
async fn a_garbage_frame_ends_the_connection_without_panicking() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone());
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
})
.await;
let _: HelloReply = client.recv().await;
client
.frames
.send(bytes::Bytes::from_static(b"not json"))
.await
.unwrap();
assert!(client.closed().await);
}
#[tokio::test]
async fn a_garbage_frame_after_subscribing_still_closes_the_connection() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone());
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Subscribe {
topics: vec!["*".to_string()],
},
})
.await;
let _: ServerFrame = client.recv().await; client
.frames
.send(bytes::Bytes::from_static(b"not json"))
.await
.unwrap();
assert!(
client.closed().await,
"a live forwarder must not keep the connection open past a decode error"
);
}
#[tokio::test]
async fn peer_credentials_gate_on_uid() {
let (a, _b) = UnixStream::pair().unwrap();
let me = daemon_uid();
assert_eq!(check_peer(&a, me).unwrap(), me);
assert_eq!(
check_peer(&a, me + 1).unwrap_err(),
AuthError::ForeignUid {
peer: me,
daemon: me + 1
}
);
}
#[tokio::test]
async fn a_slow_subscriber_gets_a_dropped_notice_instead_of_hanging_the_bus() {
let h = harness(vec![]);
let mut client = connected(h.ctx.clone());
client
.send(&Hello {
client_version: "0.1.0".to_string(),
protocol: PROTOCOL_VERSION,
})
.await;
let _: HelloReply = client.recv().await;
client
.send(&Envelope {
id: 1,
deadline_ms: None,
body: Request::Subscribe {
topics: vec!["log.*".to_string()],
},
})
.await;
let _: ServerFrame = client.recv().await;
let flood = crate::bus::BUS_CAPACITY + CONN_QUEUE + 16;
for i in 0..flood {
h.ctx
.events
.send(BusEvent::LogOut {
id: 0,
line: format!("line-{i}"),
})
.unwrap();
}
let dropped = loop {
match client.recv::<ServerFrame>().await {
ServerFrame::Event(BusEvent::Dropped { count }) => break count,
ServerFrame::Event(_) => continue,
other => panic!("expected eventually a Dropped notice, got {other:?}"),
}
};
assert!(
dropped > 0,
"a flood past CONN_QUEUE + BUS_CAPACITY must report a real lag"
);
}
}