use core::fmt;
use std::path::{Path, PathBuf};
use std::time::Duration;
use tokio::sync::{mpsc, oneshot};
use shep_core::destination::Destination;
use shep_core::protocol::{HelloAck, Request, Response, RpcError, RpcErrorCode, WireError};
use crate::actor::{self, Command};
use crate::connection::{ConnectError, Connection, HANDSHAKE_TIMEOUT};
use crate::events::EventStream;
use crate::ssh::{self, SshOptions};
pub const DEFAULT_DEADLINE: Duration = Duration::from_secs(5);
pub const START_DEADLINE: Duration = Duration::from_secs(30);
pub const RELOAD_DEADLINE: Duration = Duration::from_secs(60);
pub const LOG_PLANE_DEADLINE: Duration = Duration::from_secs(30);
pub const TRIGGER_DEADLINE: Duration = Duration::from_secs(60);
pub const STOP_DEADLINE: Duration = Duration::from_secs(60);
pub const DEADLINE_GRACE: Duration = Duration::from_secs(2);
#[derive(Debug, Clone, PartialEq, Eq)]
#[non_exhaustive]
pub enum RequestError {
Rpc(RpcError),
Timeout {
after: Duration,
},
Closed,
Wire(WireError),
Undecodable(WireError),
UnexpectedReply {
asked: &'static str,
answered: &'static str,
},
}
impl RequestError {
#[must_use]
pub const fn exit_code(&self) -> u8 {
use shep_core::exit;
match self {
Self::Rpc(rpc) => rpc.code.exit_code(),
Self::Timeout { .. } => exit::DEADLINE_EXCEEDED,
Self::Closed => exit::DAEMON_UNREACHABLE,
Self::Wire(_) | Self::UnexpectedReply { .. } => exit::INTERNAL,
Self::Undecodable(_) => exit::FAILURE,
}
}
}
impl fmt::Display for RequestError {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Rpc(err) => write!(f, "the daemon reported {:?}: {}", err.code, err.message),
Self::Timeout { after } => write!(f, "no reply within {after:?}"),
Self::Closed => f.write_str("the connection closed before a reply arrived"),
Self::Wire(err) => write!(f, "request frame error: {err}"),
Self::Undecodable(err) => write!(
f,
"this client could not decode the daemon's reply ({err}); the daemon is likely newer than this build"
),
Self::UnexpectedReply { asked, answered } => {
write!(f, "the daemon answered {asked} with {answered}")
}
}
}
}
impl core::error::Error for RequestError {
fn source(&self) -> Option<&(dyn core::error::Error + 'static)> {
match self {
Self::Wire(err) | Self::Undecodable(err) => Some(err),
Self::Rpc(_) | Self::Timeout { .. } | Self::Closed | Self::UnexpectedReply { .. } => {
None
}
}
}
}
#[derive(Debug, Clone)]
pub(crate) enum Dial {
Socket(PathBuf),
Ssh {
destination: Destination,
options: SshOptions,
},
}
pub struct Client {
commands: mpsc::Sender<Command>,
ack: HelloAck,
dial: Dial,
dog_name: Option<String>,
}
impl fmt::Debug for Client {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
f.debug_struct("Client")
.field("socket", &self.socket())
.field("host", &self.host())
.field("ack", &self.ack)
.field("dog_name", &self.dog_name)
.finish_non_exhaustive()
}
}
impl Client {
pub async fn connect(socket: &Path) -> Result<Self, ConnectError> {
Self::connect_with_timeout(socket, HANDSHAKE_TIMEOUT).await
}
pub async fn connect_with_timeout(
socket: &Path,
timeout: Duration,
) -> Result<Self, ConnectError> {
Self::connect_as(socket, timeout, None).await
}
pub(crate) async fn connect_as(
socket: &Path,
timeout: Duration,
dog_name: Option<&str>,
) -> Result<Self, ConnectError> {
let connection = Connection::open(socket, timeout, dog_name).await?;
Ok(Self::from_connection(
connection,
Dial::Socket(socket.to_path_buf()),
dog_name,
))
}
pub(crate) async fn connect_over(
stream: shep_core::transport::ClientStream,
socket: &Path,
timeout: Duration,
dog_name: Option<&str>,
) -> Result<Self, ConnectError> {
let connection = tokio::time::timeout(timeout, Connection::open_over(stream, dog_name))
.await
.map_err(|_elapsed| ConnectError::HandshakeTimeout { after: timeout })??;
Ok(Self::from_connection(
connection,
Dial::Socket(socket.to_path_buf()),
dog_name,
))
}
fn from_connection(connection: Connection, dial: Dial, dog_name: Option<&str>) -> Self {
let (frames, ack) = connection.into_parts();
let commands = actor::spawn(frames);
Self {
commands,
ack,
dial,
dog_name: dog_name.map(str::to_owned),
}
}
pub async fn connect_ssh(dest: &str) -> Result<Self, ConnectError> {
Self::connect_ssh_with(dest, &SshOptions::default()).await
}
pub async fn connect_ssh_with(dest: &str, options: &SshOptions) -> Result<Self, ConnectError> {
let destination = Destination::parse(dest).map_err(ConnectError::Destination)?;
let dial = Dial::Ssh {
destination,
options: options.clone(),
};
Self::connect_dial(&dial, options.timeout_or(None), None).await
}
pub(crate) async fn connect_dial(
dial: &Dial,
budget: Option<Duration>,
dog_name: Option<&str>,
) -> Result<Self, ConnectError> {
match dial {
Dial::Socket(socket) => {
Self::connect_as(socket, budget.unwrap_or(HANDSHAKE_TIMEOUT), dog_name).await
}
Dial::Ssh {
destination,
options,
} => {
let connection =
ssh::open(destination, options, options.timeout_or(budget), dog_name).await?;
Ok(Self::from_connection(connection, dial.clone(), dog_name))
}
}
}
pub(crate) fn dial(&self) -> &Dial {
&self.dial
}
pub(crate) fn dog_name(&self) -> Option<&str> {
self.dog_name.as_deref()
}
pub(crate) fn replace_connection(&mut self, fresh: Self) {
*self = fresh;
}
#[must_use]
pub fn daemon(&self) -> &HelloAck {
&self.ack
}
pub async fn closed(&self) {
self.commands.closed().await;
}
#[must_use]
pub fn socket(&self) -> &Path {
match &self.dial {
Dial::Socket(socket) => socket,
Dial::Ssh { .. } => Path::new(""),
}
}
#[must_use]
pub fn host(&self) -> Option<&str> {
match &self.dial {
Dial::Socket(_) => None,
Dial::Ssh { destination, .. } => Some(destination.as_str()),
}
}
async fn send_command(&self, command: Command) -> Result<(), RequestError> {
self.commands
.send(command)
.await
.map_err(|_send_error| RequestError::Closed)
}
pub async fn request(&self, body: Request) -> Result<Response, RequestError> {
self.request_with_deadline(body, None).await
}
pub async fn request_with_deadline(
&self,
body: Request,
deadline: Option<Duration>,
) -> Result<Response, RequestError> {
if let Some(gated) = body.gated_key_dropped_at(self.ack.protocol) {
return Err(RequestError::Rpc(RpcError {
code: RpcErrorCode::Unsupported,
message: format!(
"this shepherd speaks protocol {} and `{}` needs protocol {}, so it would not \
run the sheep as configured. Restart the shepherd and run this again; \
nothing was sent",
self.ack.protocol, gated.key, gated.since
),
daemon_version: None,
}));
}
let deadline = deadline.unwrap_or(DEFAULT_DEADLINE);
let (reply_to, reply_rx) = oneshot::channel();
self.send_command(Command::Request {
body,
deadline_ms: Some(millis(deadline)),
reply_to,
})
.await?;
let budget = deadline.saturating_add(DEADLINE_GRACE);
match tokio::time::timeout(budget, reply_rx).await {
Ok(Ok(outcome)) => outcome,
Ok(Err(_recv_error)) => Err(RequestError::Closed),
Err(_elapsed) => Err(RequestError::Timeout { after: budget }),
}
}
pub async fn subscribe(&self, topics: Vec<String>) -> Result<EventStream, RequestError> {
let events = self.events().await?;
self.request(Request::Subscribe { topics }).await?;
Ok(events)
}
pub(crate) async fn events(&self) -> Result<EventStream, RequestError> {
let (reply_to, reply_rx) = oneshot::channel();
self.send_command(Command::Subscribe { reply_to }).await?;
let receiver = reply_rx.await.map_err(|_recv_error| RequestError::Closed)?;
Ok(EventStream::new(receiver))
}
pub async fn close(self) -> Result<(), RequestError> {
drop(self.commands);
Ok(())
}
}
fn millis(d: Duration) -> u64 {
u64::try_from(d.as_millis()).unwrap_or(u64::MAX)
}
#[cfg(test)]
mod tests {
use shep_core::config::MemoryEnforce;
use shep_core::exit;
use shep_core::protocol::{
LISTEN_SINCE, MAX_CPU_CORES_SINCE, MAX_TASKS_SINCE, MEMORY_ENFORCE_SINCE, RpcErrorCode,
SANDBOX_DOG_SINCE,
};
use super::*;
#[test]
fn a_request_error_exits_on_its_causes_code() {
let unsupported = RequestError::Rpc(RpcError {
code: RpcErrorCode::Unsupported,
message: "no such verb".to_owned(),
daemon_version: None,
});
let cases = [
(unsupported, exit::UNSUPPORTED),
(RequestError::Closed, exit::DAEMON_UNREACHABLE),
(
RequestError::Timeout {
after: Duration::from_secs(7),
},
exit::DEADLINE_EXCEEDED,
),
(
RequestError::UnexpectedReply {
asked: "ListFlock",
answered: "Pong",
},
exit::INTERNAL,
),
];
for (err, code) in cases {
assert_eq!(err.exit_code(), code, "{err}");
}
}
fn sandboxed_start() -> Request {
let mut app = shep_core::config::AppConfig::minimal("web", "./srv");
app.dogs.insert(
"sandbox".to_owned(),
serde_json::from_value(serde_json::json!({"profile": "strict"})).unwrap(),
);
Request::Start { apps: vec![app] }
}
#[tokio::test]
async fn a_sandbox_dog_table_is_not_sent_to_a_shepherd_below_protocol_15() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: SANDBOX_DOG_SINCE - 1,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let refused = client.request(sandboxed_start()).await.unwrap_err();
let RequestError::Rpc(rpc) = &refused else {
panic!("expected a refusal, got {refused:?}");
};
assert_eq!(rpc.code, RpcErrorCode::Unsupported);
assert!(rpc.message.contains("`dogs.sandbox`"), "{}", rpc.message);
assert!(rpc.message.contains("nothing was sent"), "{}", rpc.message);
assert_eq!(refused.exit_code(), exit::UNSUPPORTED);
assert!(sent.try_recv().is_err(), "the request reached the wire");
}
fn listening_start() -> Request {
let mut app = shep_core::config::AppConfig::minimal("web", "./srv");
app.listen
.insert("http".to_owned(), "0.0.0.0:8080".to_owned());
Request::Start { apps: vec![app] }
}
#[tokio::test]
async fn a_listen_is_refused_before_it_reaches_an_older_shepherd() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: LISTEN_SINCE - 1,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let refused = client.request(listening_start()).await.unwrap_err();
let RequestError::Rpc(rpc) = &refused else {
panic!("expected a refusal, got {refused:?}");
};
assert_eq!(rpc.code, RpcErrorCode::Unsupported);
assert!(rpc.message.contains("`listen`"), "{}", rpc.message);
assert!(rpc.message.contains("nothing was sent"), "{}", rpc.message);
assert_eq!(refused.exit_code(), exit::UNSUPPORTED);
assert!(sent.try_recv().is_err(), "the request reached the wire");
}
#[tokio::test]
async fn a_listen_is_sent_to_a_shepherd_that_binds_one() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: LISTEN_SINCE,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let _ = client.request(listening_start()).await;
assert!(
sent.try_recv().is_ok(),
"the request never reached the wire"
);
}
#[tokio::test]
async fn a_sandbox_dog_table_is_sent_to_a_shepherd_at_protocol_15() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: SANDBOX_DOG_SINCE,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let _ = client.request(sandboxed_start()).await;
assert!(
sent.try_recv().is_ok(),
"the request never reached the wire"
);
}
fn kernel_enforced_start() -> Request {
let mut app = shep_core::config::AppConfig::minimal("web", "./srv");
app.max_memory = Some("512M".parse().unwrap());
app.memory_enforce = MemoryEnforce::Kernel;
Request::Start { apps: vec![app] }
}
#[tokio::test]
async fn a_memory_enforce_key_is_not_sent_to_an_older_shepherd() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: MEMORY_ENFORCE_SINCE - 1,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let refused = client.request(kernel_enforced_start()).await.unwrap_err();
let RequestError::Rpc(rpc) = &refused else {
panic!("expected a refusal, got {refused:?}");
};
assert_eq!(rpc.code, RpcErrorCode::Unsupported);
assert!(rpc.message.contains("`memory_enforce`"), "{}", rpc.message);
assert!(rpc.message.contains("nothing was sent"), "{}", rpc.message);
assert!(sent.try_recv().is_err(), "the request reached the wire");
}
#[tokio::test]
async fn a_memory_enforce_key_is_sent_to_a_shepherd_that_reads_it() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: MEMORY_ENFORCE_SINCE,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let _ = client.request(kernel_enforced_start()).await;
assert!(
sent.try_recv().is_ok(),
"the request never reached the wire"
);
}
fn cpu_capped_start() -> Request {
let mut app = shep_core::config::AppConfig::minimal("web", "./srv");
app.max_cpu_cores = Some(shep_core::config::CpuCores::from_cores(1.5));
Request::Start { apps: vec![app] }
}
#[tokio::test]
async fn a_max_cpu_cores_key_is_not_sent_to_an_older_shepherd() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: MAX_CPU_CORES_SINCE - 1,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let refused = client.request(cpu_capped_start()).await.unwrap_err();
let RequestError::Rpc(rpc) = &refused else {
panic!("expected a refusal, got {refused:?}");
};
assert_eq!(rpc.code, RpcErrorCode::Unsupported);
assert!(rpc.message.contains("`max_cpu_cores`"), "{}", rpc.message);
assert!(rpc.message.contains("nothing was sent"), "{}", rpc.message);
assert!(sent.try_recv().is_err(), "the request reached the wire");
}
#[tokio::test]
async fn a_max_cpu_cores_key_is_sent_to_a_shepherd_that_reads_it() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: MAX_CPU_CORES_SINCE,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let _ = client.request(cpu_capped_start()).await;
assert!(
sent.try_recv().is_ok(),
"the request never reached the wire"
);
}
fn task_capped_start() -> Request {
let mut app = shep_core::config::AppConfig::minimal("web", "./srv");
app.max_tasks = Some(256);
Request::Start { apps: vec![app] }
}
#[tokio::test]
async fn a_max_tasks_key_is_not_sent_to_an_older_shepherd() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: MAX_TASKS_SINCE - 1,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let refused = client.request(task_capped_start()).await.unwrap_err();
let RequestError::Rpc(rpc) = &refused else {
panic!("expected a refusal, got {refused:?}");
};
assert_eq!(rpc.code, RpcErrorCode::Unsupported);
assert!(rpc.message.contains("`max_tasks`"), "{}", rpc.message);
assert!(rpc.message.contains("nothing was sent"), "{}", rpc.message);
assert!(sent.try_recv().is_err(), "the request reached the wire");
}
#[tokio::test]
async fn a_max_tasks_key_is_sent_to_a_shepherd_that_reads_it() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let ack = HelloAck {
protocol: MAX_TASKS_SINCE,
..crate::testing::sample_ack()
};
let (client, mut sent) =
crate::testing::fake_client_capturing_envelopes_with_ack(&path, ack).await;
let _ = client.request(task_capped_start()).await;
assert!(
sent.try_recv().is_ok(),
"the request never reached the wire"
);
}
#[cfg(unix)]
#[tokio::test]
async fn an_ssh_client_has_a_host_and_no_socket() {
let dir = tempfile::tempdir().unwrap();
crate::ssh::stand_in::write_reply(dir.path());
let program = crate::ssh::stand_in::stand_in(
dir.path(),
"cat \"$(dirname \"$0\")/reply.bin\"\ncat >/dev/null",
);
let client =
Client::connect_ssh_with("deploy@build-01", &SshOptions::default().program(program))
.await
.unwrap();
assert_eq!(client.host(), Some("deploy@build-01"));
assert_eq!(client.socket(), Path::new(""));
let shown = format!("{client:?}");
assert!(shown.contains("host: Some(\"deploy@build-01\")"), "{shown}");
}
#[tokio::test]
async fn a_socket_client_has_a_socket_and_no_host() {
let dir = tempfile::tempdir().unwrap();
let path = crate::testing::control_address(dir.path());
let (client, _daemon) = crate::testing::fake_client_on(&path).await;
assert_eq!(client.host(), None);
assert_eq!(client.socket(), path);
}
}