use subc_control::{
CapabilityRequirementStatus, ClientControlRequest, ClientControlResponse,
SupervisorDaemonProvenance, SupervisorHealthEntry, SupervisorModuleProvenance,
SupervisorRouteModule,
};
use tokio::time::Instant;
use super::{is_retryable_catalog_transport_error, CallError, SubcConsumer, TerminalFrame};
#[derive(Debug, Clone, PartialEq)]
pub struct SupervisorHealthResponse {
pub generation: u64,
pub modules: Vec<SupervisorHealthEntry>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct SupervisorProvenanceResponse {
pub daemon: SupervisorDaemonProvenance,
pub modules: Vec<SupervisorModuleProvenance>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct ServerDescription {
pub protocol_ver: u8,
pub subc_ops: Vec<String>,
pub capabilities: Vec<String>,
pub connected_clients: u64,
pub counters: Option<serde_json::Value>,
pub build_git_sha: Option<String>,
pub build_lock_digest: Option<String>,
pub capability_requirements: Vec<CapabilityRequirementStatus>,
pub machine_id: Option<String>,
}
#[derive(Debug, Clone, PartialEq)]
pub struct StderrTailResponse {
pub module_id: String,
pub tail: subc_control::StderrTail,
}
#[derive(Debug, Clone, PartialEq)]
pub struct HealthProbeResponse {
pub module_id: String,
pub status: subc_protocol::session::HealthStatus,
pub detail: Option<String>,
pub metrics: Option<serde_json::Value>,
}
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SupervisorAck {
pub module_id: String,
pub applied: bool,
}
impl SubcConsumer {
async fn control_request(
&self,
request: &ClientControlRequest,
op: &str,
retry: bool,
) -> Result<ClientControlResponse, CallError> {
let deadline = Instant::now() + self.shared.opts.call_timeout;
let body = serde_json::to_vec(request)
.map_err(|err| CallError::not_sent(format!("failed to encode {op}: {err}")))?;
let mut attempt = 0usize;
loop {
attempt = attempt.saturating_add(1);
match self
.shared
.control_call(body.clone(), deadline, false, None)
.await
{
Ok(TerminalFrame::Response { body, .. }) => {
return serde_json::from_slice::<ClientControlResponse>(&body).map_err(|err| {
CallError::not_sent(format!("failed to decode {op} response: {err}"))
});
}
Ok(TerminalFrame::Error { body, .. }) => return Err(CallError::Module(body)),
Ok(TerminalFrame::StreamEnd) => {
return Err(CallError::not_sent(format!("{op} returned StreamEnd")));
}
Err(err)
if retry
&& is_retryable_catalog_transport_error(&err)
&& Instant::now() < deadline =>
{
let delay = self.shared.jittered(
self.shared
.opts
.reconnect_backoff
.delay_after_attempt(attempt),
);
self.shared.sleep_until_retry(deadline, delay).await?;
}
Err(err) => return Err(err),
}
}
}
pub async fn supervisor_health(&self) -> Result<SupervisorHealthResponse, CallError> {
const OP: &str = "supervisor.health";
match self
.control_request(&ClientControlRequest::SupervisorHealth {}, OP, true)
.await?
{
ClientControlResponse::SupervisorHealth {
generation,
modules,
} => Ok(SupervisorHealthResponse {
generation,
modules,
}),
_ => Err(unexpected(OP)),
}
}
pub async fn supervisor_provenance(
&self,
module_id: Option<String>,
) -> Result<SupervisorProvenanceResponse, CallError> {
const OP: &str = "supervisor.provenance";
match self
.control_request(
&ClientControlRequest::SupervisorProvenance { module_id },
OP,
true,
)
.await?
{
ClientControlResponse::SupervisorProvenance { daemon, modules } => {
Ok(SupervisorProvenanceResponse { daemon, modules })
}
_ => Err(unexpected(OP)),
}
}
pub async fn supervisor_routes(
&self,
module_id: Option<String>,
) -> Result<Vec<SupervisorRouteModule>, CallError> {
const OP: &str = "supervisor.routes";
match self
.control_request(
&ClientControlRequest::SupervisorRoutes { module_id },
OP,
true,
)
.await?
{
ClientControlResponse::SupervisorRoutes { modules } => Ok(modules),
_ => Err(unexpected(OP)),
}
}
pub async fn supervisor_stderr_tail(
&self,
module_id: impl Into<String>,
max_lines: Option<u32>,
max_bytes: Option<u32>,
) -> Result<StderrTailResponse, CallError> {
const OP: &str = "supervisor.stderr_tail";
match self
.control_request(
&ClientControlRequest::SupervisorStderrTail {
module_id: module_id.into(),
max_lines,
max_bytes,
},
OP,
true,
)
.await?
{
ClientControlResponse::SupervisorStderrTail { module_id, tail } => {
Ok(StderrTailResponse { module_id, tail })
}
_ => Err(unexpected(OP)),
}
}
pub async fn supervisor_health_probe(
&self,
module_id: impl Into<String>,
) -> Result<HealthProbeResponse, CallError> {
const OP: &str = "supervisor.health_probe";
match self
.control_request(
&ClientControlRequest::SupervisorHealthProbe {
module_id: module_id.into(),
},
OP,
true,
)
.await?
{
ClientControlResponse::SupervisorHealthProbe {
module_id,
status,
detail,
metrics,
} => Ok(HealthProbeResponse {
module_id,
status,
detail,
metrics,
}),
_ => Err(unexpected(OP)),
}
}
pub async fn supervisor_terminals(
&self,
module_id: impl Into<String>,
) -> Result<subc_control::TerminalHistory, CallError> {
const OP: &str = "supervisor.terminals";
match self
.control_request(
&ClientControlRequest::SupervisorTerminals {
module_id: module_id.into(),
},
OP,
true,
)
.await?
{
ClientControlResponse::SupervisorTerminals { terminals, .. } => Ok(terminals),
_ => Err(unexpected(OP)),
}
}
pub async fn server_describe(&self) -> Result<ServerDescription, CallError> {
const OP: &str = "server.describe";
match self
.control_request(&ClientControlRequest::ServerDescribe {}, OP, true)
.await?
{
ClientControlResponse::ServerDescribe {
protocol_ver,
subc_ops,
capabilities,
connected_clients,
counters,
build_git_sha,
build_lock_digest,
capability_requirements,
machine_id,
} => Ok(ServerDescription {
protocol_ver,
subc_ops,
capabilities,
connected_clients,
counters,
build_git_sha,
build_lock_digest,
capability_requirements,
machine_id,
}),
_ => Err(unexpected(OP)),
}
}
pub async fn supervisor_restart(
&self,
module_id: impl Into<String>,
drain_timeout_ms: Option<u64>,
) -> Result<SupervisorAck, CallError> {
let request = ClientControlRequest::SupervisorRestart {
module_id: module_id.into(),
drain_timeout_ms,
};
self.lifecycle(&request, "supervisor.restart").await
}
pub async fn supervisor_set_enabled(
&self,
module_id: impl Into<String>,
enabled: bool,
) -> Result<SupervisorAck, CallError> {
let request = ClientControlRequest::SupervisorSetEnabled {
module_id: module_id.into(),
enabled,
};
self.lifecycle(&request, "supervisor.set_enabled").await
}
async fn lifecycle(
&self,
request: &ClientControlRequest,
op: &str,
) -> Result<SupervisorAck, CallError> {
match self.control_request(request, op, false).await? {
ClientControlResponse::SupervisorAck { module_id, applied } => {
Ok(SupervisorAck { module_id, applied })
}
_ => Err(unexpected(op)),
}
}
}
fn unexpected(op: &str) -> CallError {
CallError::not_sent(format!("{op} returned an unexpected control response"))
}
#[cfg(test)]
mod tests {
use std::path::PathBuf;
use std::sync::Arc;
use subc_control::{SupervisorHealthStatus, SupervisorRouteModule};
use subc_protocol::{ErrorBody, Flags, Frame, FrameType, Priority};
use tokio::sync::mpsc;
use super::super::{dispatch_frame, ConsumerOptions, Shared};
use super::*;
fn consumer() -> (
SubcConsumer,
Arc<Shared>,
mpsc::Receiver<super::super::WriteCommand>,
) {
let shared = Arc::new(Shared::new(
PathBuf::from("/tmp/does-not-exist"),
ConsumerOptions::default(),
));
let (writer, rx) = mpsc::channel(1);
shared.lock_inner().writer = Some(writer);
(
SubcConsumer {
shared: Arc::clone(&shared),
},
shared,
rx,
)
}
fn frame(kind: FrameType, corr: u64, body: Vec<u8>) -> Frame {
Frame::build(
kind,
Flags::new(false, Priority::Interactive, false),
0,
0,
corr,
body,
)
.unwrap()
}
#[tokio::test]
async fn health_sends_its_op_and_returns_the_entries() {
let (consumer, shared, mut rx) = consumer();
let call = tokio::spawn(async move { consumer.supervisor_health().await });
let command = rx.recv().await.expect("a channel-0 request");
assert_eq!(command.frame.header.channel, 0);
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "supervisor.health");
let reply = serde_json::json!({
"op": "supervisor.health",
"generation": 7,
"modules": [{
"module_id": "broca",
"status": "degraded",
"detail": "provider throttled",
"metrics": {"headline": ["runs"], "runs": 3},
"consecutive_failures": 0
}]
});
let corr = command.frame.header.corr;
let reply = serde_json::to_vec(&reply).unwrap();
assert!(dispatch_frame(&shared, 1, frame(FrameType::Response, corr, reply)).await);
let health = call.await.unwrap().unwrap();
assert_eq!(health.generation, 7);
assert_eq!(health.modules[0].status, SupervisorHealthStatus::Degraded);
assert_eq!(
health.modules[0].detail.as_deref(),
Some("provider throttled")
);
}
#[tokio::test]
async fn routes_and_describe_decode_their_replies() {
let (consumer, shared, mut rx) = consumer();
let consumer = Arc::new(consumer);
let c = Arc::clone(&consumer);
let call = tokio::spawn(async move { c.supervisor_routes(Some("broca".into())).await });
let command = rx.recv().await.unwrap();
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "supervisor.routes");
assert_eq!(body["module_id"], "broca");
let reply = serde_json::to_vec(&ClientControlResponse::SupervisorRoutes {
modules: vec![SupervisorRouteModule {
module_id: "broca".into(),
routes: vec![],
}],
})
.unwrap();
let corr = command.frame.header.corr;
assert!(dispatch_frame(&shared, 1, frame(FrameType::Response, corr, reply)).await);
assert_eq!(call.await.unwrap().unwrap()[0].module_id, "broca");
let c = Arc::clone(&consumer);
let call = tokio::spawn(async move { c.server_describe().await });
let command = rx.recv().await.unwrap();
let reply = serde_json::json!({
"op": "server.describe",
"protocol_ver": 1,
"subc_ops": [],
"capabilities": [],
"connected_clients": 12,
"machine_id": "4f1a9c2e7b3d40c8"
});
let corr = command.frame.header.corr;
let reply = serde_json::to_vec(&reply).unwrap();
assert!(dispatch_frame(&shared, 1, frame(FrameType::Response, corr, reply)).await);
let described = call.await.unwrap().unwrap();
assert_eq!(described.connected_clients, 12);
assert_eq!(described.machine_id.as_deref(), Some("4f1a9c2e7b3d40c8"));
}
#[tokio::test]
async fn restart_returns_the_ack_and_a_refusal_keeps_its_code() {
let (consumer, shared, mut rx) = consumer();
let consumer = Arc::new(consumer);
let c = Arc::clone(&consumer);
let call = tokio::spawn(async move { c.supervisor_restart("broca", None).await });
let command = rx.recv().await.unwrap();
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "supervisor.restart");
assert_eq!(body["module_id"], "broca");
assert!(body.get("drain_timeout_ms").is_none());
let reply = serde_json::to_vec(&ClientControlResponse::SupervisorAck {
module_id: "broca".into(),
applied: true,
})
.unwrap();
let corr = command.frame.header.corr;
assert!(dispatch_frame(&shared, 1, frame(FrameType::Response, corr, reply)).await);
assert_eq!(
call.await.unwrap().unwrap(),
SupervisorAck {
module_id: "broca".into(),
applied: true
}
);
let c = Arc::clone(&consumer);
let call = tokio::spawn(async move { c.supervisor_set_enabled("broca", false).await });
let command = rx.recv().await.unwrap();
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "supervisor.set_enabled");
assert_eq!(body["enabled"], false);
let refusal = serde_json::to_vec(&ErrorBody {
code: "module_reserved".into(),
message: "refused by the test".into(),
detail: None,
})
.unwrap();
let corr = command.frame.header.corr;
assert!(dispatch_frame(&shared, 1, frame(FrameType::Error, corr, refusal)).await);
assert_eq!(
call.await.unwrap().unwrap_err().code(),
Some("module_reserved")
);
}
#[tokio::test]
async fn health_probe_and_terminals_decode_their_replies() {
let (consumer, shared, mut rx) = consumer();
let consumer = Arc::new(consumer);
let c = Arc::clone(&consumer);
let call = tokio::spawn(async move { c.supervisor_health_probe("aft").await });
let command = rx.recv().await.unwrap();
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "supervisor.health_probe");
assert_eq!(body["module_id"], "aft");
let reply = serde_json::json!({
"op": "supervisor.health_probe", "module_id": "aft", "status": "ok",
"metrics": {"headline": ["files"], "files": 12}
});
let corr = command.frame.header.corr;
let reply = serde_json::to_vec(&reply).unwrap();
assert!(dispatch_frame(&shared, 1, frame(FrameType::Response, corr, reply)).await);
let probe = call.await.unwrap().unwrap();
assert_eq!(probe.metrics.unwrap()["files"], 12);
let c = Arc::clone(&consumer);
let call = tokio::spawn(async move { c.supervisor_terminals("aft").await });
let command = rx.recv().await.unwrap();
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "supervisor.terminals");
let reply = serde_json::json!({
"op": "supervisor.terminals", "module_id": "aft",
"daemon_started_at_ms": 1000,
"entries": [{"exit_code": 0, "at_ms": 2000, "disposition": "restarting"}]
});
let corr = command.frame.header.corr;
let reply = serde_json::to_vec(&reply).unwrap();
assert!(dispatch_frame(&shared, 1, frame(FrameType::Response, corr, reply)).await);
let history = call.await.unwrap().unwrap();
assert_eq!(history.entries.len(), 1);
assert_eq!(history.entries[0].at_ms, 2000);
}
#[tokio::test]
async fn stderr_tail_carries_its_caps_and_decodes_lines() {
let (consumer, shared, mut rx) = consumer();
let call = tokio::spawn(async move {
consumer
.supervisor_stderr_tail("broca", Some(200), None)
.await
});
let command = rx.recv().await.unwrap();
let body: serde_json::Value = serde_json::from_slice(&command.frame.body).unwrap();
assert_eq!(body["op"], "supervisor.stderr_tail");
assert_eq!(body["max_lines"], 200);
let reply = serde_json::json!({
"op": "supervisor.stderr_tail",
"module_id": "broca",
"capture": {"state": "captured"},
"entries": [
{"kind": "line", "text": "hello", "truncated": false, "at_ms": 1000}
]
});
let corr = command.frame.header.corr;
let reply = serde_json::to_vec(&reply).unwrap();
assert!(dispatch_frame(&shared, 1, frame(FrameType::Response, corr, reply)).await);
let tail = call.await.unwrap().unwrap();
assert_eq!(tail.module_id, "broca");
assert_eq!(tail.tail.entries.len(), 1);
}
}