use std::collections::BTreeMap;
use std::io::Write;
use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use std::path::{Path, PathBuf};
use astrs_coordinator::{Coordinator, CoordinatorConfig, CoordinatorServer};
use astrs_daemon::config::{ListenConfig, RuntimePaths};
use astrs_daemon::{Daemon, DaemonConfig};
use astrs_manifest::OperatorConfig;
use astrs_node_api::Node;
use astrs_operator_api::OperatorRegistry;
use astrs_runtime::{RuntimeConfig, RuntimeHost};
use astrs_store::{AsyncStore, CoordinatorStore};
use astrs_wire::{AuthToken, MachineName, NodeSource};
use crate::command::client::resolve_text;
use crate::command::signals::Signals;
use crate::error::CliError;
use crate::runtime_dir::{PidFile, remove_pidfile, write_pidfile};
pub const STORE_FILE: &str = "coordinator.redb";
#[derive(Debug, Clone)]
pub struct ServeReport {
pub role: &'static str,
pub pid: u32,
pub address: String,
pub detail: String,
pub healthy: bool,
}
impl ServeReport {
#[must_use]
pub const fn exit_code(&self) -> i32 {
if self.healthy { 0 } else { 1 }
}
#[must_use]
pub fn to_json(&self) -> serde_json::Value {
serde_json::json!({
"role": self.role,
"pid": self.pid,
"address": self.address,
"detail": self.detail,
"healthy": self.healthy,
})
}
}
#[derive(Debug, Clone, Default)]
pub struct CoordinatorServeArgs {
pub port: u16,
pub bind: Option<String>,
pub token: Option<String>,
pub token_file: Option<PathBuf>,
pub working_dir: Option<PathBuf>,
pub store: Option<PathBuf>,
pub recreate_store: bool,
pub pidfile: Option<PathBuf>,
pub announce: bool,
pub ha_node_id: Option<u64>,
pub ha_peers: Vec<String>,
}
pub fn coordinator(
out: &mut dyn Write,
args: &CoordinatorServeArgs,
) -> Result<ServeReport, CliError> {
let addr = bind_addr(args.bind.as_deref(), args.port)?;
let working = crate::runtime_dir::working_dir(args.working_dir.as_deref());
let (token, token_note) =
resolve_token(args.token.as_deref(), args.token_file.as_deref(), &working)?;
let store = open_store(args.store.as_deref(), args.recreate_store)?;
let config = CoordinatorConfig::new(token).with_bind_addr(addr);
let hub = Coordinator::new(config, store);
install_trace_sink(&hub);
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|source| CliError::io("tokio runtime", source))?;
runtime.block_on(async move {
let mut signals = Signals::install();
let hub = start_high_availability(hub, args).await?;
let server = CoordinatorServer::bind(hub).await?;
let bound = server.local_addr()?;
let handle = server.handle();
let report = ServeReport {
role: "coordinator",
pid: std::process::id(),
address: bound.to_string(),
detail: token_note,
healthy: true,
};
if let Some(path) = &args.pidfile {
write_pidfile(path, &PidFile::listening(report.pid, bound.to_string()))?;
}
announce(out, args.announce, &report);
let mut serving = tokio::spawn(server.serve());
tokio::select! {
joined = &mut serving => finish_join(joined)?,
() = signals.next() => {
let _ = writeln!(out, "astrs coordinator: stopping on a signal");
let _ = out.flush();
handle.shutdown();
finish_join(serving.await)?;
}
}
if let Some(path) = &args.pidfile {
remove_pidfile(path)?;
}
Ok::<ServeReport, CliError>(report)
})
}
#[cfg(feature = "ha")]
async fn start_high_availability(
hub: Coordinator,
args: &CoordinatorServeArgs,
) -> Result<Coordinator, CliError> {
let Some(node_id) = args.ha_node_id else {
if !args.ha_peers.is_empty() {
return Err(CliError::BadArgument {
flag: "ha-peer",
value: args.ha_peers.join(", "),
reason: "--ha-peer needs --ha-node-id to say which of those peers this \
coordinator is"
.to_owned(),
});
}
return Ok(hub);
};
let mut config = astrs_coordinator::ha::HaConfig::parse(node_id, &args.ha_peers)?;
if let Some(store) = &args.store
&& let Some(directory) = store.parent()
&& !directory.as_os_str().is_empty()
{
config = config.with_log_dir(directory);
}
let handle = std::sync::Arc::new(astrs_coordinator::ha::HaHandle::start(&hub, config).await?);
Ok(hub.with_ha(handle))
}
#[cfg(not(feature = "ha"))]
async fn start_high_availability(
hub: Coordinator,
args: &CoordinatorServeArgs,
) -> Result<Coordinator, CliError> {
if args.ha_node_id.is_some() || !args.ha_peers.is_empty() {
return Err(CliError::BadArgument {
flag: "ha-node-id",
value: args
.ha_node_id
.map_or_else(|| args.ha_peers.join(", "), |id| id.to_string()),
reason: "this build has no coordinator high availability; rebuild the CLI with the \
`ha` feature"
.to_owned(),
});
}
Ok(hub)
}
fn finish_join(
joined: Result<astrs_coordinator::Result<()>, tokio::task::JoinError>,
) -> Result<(), CliError> {
match joined {
Ok(result) => Ok(result?),
Err(error) => Err(CliError::Cluster {
action: "run",
process: "coordinator",
reason: format!("the accept loop ended abnormally: {error}"),
}),
}
}
#[derive(Debug, Clone, Default)]
pub struct DaemonServeArgs {
pub coordinator: Option<String>,
pub machine: Option<String>,
pub port: Option<u16>,
pub peer_port: u16,
pub peer_bind: Option<String>,
pub labels: Vec<String>,
pub runtime_dir: Option<PathBuf>,
pub working_dir: Option<PathBuf>,
pub token: Option<String>,
pub token_file: Option<PathBuf>,
pub pidfile: Option<PathBuf>,
pub announce: bool,
}
pub fn daemon(out: &mut dyn Write, args: &DaemonServeArgs) -> Result<ServeReport, CliError> {
let runtime_dir = crate::runtime_dir::runtime_dir(args.runtime_dir.as_deref());
let working = crate::runtime_dir::working_dir(args.working_dir.as_deref());
let (token, token_note) =
resolve_token(args.token.as_deref(), args.token_file.as_deref(), &working)?;
let paths = RuntimePaths::under(&runtime_dir);
let mut listen = ListenConfig::uds(paths.socket_path());
if let Some(port) = args.port {
listen = listen.with_tcp_port(port);
}
let mut config = DaemonConfig::new(paths)
.with_listen(listen)
.with_auth(token.clone())
.with_working_dir(working);
let machine = machine_name(args.machine.as_deref())?;
if let Some(machine) = machine.clone() {
config = config.with_machine(machine);
}
let uplink = match &args.coordinator {
Some(address) => {
let address = astrs_daemon::coordinator::resolve_coordinator_addr(address)
.map_err(CliError::Daemon)?;
config = config.with_peer(peer_config(&token, args)?);
let mut uplink = astrs_daemon::coordinator::UplinkConfig::new(address, token);
if let Some(machine) = machine {
uplink = uplink.with_machine(machine);
}
uplink = uplink.with_labels(parse_labels(&args.labels)?);
Some(uplink)
}
None => None,
};
let runtime = tokio::runtime::Builder::new_multi_thread()
.enable_all()
.build()
.map_err(|source| CliError::io("tokio runtime", source))?;
runtime.block_on(async move {
let mut signals = Signals::install();
let mut daemon = Daemon::new(config)?;
daemon.bind().await?;
let endpoints = daemon.config().listen().endpoints().join(", ");
let mut detail = token_note;
if let Some(uplink) = uplink {
let coordinator = uplink.address();
daemon.connect_coordinator(uplink).await?;
let peer = daemon
.peers()
.listen_addr()
.map_or_else(|| "no peer listener".to_owned(), |addr| addr.to_string());
detail.push_str(&format!(
"; registering with the coordinator at {coordinator} (peers dial {peer})"
));
} else {
detail.push_str("; no coordinator — this daemon serves local nodes only");
}
let report = ServeReport {
role: "daemon",
pid: std::process::id(),
address: endpoints.clone(),
detail,
healthy: true,
};
if let Some(path) = &args.pidfile {
write_pidfile(path, &PidFile::listening(report.pid, endpoints))?;
}
announce(out, args.announce, &report);
let handle = daemon.handle();
tokio::spawn(async move {
signals.next().await;
handle.shutdown();
});
let results = daemon.run().await;
if let Some(path) = &args.pidfile {
remove_pidfile(path)?;
}
let finished = results.len();
let _ = writeln!(
out,
"astrs daemon: stopped after {finished} dataflow(s) finished here"
);
let _ = out.flush();
Ok::<ServeReport, CliError>(ServeReport {
detail: format!("{} ({finished} dataflow(s) finished)", report.detail),
..report
})
})
}
#[derive(Debug, Clone, Default)]
pub struct RuntimeServeArgs {
pub node_id: Option<String>,
}
pub fn runtime(out: &mut dyn Write, args: &RuntimeServeArgs) -> Result<ServeReport, CliError> {
let (node, events) = Node::init_from_env()?;
let assigned = node.descriptor().node.as_str().to_owned();
if let Some(expected) = &args.node_id
&& expected != &assigned
{
return Err(CliError::BadArgument {
flag: "node-id",
value: expected.clone(),
reason: format!("the daemon assigned this process the node `{assigned}`"),
});
}
let operators: Vec<OperatorConfig> = match &node.descriptor().source {
NodeSource::Runtime { operators } => operators
.iter()
.map(|spec| OperatorConfig {
id: spec.id.as_str().to_owned(),
operator: spec.registry_name.clone(),
dylib: None,
wasm: None,
hub: None,
inputs: std::collections::BTreeMap::new(),
outputs: Vec::new(),
config: std::collections::BTreeMap::new(),
})
.collect(),
_ => Vec::new(),
};
let hosted = operators.len();
let config = RuntimeConfig::new(operators, OperatorRegistry::new());
let host = RuntimeHost::new(node, events, config)?;
let report = host.run()?;
for operator in &report.operators {
let _ = writeln!(out, "{}: {:?}", operator.id, operator.outcome);
}
let _ = out.flush();
Ok(ServeReport {
role: "runtime",
pid: std::process::id(),
address: assigned,
detail: format!(
"{hosted} operator(s) hosted, {}",
if report.all_healthy() {
"all healthy"
} else {
"at least one failed"
}
),
healthy: report.all_healthy(),
})
}
fn bind_addr(bind: Option<&str>, port: u16) -> Result<SocketAddr, CliError> {
let Some(text) = bind else {
return Ok(SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), port));
};
let trimmed = text.trim();
if trimmed.contains(':') {
return resolve_text(trimmed);
}
let mut addr = resolve_text(trimmed)?;
addr.set_port(port);
Ok(addr)
}
fn resolve_token(
token: Option<&str>,
token_file: Option<&Path>,
working_dir: &Path,
) -> Result<(AuthToken, String), CliError> {
match crate::runtime_dir::find_token(token, token_file, working_dir)? {
Some((token, source)) => Ok((
token,
format!("authenticating with the token from {source}"),
)),
None => Ok((
AuthToken::ZERO,
format!(
"no cluster token found (looked for {}, then `{}`): accepting the all-zero token, \
so anyone who can reach this listener can drive it",
crate::runtime_dir::ENV_TOKEN,
crate::runtime_dir::token_path(working_dir).display()
),
)),
}
}
fn open_store(path: Option<&Path>, recreate: bool) -> Result<AsyncStore, CliError> {
let Some(path) = path else {
return Ok(AsyncStore::new(CoordinatorStore::open_in_memory()?));
};
if recreate {
match std::fs::remove_file(path) {
Ok(()) => {}
Err(error) if error.kind() == std::io::ErrorKind::NotFound => {}
Err(source) => return Err(CliError::io(path, source)),
}
}
if let Some(parent) = path.parent() {
crate::runtime_dir::ensure_dir(parent)?;
}
Ok(AsyncStore::new(CoordinatorStore::open(path)?))
}
fn install_trace_sink(hub: &Coordinator) {
let directives = std::env::var("RUST_LOG").unwrap_or_else(|_| "info".to_owned());
let config = astrs_telemetry::subscriber::TelemetryConfig {
service_name: "astrs-coordinator".to_owned(),
filter_directives: directives,
..astrs_telemetry::subscriber::TelemetryConfig::default()
};
let _ = astrs_telemetry::subscriber::init_telemetry_with_spans(config, hub.trace_sink());
}
fn machine_name(machine: Option<&str>) -> Result<Option<MachineName>, CliError> {
let Some(text) = machine else {
return Ok(None);
};
MachineName::new(text)
.map(Some)
.map_err(|error| CliError::BadArgument {
flag: "machine",
value: text.to_owned(),
reason: error.to_string(),
})
}
fn peer_config(
token: &AuthToken,
args: &DaemonServeArgs,
) -> Result<astrs_daemon::PeerConfig, CliError> {
let config = astrs_daemon::PeerConfig::new(token.clone());
let Some(bind) = args.peer_bind.as_deref() else {
return Ok(config.with_loopback(args.peer_port));
};
let addr = resolve_bind(bind, args.peer_port)?;
Ok(config.with_listen(addr))
}
fn resolve_bind(bind: &str, port: u16) -> Result<SocketAddr, CliError> {
use std::net::ToSocketAddrs as _;
let candidate = if bind.contains(':') {
bind.to_owned()
} else {
format!("{bind}:{port}")
};
candidate
.to_socket_addrs()
.ok()
.and_then(|mut addrs| addrs.next())
.ok_or_else(|| CliError::BadAddress {
input: bind.to_owned(),
reason: "no address resolves for it".to_owned(),
})
}
fn parse_labels(labels: &[String]) -> Result<BTreeMap<String, String>, CliError> {
let mut parsed = BTreeMap::new();
for label in labels {
let Some((key, value)) = label.split_once('=') else {
return Err(CliError::BadArgument {
flag: "label",
value: label.clone(),
reason: "a label is written KEY=VALUE".to_owned(),
});
};
if key.is_empty() {
return Err(CliError::BadArgument {
flag: "label",
value: label.clone(),
reason: "a label's key may not be empty".to_owned(),
});
}
parsed.insert(key.to_owned(), value.to_owned());
}
Ok(parsed)
}
fn announce(out: &mut dyn Write, enabled: bool, report: &ServeReport) {
if !enabled {
return;
}
let line = serde_json::to_string(&report.to_json())
.unwrap_or_else(|_| format!("{{\"role\":\"{}\"}}", report.role));
let _ = writeln!(out, "{line}");
let _ = out.flush();
}
#[cfg(test)]
mod tests {
#![allow(clippy::unwrap_used, clippy::expect_used, clippy::panic)]
use super::*;
fn scratch(name: &str) -> PathBuf {
let dir =
std::env::temp_dir().join(format!("astrs-cli-serve-{}-{name}", std::process::id()));
let _ = std::fs::remove_dir_all(&dir);
std::fs::create_dir_all(&dir).unwrap();
dir
}
#[test]
fn no_bind_argument_means_loopback_on_the_asked_for_port() {
let addr = bind_addr(None, 7407).unwrap();
assert_eq!(addr, SocketAddr::from(([127, 0, 0, 1], 7407)));
}
#[test]
fn an_interface_keeps_the_port_flag() {
let addr = bind_addr(Some("0.0.0.0"), 9000).unwrap();
assert_eq!(addr, SocketAddr::from(([0, 0, 0, 0], 9000)));
}
#[test]
fn a_full_host_and_port_wins_over_the_port_flag() {
let addr = bind_addr(Some("127.0.0.1:9100"), 7407).unwrap();
assert_eq!(addr.port(), 9100);
}
#[test]
fn an_unresolvable_interface_is_refused() {
let error = bind_addr(Some("no-such-host.invalid"), 1).unwrap_err();
assert!(matches!(error, CliError::BadAddress { .. }), "{error}");
}
#[test]
fn a_missing_token_falls_back_to_zero_and_says_so_loudly() {
let dir = scratch("no-token");
let (token, note) = resolve_token(None, None, &dir).unwrap();
assert!(token.is_zero());
assert!(note.contains("all-zero"), "{note}");
}
#[test]
fn a_token_file_is_used_and_named_in_the_note() {
let dir = scratch("token");
let (written, path) = crate::runtime_dir::ensure_token_file(&dir).unwrap();
let (token, note) = resolve_token(None, None, &dir).unwrap();
assert_eq!(token, written);
assert!(note.contains(&path.display().to_string()), "{note}");
}
#[test]
fn a_store_path_is_created_and_recreated_on_request() {
let dir = scratch("store");
let path = dir.join("nested").join(STORE_FILE);
let store = open_store(Some(&path), false).unwrap();
drop(store);
assert!(path.exists(), "the store file must exist after opening it");
let before = std::fs::metadata(&path).unwrap().len();
let store = open_store(Some(&path), true).unwrap();
drop(store);
assert!(path.exists());
let _ = before;
}
#[test]
fn no_store_path_means_an_in_memory_store() {
let store = open_store(None, false).unwrap();
drop(store);
}
#[test]
fn a_machine_name_is_validated() {
assert!(machine_name(None).unwrap().is_none());
assert_eq!(
machine_name(Some("robot-1"))
.unwrap()
.map(|m| m.to_string()),
Some("robot-1".to_owned())
);
let error = machine_name(Some("robot 1")).unwrap_err();
match error {
CliError::BadArgument { flag, .. } => assert_eq!(flag, "machine"),
other => panic!("expected BadArgument, got {other}"),
}
}
#[test]
fn the_announce_line_is_one_json_object_and_is_skipped_when_off() {
let report = ServeReport {
role: "coordinator",
pid: 42,
address: "127.0.0.1:7407".to_owned(),
detail: "note".to_owned(),
healthy: true,
};
let mut out = Vec::new();
announce(&mut out, false, &report);
assert!(out.is_empty());
announce(&mut out, true, &report);
let text = String::from_utf8(out).unwrap();
assert_eq!(text.lines().count(), 1, "{text}");
let value: serde_json::Value = serde_json::from_str(text.trim_end()).unwrap();
assert_eq!(value["role"], "coordinator");
assert_eq!(value["address"], "127.0.0.1:7407");
assert_eq!(value["pid"], 42);
}
#[test]
fn ha_peers_without_a_node_id_are_refused_rather_than_ignored() {
let args = CoordinatorServeArgs {
ha_peers: vec!["1=127.0.0.1:7601".to_owned()],
..CoordinatorServeArgs::default()
};
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let hub = astrs_coordinator::Coordinator::open_in_memory(
astrs_coordinator::CoordinatorConfig::new(astrs_wire::AuthToken::from_bytes([3; 32]))
.with_port(0),
)
.unwrap();
let outcome = runtime.block_on(start_high_availability(hub, &args));
assert!(
outcome.is_err(),
"an incomplete HA flag set must be refused"
);
}
#[test]
fn no_ha_flags_leaves_the_coordinator_exactly_as_it_was() {
let args = CoordinatorServeArgs::default();
let runtime = tokio::runtime::Builder::new_current_thread()
.enable_all()
.build()
.unwrap();
let hub = astrs_coordinator::Coordinator::open_in_memory(
astrs_coordinator::CoordinatorConfig::new(astrs_wire::AuthToken::from_bytes([3; 32]))
.with_port(0),
)
.unwrap();
assert!(
runtime
.block_on(start_high_availability(hub, &args))
.is_ok()
);
}
#[test]
fn a_coordinator_binds_announces_and_stops_on_a_signal() {
let dir = scratch("coordinator-lifecycle");
let pidfile = dir.join(crate::runtime_dir::COORDINATOR_PIDFILE);
let args = CoordinatorServeArgs {
port: 0,
working_dir: Some(dir.clone()),
store: Some(dir.join(STORE_FILE)),
pidfile: Some(pidfile.clone()),
announce: true,
..CoordinatorServeArgs::default()
};
let waiter = std::thread::spawn(move || {
let pid =
rustix::process::Pid::from_raw(std::process::id().cast_signed()).expect("a pid");
for _ in 0..200 {
if crate::runtime_dir::read_pidfile(&pidfile)
.and_then(|record| record.addr)
.is_some()
{
break;
}
std::thread::sleep(std::time::Duration::from_millis(25));
}
let _ = rustix::process::kill_process(pid, rustix::process::Signal::TERM);
});
let mut out = Vec::new();
let report = coordinator(&mut out, &args).expect("a coordinator");
waiter.join().expect("the signalling thread");
assert_eq!(report.role, "coordinator");
assert!(report.address.starts_with("127.0.0.1:"), "{report:?}");
assert_ne!(
report.address, "127.0.0.1:0",
"an ephemeral port must be reported as the one actually bound"
);
let text = String::from_utf8(out).unwrap();
assert!(text.contains("\"role\":\"coordinator\""), "{text}");
assert!(
!dir.join(crate::runtime_dir::COORDINATOR_PIDFILE).exists(),
"the pidfile must not outlive the process that wrote it"
);
}
#[test]
fn a_daemon_binds_its_socket_starts_its_uplink_and_stops() {
let dir = scratch("daemon-lifecycle");
let pidfile = dir.join(crate::runtime_dir::DAEMON_PIDFILE);
let args = DaemonServeArgs {
coordinator: Some("127.0.0.1:1".to_owned()),
peer_port: 0,
runtime_dir: Some(dir.clone()),
working_dir: Some(dir.clone()),
pidfile: Some(pidfile.clone()),
announce: true,
..DaemonServeArgs::default()
};
let waiter = std::thread::spawn(move || {
let pid =
rustix::process::Pid::from_raw(std::process::id().cast_signed()).expect("a pid");
for _ in 0..200 {
if crate::runtime_dir::read_pidfile(&pidfile).is_some() {
break;
}
std::thread::sleep(std::time::Duration::from_millis(25));
}
let _ = rustix::process::kill_process(pid, rustix::process::Signal::TERM);
});
let mut out = Vec::new();
let report = daemon(&mut out, &args).expect("a daemon");
waiter.join().expect("the signalling thread");
assert_eq!(report.role, "daemon");
assert!(report.address.contains("daemon.sock"), "{report:?}");
assert!(
report
.detail
.contains("registering with the coordinator at 127.0.0.1:1"),
"{report:?}"
);
assert!(
report.detail.contains("peers dial 127.0.0.1:"),
"{report:?}"
);
assert!(!dir.join(crate::runtime_dir::DAEMON_PIDFILE).exists());
}
#[test]
fn a_runtime_process_without_a_handshake_blob_reports_a_node_error() {
let error = runtime(&mut Vec::new(), &RuntimeServeArgs::default()).unwrap_err();
assert!(matches!(error, CliError::Node(_)), "{error}");
}
}