mod auth;
mod autostart;
mod gate_registry;
mod locks;
mod middleware;
mod routes;
mod state;
use std::env;
use std::io::Write;
use std::net::SocketAddr;
use std::os::unix::process::CommandExt;
use std::path::PathBuf;
use std::process::{exit, Command};
use std::sync::Arc;
use std::time::Duration;
use anyhow::Context;
use clap::Parser;
use tokio::sync::Semaphore;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};
use crate::routes::RouterConfig;
use crate::state::{AppState, SweepStatus};
const SONDA_SUBCOMMANDS: &[&str] = &["run", "list", "show", "new", "test", "completions"];
#[derive(Parser)]
#[command(name = "sonda-server", version, about = "HTTP control plane for Sonda")]
struct Args {
#[arg(long, default_value_t = 8080)]
port: u16,
#[arg(long, default_value = "0.0.0.0")]
bind: String,
#[arg(long, env = "SONDA_API_KEY")]
api_key: Option<String>,
#[arg(long, env = "SONDA_CATALOG")]
catalog: Option<PathBuf>,
#[arg(
long,
env = "SONDA_AUTOSTART",
action = clap::ArgAction::SetTrue,
value_parser = clap::builder::FalseyValueParser::new()
)]
autostart: bool,
#[arg(long, value_parser = clap::builder::RangedU64ValueParser::<u64>::new().range(1..))]
workers: Option<u64>,
#[arg(long, default_value_t = 0)]
max_scenarios: usize,
#[arg(long)]
max_inflight_requests: Option<usize>,
#[arg(long, default_value_t = 30,
value_parser = clap::builder::RangedU64ValueParser::<u64>::new().range(1..))]
request_timeout: u64,
#[arg(long, default_value_t = 1_048_576)]
max_body_bytes: usize,
}
impl std::fmt::Debug for Args {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("Args")
.field("port", &self.port)
.field("bind", &self.bind)
.field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
.field("catalog", &self.catalog)
.field("autostart", &self.autostart)
.field("workers", &self.workers)
.field("max_scenarios", &self.max_scenarios)
.field("max_inflight_requests", &self.max_inflight_requests)
.field("request_timeout", &self.request_timeout)
.field("max_body_bytes", &self.max_body_bytes)
.finish()
}
}
fn main() -> anyhow::Result<()> {
maybe_dispatch_to_sonda_cli();
let args = Args::parse();
let workers = args.workers.map(|n| n as usize).unwrap_or_else(|| {
std::thread::available_parallelism()
.map(|n| n.get())
.unwrap_or(1)
.min(16)
});
let max_inflight_requests = args.max_inflight_requests.unwrap_or(workers * 4);
let runtime = tokio::runtime::Builder::new_multi_thread()
.worker_threads(workers)
.enable_all()
.build()
.context("failed to build tokio runtime")?;
runtime.block_on(async move { run(args, workers, max_inflight_requests).await })
}
async fn run(args: Args, workers: usize, max_inflight_requests: usize) -> anyhow::Result<()> {
tracing_subscriber::fmt()
.with_env_filter(
tracing_subscriber::EnvFilter::try_from_default_env()
.unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
)
.with_writer(std::io::stderr)
.init();
let bind_addr: SocketAddr = format!("{}:{}", args.bind, args.port)
.parse()
.with_context(|| format!("invalid bind address: {}:{}", args.bind, args.port))?;
let api_key = args.api_key.filter(|k| {
if k.is_empty() {
warn!("--api-key / SONDA_API_KEY is empty — authentication disabled");
false
} else {
true
}
});
if api_key.is_some() {
info!(
"API key authentication enabled for /scenarios/*, /events, /metrics, and /scenarios/metrics endpoints"
);
} else {
info!("API key authentication disabled — all endpoints are public");
}
if args.autostart && args.catalog.is_none() {
anyhow::bail!(
"--autostart requires --catalog <DIR> (or SONDA_AUTOSTART requires SONDA_CATALOG): \
there is no scenario catalog to start from"
);
}
if let Some(dir) = &args.catalog {
if !dir.is_dir() {
anyhow::bail!(
"--catalog {}: does not exist or is not a directory",
dir.display()
);
}
info!(catalog = %dir.display(), "pack catalog enabled for POST /scenarios");
}
let autostart_entries = match args.catalog.as_deref() {
Some(dir) if args.autostart => autostart::runnable_entries(dir)?,
_ => Vec::new(),
};
let sweep_status = Arc::new(if args.autostart {
SweepStatus::in_progress(autostart_entries.len())
} else {
SweepStatus::not_configured()
});
let permits = if args.max_scenarios == 0 {
warn!("--max-scenarios 0 — scenario row cap disabled (unlimited)");
Semaphore::new(Semaphore::MAX_PERMITS)
} else {
Semaphore::new(args.max_scenarios)
};
let state = AppState {
api_key: api_key.map(Arc::new),
catalog_dir: args.catalog.clone().map(Arc::new),
scenario_permits: Arc::new(permits),
worker_threads: workers,
max_scenarios: args.max_scenarios,
sweep_status,
..AppState::new()
};
let inflight_semaphore = Arc::new(Semaphore::new(max_inflight_requests));
let router_cfg = RouterConfig {
request_timeout: Duration::from_secs(args.request_timeout),
max_body_bytes: args.max_body_bytes,
inflight_semaphore,
};
let app = routes::router_with_config(state.clone(), router_cfg);
let listener = tokio::net::TcpListener::bind(bind_addr)
.await
.with_context(|| format!("failed to bind to {bind_addr}"))?;
let bound_addr = listener
.local_addr()
.context("failed to read local address from bound listener")?;
#[cfg(unix)]
let sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
.context("failed to install SIGTERM handler")?;
announce_bound_port(bound_addr.port())?;
info!(addr = %bound_addr, workers, "sonda-server listening");
let sweep_cancel = CancellationToken::new();
let sweep = args.autostart.then(|| {
let sweep_state = state.clone();
let cancel = sweep_cancel.clone();
spawn_supervised_sweep(
async move { autostart::start_entries(&sweep_state, &autostart_entries, &cancel).await },
state.sweep_status.clone(),
)
});
axum::serve(listener, app)
.with_graceful_shutdown(shutdown_signal(
state,
sweep,
sweep_cancel,
#[cfg(unix)]
sigterm,
))
.await
.context("server error")?;
info!("sonda-server shut down cleanly");
Ok(())
}
fn spawn_supervised_sweep<F>(sweep: F, status: Arc<SweepStatus>) -> tokio::task::JoinHandle<()>
where
F: std::future::Future<Output = ()> + Send + 'static,
{
let sweeping = tokio::spawn(sweep);
tokio::spawn(report_sweep_outcome(sweeping, status))
}
async fn report_sweep_outcome(sweeping: tokio::task::JoinHandle<()>, status: Arc<SweepStatus>) {
if let Err(error) = sweeping.await {
status.fail();
error!(%error, "autostart: sweep ended without finishing — the catalog entries it had not reached will not be started");
}
}
fn maybe_dispatch_to_sonda_cli() {
let mut args = env::args_os();
let _self_arg = args.next();
let first = match args.next() {
Some(arg) => arg,
None => return,
};
let Some(first_str) = first.to_str() else {
return;
};
if !SONDA_SUBCOMMANDS.contains(&first_str) {
return;
}
let sibling = match env::current_exe()
.ok()
.and_then(|p| p.parent().map(|d| d.join("sonda")))
{
Some(path) => path,
None => {
eprintln!(
"sonda-server: failed to resolve sibling `sonda` binary path; \
cannot dispatch `{first_str}` subcommand"
);
exit(127);
}
};
let err = Command::new(&sibling).arg(&first).args(args).exec();
eprintln!(
"sonda-server: failed to exec sibling sonda binary at {}: {err}",
sibling.display()
);
exit(127);
}
fn announce_bound_port(port: u16) -> anyhow::Result<()> {
let line = serde_json::json!({ "sonda_server": { "port": port } });
let stdout = std::io::stdout();
let mut handle = stdout.lock();
writeln!(handle, "{line}").context("failed to write stdout announce")?;
handle.flush().context("failed to flush stdout announce")?;
Ok(())
}
async fn shutdown_signal(
state: AppState,
sweep: Option<tokio::task::JoinHandle<()>>,
sweep_cancel: CancellationToken,
#[cfg(unix)] mut sigterm: tokio::signal::unix::Signal,
) {
let ctrl_c = async {
tokio::signal::ctrl_c()
.await
.expect("failed to install ctrl_c handler");
};
#[cfg(unix)]
let terminate = async {
sigterm.recv().await;
};
#[cfg(not(unix))]
let terminate = std::future::pending::<()>();
tokio::select! {
_ = ctrl_c => {},
_ = terminate => {},
}
info!("shutdown signal received — stopping all running scenarios");
sweep_cancel.cancel();
if let Some(sweep) = sweep {
let _ = sweep.await;
}
stop_and_join_all(
&state,
sonda_core::STOP_GRACE + routes::scenarios::STOP_JOIN_MARGIN,
)
.await;
}
async fn stop_and_join_all(state: &AppState, join_window: Duration) {
for handle in state.scenarios.read().values() {
handle.stop();
}
let ids_handles: Vec<(String, sonda_core::ScenarioHandle)> =
state.scenarios.write().drain().collect();
for (id, mut handle) in ids_handles {
let joined = handle.join_async(Some(join_window)).await;
if handle.is_running() {
warn!(
scenario = %id,
name = %handle.name,
?join_window,
"scenario task still running after the stop window"
);
continue;
}
match joined {
Ok(()) => info!(scenario = %id, "scenario task joined"),
Err(e) => warn!(scenario = %id, error = %e, "scenario task join failed"),
}
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn dispatch_list_covers_all_known_subcommands() {
let expected = ["run", "list", "show", "new", "test", "completions"];
assert_eq!(SONDA_SUBCOMMANDS.len(), expected.len());
for name in expected {
assert!(
SONDA_SUBCOMMANDS.contains(&name),
"{name} must be in SONDA_SUBCOMMANDS"
);
}
}
#[test]
fn server_flags_are_not_treated_as_subcommands() {
for flag in [
"--port",
"--bind",
"--api-key",
"--help",
"--version",
"-h",
"-V",
] {
assert!(
!SONDA_SUBCOMMANDS.contains(&flag),
"{flag} must not be in SONDA_SUBCOMMANDS"
);
}
}
#[test]
fn dispatch_list_has_no_duplicates() {
let mut sorted: Vec<&str> = SONDA_SUBCOMMANDS.to_vec();
sorted.sort_unstable();
let len_before = sorted.len();
sorted.dedup();
assert_eq!(
len_before,
sorted.len(),
"SONDA_SUBCOMMANDS contains duplicates"
);
}
#[tokio::test]
async fn a_spawned_sweep_that_panics_is_reported_as_failed_without_waiting_for_shutdown() {
let status = Arc::new(SweepStatus::in_progress(3));
spawn_supervised_sweep(async { panic!("sweep exploded") }, Arc::clone(&status))
.await
.expect("the supervisor must not panic with the sweep");
assert_eq!(status.snapshot().phase, crate::state::SweepPhase::Failed);
}
#[tokio::test]
async fn a_spawned_sweep_that_finishes_stays_ready() {
let status = Arc::new(SweepStatus::in_progress(1));
let sweep_status = Arc::clone(&status);
spawn_supervised_sweep(
async move {
sweep_status.record_started();
sweep_status.finish();
},
Arc::clone(&status),
)
.await
.expect("the supervisor must not panic");
let snap = status.snapshot();
assert_eq!(snap.phase, crate::state::SweepPhase::Finished);
assert_eq!((snap.started, snap.expected), (1, 1));
}
#[tokio::test]
async fn a_panicking_sweep_is_reported_as_failed_without_waiting_for_shutdown() {
let status = Arc::new(SweepStatus::in_progress(3));
status.record_started();
let sweeping = tokio::spawn(async { panic!("sweep exploded") });
report_sweep_outcome(sweeping, Arc::clone(&status)).await;
let snap = status.snapshot();
assert_eq!(snap.phase, crate::state::SweepPhase::Failed);
assert!(!snap.phase.is_ready());
assert_eq!((snap.started, snap.expected), (1, 3));
}
#[tokio::test]
async fn an_aborted_sweep_is_reported_as_failed() {
let status = Arc::new(SweepStatus::in_progress(3));
let sweeping = tokio::spawn(std::future::pending::<()>());
sweeping.abort();
report_sweep_outcome(sweeping, Arc::clone(&status)).await;
assert_eq!(status.snapshot().phase, crate::state::SweepPhase::Failed);
}
#[tokio::test]
async fn a_sweep_that_returns_normally_keeps_the_status_it_set_itself() {
let status = Arc::new(SweepStatus::in_progress(3));
let sweep_status = Arc::clone(&status);
let sweeping = tokio::spawn(async move { sweep_status.finish() });
report_sweep_outcome(sweeping, Arc::clone(&status)).await;
assert_eq!(status.snapshot().phase, crate::state::SweepPhase::Finished);
}
#[derive(Clone, Default)]
struct Captured(Arc<std::sync::Mutex<Vec<u8>>>);
impl std::io::Write for Captured {
fn write(&mut self, buf: &[u8]) -> std::io::Result<usize> {
self.0.lock().expect("capture lock").extend_from_slice(buf);
Ok(buf.len())
}
fn flush(&mut self) -> std::io::Result<()> {
Ok(())
}
}
impl Captured {
fn lines_about(&self, id: &str) -> Vec<String> {
let field = format!("scenario={id}");
String::from_utf8_lossy(&self.0.lock().expect("capture lock"))
.lines()
.filter(|line| line.contains(&field))
.map(str::to_owned)
.collect()
}
}
fn scenario_handle(id: &str, ignores_stop: bool) -> sonda_core::ScenarioHandle {
let cancel = CancellationToken::new();
let stopped = cancel.clone();
let task = tokio::spawn(async move {
if ignores_stop {
std::future::pending::<()>().await;
}
stopped.cancelled().await;
Ok::<_, sonda_core::SondaError>(())
});
sonda_core::ScenarioHandle::new(
id.to_string(),
id.to_string(),
None,
cancel,
Some(task),
std::time::Instant::now(),
Arc::new(std::sync::RwLock::new(sonda_core::ScenarioStats::default())),
1.0,
Arc::new(std::sync::atomic::AtomicBool::new(true)),
Arc::new(std::collections::HashMap::new()),
None,
Arc::new(std::sync::atomic::AtomicBool::new(false)),
)
}
#[tokio::test]
async fn a_scenario_still_running_after_the_stop_window_is_not_logged_as_joined() {
let captured = Captured::default();
let writer = captured.clone();
let subscriber = tracing_subscriber::fmt()
.with_writer(move || writer.clone())
.with_ansi(false)
.with_max_level(tracing::Level::INFO)
.finish();
let _default = tracing::subscriber::set_default(subscriber);
let state = AppState::new();
{
let mut scenarios = state.scenarios.write();
for (id, ignores_stop) in [("ends-on-stop", false), ("ignores-the-stop", true)] {
scenarios.insert(id.to_string(), scenario_handle(id, ignores_stop));
}
}
stop_and_join_all(&state, Duration::from_millis(200)).await;
assert!(state.scenarios.read().is_empty(), "the map was drained");
let ended = captured.lines_about("ends-on-stop");
assert!(
ended
.iter()
.any(|line| line.contains("INFO") && line.contains("scenario task joined")),
"guard: the capture sees a join: {ended:?}"
);
let stuck = captured.lines_about("ignores-the-stop");
assert!(
stuck.iter().any(|line| line.contains("WARN")
&& line.contains("scenario task still running after the stop window")),
"{stuck:?}"
);
assert!(
!stuck.iter().any(|line| line.contains("joined")),
"a task still running was logged as joined: {stuck:?}"
);
}
#[test]
fn sonda_git_sha_env_is_injected_by_build_rs() {
let sha = env!("SONDA_GIT_SHA");
assert!(
!sha.is_empty(),
"SONDA_GIT_SHA must be injected (either a git rev or the 'unknown' fallback)"
);
}
}