sonda-server 1.22.0

HTTP control plane for Sonda — synthetic telemetry generator
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
//! sonda-server — HTTP control plane for the Sonda telemetry generator.
//!
//! Exposes a REST API that allows scenarios to be started, inspected, and
//! stopped over HTTP. All scenario lifecycle logic is delegated to sonda-core.

mod auth;
mod autostart;
mod gate_registry;
mod locks;
mod middleware;
mod routes;
mod state;

use std::env;
use std::io::Write;
use std::net::SocketAddr;
use std::os::unix::process::CommandExt;
use std::path::PathBuf;
use std::process::{exit, Command};
use std::sync::Arc;
use std::time::Duration;

use anyhow::Context;
use clap::Parser;
use tokio::sync::Semaphore;
use tokio_util::sync::CancellationToken;
use tracing::{error, info, warn};

use crate::routes::RouterConfig;
use crate::state::{AppState, SweepStatus};

/// Subcommands the dispatch shim forwards to the sibling `sonda` binary.
/// Mirror of `sonda`'s clap definition.
const SONDA_SUBCOMMANDS: &[&str] = &["run", "list", "show", "new", "test", "completions"];

/// Command-line arguments for sonda-server.
///
/// Uses a manual `Debug` implementation to redact `api_key`, preventing
/// accidental exposure of the secret in log output.
#[derive(Parser)]
#[command(name = "sonda-server", version, about = "HTTP control plane for Sonda")]
struct Args {
    /// Port to listen on.
    #[arg(long, default_value_t = 8080)]
    port: u16,

    /// Address to bind to.
    #[arg(long, default_value = "0.0.0.0")]
    bind: String,

    /// API key for bearer-token authentication on `/scenarios/*`, `/events`, `/metrics`, and `/scenarios/metrics` endpoints.
    ///
    /// When set, requests to these endpoints must include an
    /// `Authorization: Bearer <key>` header. The `/health` and `/ready`
    /// endpoints remain public regardless of this setting.
    ///
    /// Can also be set via the `SONDA_API_KEY` environment variable.
    #[arg(long, env = "SONDA_API_KEY")]
    api_key: Option<String>,

    /// Directory of scenario and pack YAML files for resolving `pack: <name>`
    /// references in posted scenario bodies.
    ///
    /// Can also be set via the `SONDA_CATALOG` environment variable.
    #[arg(long, env = "SONDA_CATALOG")]
    catalog: Option<PathBuf>,

    /// Start every `kind: runnable` entry in `--catalog` at server startup.
    ///
    /// Can also be set via the `SONDA_AUTOSTART` environment variable, where
    /// an empty value and `0` / `no` / `off` / `false` all mean disabled.
    #[arg(
        long,
        env = "SONDA_AUTOSTART",
        action = clap::ArgAction::SetTrue,
        value_parser = clap::builder::FalseyValueParser::new()
    )]
    autostart: bool,

    /// Tokio worker thread count. Defaults to min(available_parallelism(), 16).
    #[arg(long, value_parser = clap::builder::RangedU64ValueParser::<u64>::new().range(1..))]
    workers: Option<u64>,

    /// Maximum concurrent scenario rows in `AppState`. `0` means unlimited.
    #[arg(long, default_value_t = 0)]
    max_scenarios: usize,

    /// Maximum concurrent in-flight control-plane HTTP requests. Defaults to `4 * workers`.
    #[arg(long)]
    max_inflight_requests: Option<usize>,

    /// Per-request timeout in seconds applied to control-plane routes. Returns 408 on expiry.
    #[arg(long, default_value_t = 30,
          value_parser = clap::builder::RangedU64ValueParser::<u64>::new().range(1..))]
    request_timeout: u64,

    /// Maximum request body size in bytes accepted by control-plane routes. Returns 413 when exceeded.
    #[arg(long, default_value_t = 1_048_576)]
    max_body_bytes: usize,
}

impl std::fmt::Debug for Args {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        f.debug_struct("Args")
            .field("port", &self.port)
            .field("bind", &self.bind)
            .field("api_key", &self.api_key.as_ref().map(|_| "[REDACTED]"))
            .field("catalog", &self.catalog)
            .field("autostart", &self.autostart)
            .field("workers", &self.workers)
            .field("max_scenarios", &self.max_scenarios)
            .field("max_inflight_requests", &self.max_inflight_requests)
            .field("request_timeout", &self.request_timeout)
            .field("max_body_bytes", &self.max_body_bytes)
            .finish()
    }
}

fn main() -> anyhow::Result<()> {
    maybe_dispatch_to_sonda_cli();

    let args = Args::parse();
    let workers = args.workers.map(|n| n as usize).unwrap_or_else(|| {
        std::thread::available_parallelism()
            .map(|n| n.get())
            .unwrap_or(1)
            .min(16)
    });
    let max_inflight_requests = args.max_inflight_requests.unwrap_or(workers * 4);

    let runtime = tokio::runtime::Builder::new_multi_thread()
        .worker_threads(workers)
        .enable_all()
        .build()
        .context("failed to build tokio runtime")?;

    runtime.block_on(async move { run(args, workers, max_inflight_requests).await })
}

async fn run(args: Args, workers: usize, max_inflight_requests: usize) -> anyhow::Result<()> {
    tracing_subscriber::fmt()
        .with_env_filter(
            tracing_subscriber::EnvFilter::try_from_default_env()
                .unwrap_or_else(|_| tracing_subscriber::EnvFilter::new("info")),
        )
        .with_writer(std::io::stderr)
        .init();

    let bind_addr: SocketAddr = format!("{}:{}", args.bind, args.port)
        .parse()
        .with_context(|| format!("invalid bind address: {}:{}", args.bind, args.port))?;

    // Normalise the API key: treat empty strings as None (disabled).
    let api_key = args.api_key.filter(|k| {
        if k.is_empty() {
            warn!("--api-key / SONDA_API_KEY is empty — authentication disabled");
            false
        } else {
            true
        }
    });

    if api_key.is_some() {
        info!(
            "API key authentication enabled for /scenarios/*, /events, /metrics, and /scenarios/metrics endpoints"
        );
    } else {
        info!("API key authentication disabled — all endpoints are public");
    }

    if args.autostart && args.catalog.is_none() {
        anyhow::bail!(
            "--autostart requires --catalog <DIR> (or SONDA_AUTOSTART requires SONDA_CATALOG): \
             there is no scenario catalog to start from"
        );
    }

    if let Some(dir) = &args.catalog {
        if !dir.is_dir() {
            anyhow::bail!(
                "--catalog {}: does not exist or is not a directory",
                dir.display()
            );
        }
        info!(catalog = %dir.display(), "pack catalog enabled for POST /scenarios");
    }

    let autostart_entries = match args.catalog.as_deref() {
        Some(dir) if args.autostart => autostart::runnable_entries(dir)?,
        _ => Vec::new(),
    };

    let sweep_status = Arc::new(if args.autostart {
        SweepStatus::in_progress(autostart_entries.len())
    } else {
        SweepStatus::not_configured()
    });

    let permits = if args.max_scenarios == 0 {
        warn!("--max-scenarios 0 — scenario row cap disabled (unlimited)");
        Semaphore::new(Semaphore::MAX_PERMITS)
    } else {
        Semaphore::new(args.max_scenarios)
    };

    let state = AppState {
        api_key: api_key.map(Arc::new),
        catalog_dir: args.catalog.clone().map(Arc::new),
        scenario_permits: Arc::new(permits),
        worker_threads: workers,
        max_scenarios: args.max_scenarios,
        sweep_status,
        ..AppState::new()
    };

    let inflight_semaphore = Arc::new(Semaphore::new(max_inflight_requests));
    let router_cfg = RouterConfig {
        request_timeout: Duration::from_secs(args.request_timeout),
        max_body_bytes: args.max_body_bytes,
        inflight_semaphore,
    };
    let app = routes::router_with_config(state.clone(), router_cfg);

    let listener = tokio::net::TcpListener::bind(bind_addr)
        .await
        .with_context(|| format!("failed to bind to {bind_addr}"))?;

    let bound_addr = listener
        .local_addr()
        .context("failed to read local address from bound listener")?;

    #[cfg(unix)]
    let sigterm = tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
        .context("failed to install SIGTERM handler")?;

    announce_bound_port(bound_addr.port())?;

    info!(addr = %bound_addr, workers, "sonda-server listening");

    let sweep_cancel = CancellationToken::new();
    let sweep = args.autostart.then(|| {
        let sweep_state = state.clone();
        let cancel = sweep_cancel.clone();
        spawn_supervised_sweep(
            async move { autostart::start_entries(&sweep_state, &autostart_entries, &cancel).await },
            state.sweep_status.clone(),
        )
    });

    axum::serve(listener, app)
        .with_graceful_shutdown(shutdown_signal(
            state,
            sweep,
            sweep_cancel,
            #[cfg(unix)]
            sigterm,
        ))
        .await
        .context("server error")?;

    info!("sonda-server shut down cleanly");
    Ok(())
}

/// Run the sweep under a task that observes how it ended. The returned handle is
/// the supervisor's: awaiting it joins the sweep and its outcome report.
fn spawn_supervised_sweep<F>(sweep: F, status: Arc<SweepStatus>) -> tokio::task::JoinHandle<()>
where
    F: std::future::Future<Output = ()> + Send + 'static,
{
    let sweeping = tokio::spawn(sweep);
    tokio::spawn(report_sweep_outcome(sweeping, status))
}

/// Report a sweep that ended without finishing when it happens, not at shutdown.
async fn report_sweep_outcome(sweeping: tokio::task::JoinHandle<()>, status: Arc<SweepStatus>) {
    if let Err(error) = sweeping.await {
        status.fail();
        error!(%error, "autostart: sweep ended without finishing — the catalog entries it had not reached will not be started");
    }
}

/// If `argv[1]` is a sonda subcommand, exec the sibling `sonda` binary and
/// never return. Otherwise no-op. Sibling resolved via `current_exe()` so dev
/// builds dispatch within the same `target/<profile>/`.
fn maybe_dispatch_to_sonda_cli() {
    let mut args = env::args_os();
    let _self_arg = args.next();
    let first = match args.next() {
        Some(arg) => arg,
        None => return,
    };

    let Some(first_str) = first.to_str() else {
        return;
    };

    if !SONDA_SUBCOMMANDS.contains(&first_str) {
        return;
    }

    let sibling = match env::current_exe()
        .ok()
        .and_then(|p| p.parent().map(|d| d.join("sonda")))
    {
        Some(path) => path,
        None => {
            eprintln!(
                "sonda-server: failed to resolve sibling `sonda` binary path; \
                 cannot dispatch `{first_str}` subcommand"
            );
            exit(127);
        }
    };

    // exec only returns on failure (replaces process image otherwise).
    let err = Command::new(&sibling).arg(&first).args(args).exec();
    eprintln!(
        "sonda-server: failed to exec sibling sonda binary at {}: {err}",
        sibling.display()
    );
    exit(127);
}

/// Write `{"sonda_server":{"port":N}}\n` to stdout and flush.
fn announce_bound_port(port: u16) -> anyhow::Result<()> {
    let line = serde_json::json!({ "sonda_server": { "port": port } });
    let stdout = std::io::stdout();
    let mut handle = stdout.lock();
    writeln!(handle, "{line}").context("failed to write stdout announce")?;
    handle.flush().context("failed to flush stdout announce")?;
    Ok(())
}

/// Wait for Ctrl+C or SIGTERM, then cancel and join the autostart sweep before
/// draining the scenarios it started. SIGTERM coverage is what `docker stop`
/// and Kubernetes pod eviction rely on; without it the process is SIGKILLed
/// after the grace period.
async fn shutdown_signal(
    state: AppState,
    sweep: Option<tokio::task::JoinHandle<()>>,
    sweep_cancel: CancellationToken,
    #[cfg(unix)] mut sigterm: tokio::signal::unix::Signal,
) {
    let ctrl_c = async {
        tokio::signal::ctrl_c()
            .await
            .expect("failed to install ctrl_c handler");
    };

    #[cfg(unix)]
    let terminate = async {
        sigterm.recv().await;
    };
    #[cfg(not(unix))]
    let terminate = std::future::pending::<()>();

    tokio::select! {
        _ = ctrl_c => {},
        _ = terminate => {},
    }

    info!("shutdown signal received — stopping all running scenarios");

    sweep_cancel.cancel();
    if let Some(sweep) = sweep {
        // report_sweep_outcome already reported anything worth reporting; this join
        // only orders the sweep ahead of the drain.
        let _ = sweep.await;
    }

    for handle in state.scenarios.read().values() {
        handle.stop();
    }

    // Write lock: join consumes the inner JoinHandle.
    let ids_handles: Vec<(String, sonda_core::ScenarioHandle)> =
        state.scenarios.write().drain().collect();
    for (id, mut handle) in ids_handles {
        match handle.join_async(Some(Duration::from_secs(5))).await {
            Ok(_) => info!(scenario = %id, "scenario task joined"),
            Err(e) => warn!(scenario = %id, error = %e, "scenario task join failed"),
        }
    }
}

#[cfg(test)]
mod tests {
    use super::*;

    // NOTE: this list is a *fourth* copy of the CLI's verbs, and it is the one
    // that has to be updated by hand alongside SONDA_SUBCOMMANDS itself — it
    // cannot reach the `sonda` crate's parser from here. The authoritative
    // comparison against `sonda --help` lives in
    // `sonda/tests/cli_subcommand_parity.rs`; this one keeps the server's own
    // suite honest about its own constant.
    #[test]
    fn dispatch_list_covers_all_known_subcommands() {
        let expected = ["run", "list", "show", "new", "test", "completions"];
        assert_eq!(SONDA_SUBCOMMANDS.len(), expected.len());
        for name in expected {
            assert!(
                SONDA_SUBCOMMANDS.contains(&name),
                "{name} must be in SONDA_SUBCOMMANDS"
            );
        }
    }

    #[test]
    fn server_flags_are_not_treated_as_subcommands() {
        for flag in [
            "--port",
            "--bind",
            "--api-key",
            "--help",
            "--version",
            "-h",
            "-V",
        ] {
            assert!(
                !SONDA_SUBCOMMANDS.contains(&flag),
                "{flag} must not be in SONDA_SUBCOMMANDS"
            );
        }
    }

    #[test]
    fn dispatch_list_has_no_duplicates() {
        let mut sorted: Vec<&str> = SONDA_SUBCOMMANDS.to_vec();
        sorted.sort_unstable();
        let len_before = sorted.len();
        sorted.dedup();
        assert_eq!(
            len_before,
            sorted.len(),
            "SONDA_SUBCOMMANDS contains duplicates"
        );
    }

    #[tokio::test]
    async fn a_spawned_sweep_that_panics_is_reported_as_failed_without_waiting_for_shutdown() {
        let status = Arc::new(SweepStatus::in_progress(3));

        spawn_supervised_sweep(async { panic!("sweep exploded") }, Arc::clone(&status))
            .await
            .expect("the supervisor must not panic with the sweep");

        assert_eq!(status.snapshot().phase, crate::state::SweepPhase::Failed);
    }

    #[tokio::test]
    async fn a_spawned_sweep_that_finishes_stays_ready() {
        let status = Arc::new(SweepStatus::in_progress(1));
        let sweep_status = Arc::clone(&status);

        spawn_supervised_sweep(
            async move {
                sweep_status.record_started();
                sweep_status.finish();
            },
            Arc::clone(&status),
        )
        .await
        .expect("the supervisor must not panic");

        let snap = status.snapshot();
        assert_eq!(snap.phase, crate::state::SweepPhase::Finished);
        assert_eq!((snap.started, snap.expected), (1, 1));
    }

    #[tokio::test]
    async fn a_panicking_sweep_is_reported_as_failed_without_waiting_for_shutdown() {
        let status = Arc::new(SweepStatus::in_progress(3));
        status.record_started();
        let sweeping = tokio::spawn(async { panic!("sweep exploded") });

        report_sweep_outcome(sweeping, Arc::clone(&status)).await;

        let snap = status.snapshot();
        assert_eq!(snap.phase, crate::state::SweepPhase::Failed);
        assert!(!snap.phase.is_ready());
        assert_eq!((snap.started, snap.expected), (1, 3));
    }

    #[tokio::test]
    async fn an_aborted_sweep_is_reported_as_failed() {
        let status = Arc::new(SweepStatus::in_progress(3));
        let sweeping = tokio::spawn(std::future::pending::<()>());
        sweeping.abort();

        report_sweep_outcome(sweeping, Arc::clone(&status)).await;

        assert_eq!(status.snapshot().phase, crate::state::SweepPhase::Failed);
    }

    #[tokio::test]
    async fn a_sweep_that_returns_normally_keeps_the_status_it_set_itself() {
        let status = Arc::new(SweepStatus::in_progress(3));
        let sweep_status = Arc::clone(&status);
        let sweeping = tokio::spawn(async move { sweep_status.finish() });

        report_sweep_outcome(sweeping, Arc::clone(&status)).await;

        assert_eq!(status.snapshot().phase, crate::state::SweepPhase::Finished);
    }

    #[test]
    fn sonda_git_sha_env_is_injected_by_build_rs() {
        let sha = env!("SONDA_GIT_SHA");
        assert!(
            !sha.is_empty(),
            "SONDA_GIT_SHA must be injected (either a git rev or the 'unknown' fallback)"
        );
    }
}