fv-streams 0.9.0

The FusionVault Streams standalone runner: run a stream pipeline defined in one TOML file (steps or SQL) with no control plane — every connector built in, the Nexmark event generator, checkpoints to a local directory or an object store, scale-out across workers.
Documentation
//! `fv-streams run <pipeline.toml>` — the standalone runner. Everything it does is the library
//! (`fv_streams::Pipeline`); this file is argument parsing, Ctrl-C, and the optional profiler.

use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;

use fv_streams::Pipeline;

/// The stop flag every serving mode watches, raised by the first SIGINT (Ctrl-C) or SIGTERM — what
/// `docker stop`, `kubectl delete` and `systemctl stop` send. Without SIGTERM here the process would
/// die at once on every orchestrated stop and the ordered shutdown would never run. A second signal
/// ends the process at once, saying `forced`: a stop held open by something stuck is never left
/// without a way out (the handler owns both signals, so they would otherwise be swallowed).
fn stop_on_signal(what: &'static str, forced: &'static str) -> Arc<AtomicBool> {
    let stop = Arc::new(AtomicBool::new(false));
    let flag = stop.clone();
    tokio::spawn(async move {
        #[cfg(unix)]
        let mut term =
            tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate()).expect("SIGTERM handler");
        let mut asked = false;
        loop {
            #[cfg(unix)]
            tokio::select! {
                _ = tokio::signal::ctrl_c() => {}
                _ = term.recv() => {}
            }
            #[cfg(not(unix))]
            {
                let _ = tokio::signal::ctrl_c().await;
            }
            if asked {
                eprintln!("\nfv-streams: a second signal: {forced}");
                std::process::exit(130);
            }
            asked = true;
            eprintln!("\nfv-streams: {what}");
            flag.store(true, Ordering::Relaxed);
        }
    });
    stop
}

fn usage() -> ! {
    eprintln!("fv-streams — run TOML-defined stream pipelines (FusionVault Streams)\n");
    eprintln!("USAGE:\n  fv-streams run [--join <leader-addr>] [--workers <n>] [--serve <addr>] <pipeline.toml>\n  fv-streams probe <url>    exit 0 when <url> answers 2xx (a container / exec health check)");
    eprintln!("  fv-streams flight-sql [--host <h>] [--port <n>] <catalog.toml>\n");
    eprintln!("  run          run one pipeline to completion (or until stopped)");
    eprintln!("  --join <addr>    join a leader's cluster at its control address (this is a worker)");
    eprintln!("  --workers <n>    the leader waits for <n> workers total (this is the leader)");
    eprintln!("  --serve <addr>   serve /health /ready /status /metrics on <addr> (or STREAM_SERVE)\n");
    eprintln!("  flight-sql   serve a Flight control endpoint: submit a SQL statement, read its Arrow");
    eprintln!("               result over Flight. <catalog.toml> declares the datasets the SQL reads.");
    eprintln!("  --host <h>       host the endpoint + each query's result bind (default 127.0.0.1)");
    eprintln!("  --port <n>       control endpoint port (default 8817)\n");
    eprintln!("Engine settings come from STREAM_* environment variables or the pipeline's [settings]");
    eprintln!("table (the environment wins). Every setting: docs/settings.md.");
    std::process::exit(2)
}

#[tokio::main]
async fn main() {
    let args: Vec<String> = std::env::args().collect();
    match args.get(1).map(String::as_str) {
        Some("run") => {}                                   // handled below
        Some("flight-sql") => flight_sql_main(&args).await, // diverges (serves until stopped)
        Some("probe") => probe_main(&args),                 // diverges (exits with the probe's verdict)
        _ => usage(),
    }
    // `run [--join <addr>] [--workers <n>] <pipeline.toml>`. The flags are sugar for the STREAM_JOIN
    // / STREAM_WORKERS env vars the engine reads (the environment still wins if both are set).
    let mut path: Option<String> = None;
    let mut serve: Option<String> = Some(fv_streams::SERVE.raw()).filter(|s| !s.is_empty());
    let mut i = 2;
    while i < args.len() {
        match args[i].as_str() {
            "--join" => {
                i += 1;
                match args.get(i) {
                    Some(addr) if !fv_streams_engine::settings::JOIN.is_set() => {
                        std::env::set_var(fv_streams_engine::settings::JOIN.key, addr)
                    }
                    Some(_) => {}
                    None => usage(),
                }
            }
            "--serve" => {
                i += 1;
                match args.get(i) {
                    Some(a) => serve = Some(a.clone()),
                    None => usage(),
                }
            }
            "--workers" => {
                i += 1;
                match args.get(i) {
                    Some(n) if !fv_streams_engine::settings::WORKERS.is_set() => {
                        std::env::set_var(fv_streams_engine::settings::WORKERS.key, n)
                    }
                    Some(_) => {}
                    None => usage(),
                }
            }
            p if !p.starts_with("--") => path = Some(p.to_string()),
            _ => usage(),
        }
        i += 1;
    }
    let path = match path {
        Some(p) => p,
        None => usage(),
    };
    let pipeline = match Pipeline::load(&path) {
        Ok(p) => p,
        Err(e) => {
            eprintln!("fv-streams: {e}");
            std::process::exit(1);
        }
    };

    // Ctrl-C / SIGTERM → the stop flag → the next heartbeat returns Stop → the engine's ordered
    // shutdown (final checkpoint → its commit; windows still open stay in that checkpoint).
    let stop = stop_on_signal(
        "stop requested — taking a final checkpoint…",
        "exiting now, and the final checkpoint was abandoned. The next start replays from the last checkpoint.",
    );

    // Opt-in in-process CPU profiling. Build with `--features profiling`, run with
    // `FV_PROFILE=/path/flame.svg` — a ~499Hz signal-sampling profiler wraps the whole run and
    // writes a flamegraph on shutdown. Zero cost when the variable is unset; absent from default
    // builds. Frequency 499 (prime) avoids lockstep with periodic work.
    #[cfg(feature = "profiling")]
    let profiler = std::env::var("FV_PROFILE").ok().and_then(|out| {
        match pprof::ProfilerGuardBuilder::default()
            .frequency(499)
            .blocklist(&["libc", "libgcc", "pthread", "vdso"])
            .build()
        {
            Ok(g) => {
                println!("fv-streams: profiling ON → {out}");
                Some((g, out))
            }
            Err(e) => {
                eprintln!("fv-streams: profiler failed to start: {e}");
                None
            }
        }
    });

    if let Err(e) = pipeline.run_serving(stop, serve.as_deref()).await {
        eprintln!("fv-streams: {e}");
        std::process::exit(1);
    }

    #[cfg(feature = "profiling")]
    if let Some((guard, out)) = profiler {
        match guard.report().build() {
            Ok(report) => match std::fs::File::create(&out) {
                Ok(f) => match report.flamegraph(f) {
                    Ok(()) => println!("fv-streams: flamegraph written → {out}"),
                    Err(e) => eprintln!("fv-streams: flamegraph write failed: {e}"),
                },
                Err(e) => eprintln!("fv-streams: cannot create {out}: {e}"),
            },
            Err(e) => eprintln!("fv-streams: profile report failed: {e}"),
        }
    }
}

/// `fv-streams flight-sql [--host <h>] [--port <n>] <catalog.toml>`: serve the Flight control
/// endpoint. A client submits a SQL statement (a `query` do_action) and reads its Arrow result over
/// Flight; `<catalog.toml>` declares the datasets the statements read.
async fn flight_sql_main(args: &[String]) -> ! {
    let mut catalog_path: Option<String> = None;
    let mut host = "127.0.0.1".to_string();
    let mut port: u16 = 8817;
    let mut i = 2;
    while i < args.len() {
        match args[i].as_str() {
            "--host" => {
                i += 1;
                match args.get(i) {
                    Some(h) => host = h.clone(),
                    None => usage(),
                }
            }
            "--port" => {
                i += 1;
                match args.get(i).and_then(|p| p.parse().ok()) {
                    Some(p) => port = p,
                    None => usage(),
                }
            }
            p if !p.starts_with("--") => catalog_path = Some(p.to_string()),
            _ => usage(),
        }
        i += 1;
    }
    let catalog = match catalog_path.map(|p| Pipeline::load(&p)) {
        Some(Ok(c)) => c,
        Some(Err(e)) => {
            eprintln!("fv-streams: {e}");
            std::process::exit(1);
        }
        None => usage(),
    };
    // The catalog's [settings] apply to every query it launches (e.g. STREAM_STATE_STORE).
    if let Err(e) = catalog.apply_settings() {
        eprintln!("fv-streams: {e}");
        std::process::exit(1);
    }

    let stop = stop_on_signal("flight-sql shutting down…", "exiting now.");
    match fv_streams::flight_sql::serve(catalog, &host, port, stop).await {
        Ok(()) => std::process::exit(0),
        Err(e) => {
            eprintln!("fv-streams: {e}");
            std::process::exit(1);
        }
    }
}

/// `fv-streams probe <url>`: GET the URL and exit 0 on a 2xx, 1 otherwise — the image's health
/// check (distroless carries no shell or curl) and any orchestrator's exec probe, from the one
/// binary. A two-second timeout: a probe must never hang the checker.
fn probe_main(args: &[String]) -> ! {
    let Some(url) = args.get(2) else { usage() };
    let agent: ureq::Agent = ureq::Agent::config_builder()
        .timeout_global(Some(std::time::Duration::from_secs(2)))
        .build()
        .into();
    match agent.get(url).call() {
        Ok(resp) if (200..300).contains(&resp.status().as_u16()) => std::process::exit(0),
        Ok(resp) => {
            eprintln!("probe {url}: HTTP {}", resp.status().as_u16());
            std::process::exit(1)
        }
        Err(e) => {
            eprintln!("probe {url}: {e}");
            std::process::exit(1)
        }
    }
}