fv-streams 0.4.0

The FusionVault Streams standalone runner: run a stream pipeline defined in one TOML file against Kafka or Redpanda with no control plane, including the built-in Nexmark event generator.
Documentation
//! `fv-streams run <pipeline.toml>` — the standalone runner. Everything it does is the library
//! (`fv_streams::Pipeline`); this file is argument parsing, Ctrl-C, and the optional profiler.

use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::Arc;

use fv_streams::Pipeline;

fn usage() -> ! {
    eprintln!("fv-streams — run TOML-defined stream pipelines (FusionVault Streams)\n");
    eprintln!("USAGE:\n  fv-streams run [--join <leader-addr>] [--workers <n>] <pipeline.toml>");
    eprintln!("  fv-streams flight-sql [--host <h>] [--port <n>] <catalog.toml>\n");
    eprintln!("  run          run one pipeline to completion (or until stopped)");
    eprintln!("  --join <addr>    join a leader's cluster at its control address (this is a worker)");
    eprintln!("  --workers <n>    the leader waits for <n> workers total (this is the leader)\n");
    eprintln!("  flight-sql   serve a Flight control endpoint: submit a SQL statement, read its Arrow");
    eprintln!("               result over Flight. <catalog.toml> declares the datasets the SQL reads.");
    eprintln!("  --host <h>       host the endpoint + each query's result bind (default 127.0.0.1)");
    eprintln!("  --port <n>       control endpoint port (default 8817)\n");
    eprintln!("  sandbox      serve the browser showcase (streams#6): the HTTP control + Arrow firehose.");
    eprintln!("  --host <h>       bind host (default 127.0.0.1)");
    eprintln!("  --port <n>       sandbox port (default 8816)\n");
    eprintln!("Engine settings come from STREAM_* environment variables or the pipeline's [settings]");
    eprintln!("table (the environment wins). See the README for the full table.");
    std::process::exit(2)
}

#[tokio::main]
async fn main() {
    let args: Vec<String> = std::env::args().collect();
    match args.get(1).map(String::as_str) {
        Some("run") => {}                                   // handled below
        Some("flight-sql") => flight_sql_main(&args).await, // diverges (serves until stopped)
        Some("sandbox") => sandbox_main(&args).await,       // diverges (serves until stopped)
        _ => usage(),
    }
    // `run [--join <addr>] [--workers <n>] <pipeline.toml>`. The flags are sugar for the STREAM_JOIN
    // / STREAM_WORKERS env vars the engine reads (the environment still wins if both are set).
    let mut path: Option<String> = None;
    let mut i = 2;
    while i < args.len() {
        match args[i].as_str() {
            "--join" => {
                i += 1;
                match args.get(i) {
                    Some(addr) if std::env::var_os("STREAM_JOIN").is_none() => std::env::set_var("STREAM_JOIN", addr),
                    Some(_) => {}
                    None => usage(),
                }
            }
            "--workers" => {
                i += 1;
                match args.get(i) {
                    Some(n) if std::env::var_os("STREAM_WORKERS").is_none() => std::env::set_var("STREAM_WORKERS", n),
                    Some(_) => {}
                    None => usage(),
                }
            }
            p if !p.starts_with("--") => path = Some(p.to_string()),
            _ => usage(),
        }
        i += 1;
    }
    let path = match path {
        Some(p) => p,
        None => usage(),
    };
    let pipeline = match Pipeline::load(&path) {
        Ok(p) => p,
        Err(e) => {
            eprintln!("fv-streams: {e}");
            std::process::exit(1);
        }
    };

    // Ctrl-C / SIGTERM → the stop flag → the next heartbeat returns Stop → the engine's ordered
    // shutdown (final checkpoint → flush-emit → offsets land synchronously).
    let stop = Arc::new(AtomicBool::new(false));
    {
        let stop = stop.clone();
        tokio::spawn(async move {
            let _ = tokio::signal::ctrl_c().await;
            eprintln!("\nfv-streams: stop requested — unwinding (final checkpoint + flush)…");
            stop.store(true, Ordering::Relaxed);
        });
    }

    // Opt-in in-process CPU profiling. Build with `--features profiling`, run with
    // `FV_PROFILE=/path/flame.svg` — a ~499Hz signal-sampling profiler wraps the whole run and
    // writes a flamegraph on shutdown. Zero cost when the variable is unset; absent from default
    // builds. Frequency 499 (prime) avoids lockstep with periodic work.
    #[cfg(feature = "profiling")]
    let profiler = std::env::var("FV_PROFILE").ok().and_then(|out| {
        match pprof::ProfilerGuardBuilder::default()
            .frequency(499)
            .blocklist(&["libc", "libgcc", "pthread", "vdso"])
            .build()
        {
            Ok(g) => {
                println!("fv-streams: profiling ON → {out}");
                Some((g, out))
            }
            Err(e) => {
                eprintln!("fv-streams: profiler failed to start: {e}");
                None
            }
        }
    });

    if let Err(e) = pipeline.run(stop).await {
        eprintln!("fv-streams: {e}");
        std::process::exit(1);
    }

    #[cfg(feature = "profiling")]
    if let Some((guard, out)) = profiler {
        match guard.report().build() {
            Ok(report) => match std::fs::File::create(&out) {
                Ok(f) => match report.flamegraph(f) {
                    Ok(()) => println!("fv-streams: flamegraph written → {out}"),
                    Err(e) => eprintln!("fv-streams: flamegraph write failed: {e}"),
                },
                Err(e) => eprintln!("fv-streams: cannot create {out}: {e}"),
            },
            Err(e) => eprintln!("fv-streams: profile report failed: {e}"),
        }
    }
}

/// `fv-streams flight-sql [--host <h>] [--port <n>] <catalog.toml>`: serve the Flight control
/// endpoint. A client submits a SQL statement (a `query` do_action) and reads its Arrow result over
/// Flight; `<catalog.toml>` declares the datasets the statements read.
async fn flight_sql_main(args: &[String]) -> ! {
    let mut catalog_path: Option<String> = None;
    let mut host = "127.0.0.1".to_string();
    let mut port: u16 = 8817;
    let mut i = 2;
    while i < args.len() {
        match args[i].as_str() {
            "--host" => {
                i += 1;
                match args.get(i) {
                    Some(h) => host = h.clone(),
                    None => usage(),
                }
            }
            "--port" => {
                i += 1;
                match args.get(i).and_then(|p| p.parse().ok()) {
                    Some(p) => port = p,
                    None => usage(),
                }
            }
            p if !p.starts_with("--") => catalog_path = Some(p.to_string()),
            _ => usage(),
        }
        i += 1;
    }
    let catalog = match catalog_path.map(|p| Pipeline::load(&p)) {
        Some(Ok(c)) => c,
        Some(Err(e)) => {
            eprintln!("fv-streams: {e}");
            std::process::exit(1);
        }
        None => usage(),
    };
    // The catalog's [settings] apply to every query it launches (e.g. STREAM_STATE_STORE).
    catalog.apply_settings();

    let stop = Arc::new(AtomicBool::new(false));
    {
        let stop = stop.clone();
        tokio::spawn(async move {
            let _ = tokio::signal::ctrl_c().await;
            eprintln!("\nfv-streams: flight-sql shutting down…");
            stop.store(true, Ordering::Relaxed);
        });
    }
    match fv_streams::flight_sql::serve(catalog, &host, port, stop).await {
        Ok(()) => std::process::exit(0),
        Err(e) => {
            eprintln!("fv-streams: {e}");
            std::process::exit(1);
        }
    }
}

/// Without the `sandbox` feature the browser showcase is not compiled in (a published runner carries
/// no web server). Point the user at the build flag.
#[cfg(not(feature = "sandbox"))]
async fn sandbox_main(_args: &[String]) -> ! {
    eprintln!("fv-streams: the `sandbox` subcommand needs the `sandbox` feature — build with `--features sandbox`.");
    std::process::exit(2);
}

/// `fv-streams sandbox [--host <h>] [--port <n>]`: serve the browser showcase (streams#6).
#[cfg(feature = "sandbox")]
async fn sandbox_main(args: &[String]) -> ! {
    let mut host = "127.0.0.1".to_string();
    let mut port: u16 = 8816;
    let mut i = 2;
    while i < args.len() {
        match args[i].as_str() {
            "--host" => {
                i += 1;
                match args.get(i) {
                    Some(h) => host = h.clone(),
                    None => usage(),
                }
            }
            "--port" => {
                i += 1;
                match args.get(i).and_then(|p| p.parse().ok()) {
                    Some(p) => port = p,
                    None => usage(),
                }
            }
            _ => usage(),
        }
        i += 1;
    }
    let stop = Arc::new(AtomicBool::new(false));
    {
        let stop = stop.clone();
        tokio::spawn(async move {
            let _ = tokio::signal::ctrl_c().await;
            eprintln!("\nfv-streams: sandbox shutting down…");
            stop.store(true, Ordering::Relaxed);
        });
    }
    match fv_streams::sandbox::serve(&host, port, stop).await {
        Ok(()) => std::process::exit(0),
        Err(e) => {
            eprintln!("fv-streams: {e}");
            std::process::exit(1);
        }
    }
}