fv-streams 0.7.0

The FusionVault Streams standalone runner: run a stream pipeline defined in one TOML file (steps or SQL) with no control plane — every connector built in, the Nexmark event generator, checkpoints to a local directory or an object store, scale-out across workers.
Documentation
//! THE OBSERVABILITY SURFACE for an orchestrator (`fv-streams run --serve <addr>` or
//! `STREAM_SERVE=<addr>`): one small HTTP listener beside the run.
//!
//! - `GET /health` — liveness: the process is up (200 always). `/healthz` is an alias.
//! - `GET /ready` — readiness: the build is RUNNING (200), else 503 with the status. `/readyz` too.
//! - `GET /status` — the live build record as JSON (status, counters, the CPU ledger, the last
//!   checkpoint, timestamps).
//! - `GET /metrics` — the same in Prometheus exposition: `fv_streams_consumed_total`,
//!   `fv_streams_emitted_total`, `fv_streams_dropped_total`, `fv_streams_cpu_ms_total{family}`,
//!   `fv_streams_task_cpu_ms_total{role}`, `fv_streams_checkpoint_epoch`,
//!   `fv_streams_checkpoint_age_seconds`, `fv_streams_checkpoint_file_bytes`,
//!   `fv_streams_checkpoint_took_ms`, `fv_streams_build_status{status}` (1 for the current one).
//!
//! Nothing here touches the data path: the record is the one the heartbeat already builds
//! (`STREAM_HEARTBEAT_SECONDS`), published through a `watch` channel the handlers read.

use std::sync::Arc;

use axum::extract::State;
use axum::http::StatusCode;
use axum::response::IntoResponse;
use axum::routing::get;
use axum::{Json, Router};
use serde_json::Value as J;
use tokio::sync::watch;

/// The live record, as published by the control plane on every heartbeat.
pub type Live = watch::Receiver<J>;

/// Bind `addr` and serve until the process ends. Returns once bound (the server runs on the
/// runtime); a bind failure is the caller's to report.
pub async fn start(addr: &str, live: Live) -> Result<std::net::SocketAddr, String> {
    let app = Router::new()
        // `/health` and `/ready`; the Kubernetes-convention `z` spellings as aliases (a probe takes any path)
        .route("/health", get(|| async { "ok" }))
        .route("/healthz", get(|| async { "ok" }))
        .route("/ready", get(readyz))
        .route("/readyz", get(readyz))
        .route("/status", get(status))
        .route("/metrics", get(metrics))
        .with_state(Arc::new(live));
    let listener = tokio::net::TcpListener::bind(addr)
        .await
        .map_err(|e| format!("--serve {addr}: {e}"))?;
    let bound = listener.local_addr().map_err(|e| e.to_string())?;
    tokio::spawn(async move {
        let _ = axum::serve(listener, app).await;
    });
    Ok(bound)
}

async fn readyz(State(live): State<Arc<Live>>) -> impl IntoResponse {
    let r = live.borrow().clone();
    let status = r["status"].as_str().unwrap_or("STARTING").to_string();
    if status == "RUNNING" {
        (StatusCode::OK, status)
    } else {
        (StatusCode::SERVICE_UNAVAILABLE, status)
    }
}

async fn status(State(live): State<Arc<Live>>) -> Json<J> {
    Json(live.borrow().clone())
}

async fn metrics(State(live): State<Arc<Live>>) -> impl IntoResponse {
    let r = live.borrow().clone();
    ([("content-type", "text/plain; version=0.0.4")], render(&r))
}

/// The record in Prometheus exposition format.
pub fn render(r: &J) -> String {
    let m = &r["metrics"];
    let mut out = String::with_capacity(2048);
    let mut counter = |name: &str, help: &str, v: u64| {
        out.push_str(&format!("# HELP {name} {help}\n# TYPE {name} counter\n{name} {v}\n"));
    };
    counter(
        "fv_streams_consumed_total",
        "Rows the sources produced.",
        m["consumed"].as_u64().unwrap_or(0),
    );
    counter(
        "fv_streams_emitted_total",
        "Rows the sinks wrote.",
        m["emitted"].as_u64().unwrap_or(0),
    );
    counter(
        "fv_streams_dropped_total",
        "Rows dropped as bad data.",
        m["dropped"].as_u64().unwrap_or(0),
    );
    if let Some(fam) = m["cpuMs"].as_object() {
        out.push_str("# HELP fv_streams_cpu_ms_total CPU milliseconds by thread family.\n# TYPE fv_streams_cpu_ms_total counter\n");
        for (k, v) in fam {
            out.push_str(&format!(
                "fv_streams_cpu_ms_total{{family=\"{}\"}} {}\n",
                label(k),
                v.as_u64().unwrap_or(0)
            ));
        }
    }
    if let Some(roles) = m["taskCpuMs"].as_object() {
        out.push_str("# HELP fv_streams_task_cpu_ms_total CPU milliseconds by task role.\n# TYPE fv_streams_task_cpu_ms_total counter\n");
        for (k, v) in roles {
            out.push_str(&format!(
                "fv_streams_task_cpu_ms_total{{role=\"{}\"}} {}\n",
                label(k),
                v.as_u64().unwrap_or(0)
            ));
        }
    }
    let c = &m["checkpoint"];
    if let Some(epoch) = c["epoch"].as_u64() {
        let at_ms = c["atMs"].as_i64().unwrap_or(0);
        let age = (chrono::Utc::now().timestamp_millis() - at_ms).max(0) as f64 / 1000.0;
        out.push_str(&format!("# HELP fv_streams_checkpoint_epoch The last checkpoint's epoch.\n# TYPE fv_streams_checkpoint_epoch gauge\nfv_streams_checkpoint_epoch {epoch}\n"));
        out.push_str(&format!("# HELP fv_streams_checkpoint_age_seconds Seconds since the last checkpoint.\n# TYPE fv_streams_checkpoint_age_seconds gauge\nfv_streams_checkpoint_age_seconds {age:.3}\n"));
        out.push_str(&format!("# HELP fv_streams_checkpoint_file_bytes State file bytes in the last checkpoint.\n# TYPE fv_streams_checkpoint_file_bytes gauge\nfv_streams_checkpoint_file_bytes {}\n", c["fileBytes"].as_u64().unwrap_or(0)));
        out.push_str(&format!("# HELP fv_streams_checkpoint_took_ms How long the last checkpoint took.\n# TYPE fv_streams_checkpoint_took_ms gauge\nfv_streams_checkpoint_took_ms {}\n", c["tookMs"].as_u64().unwrap_or(0)));
    }
    let status = r["status"].as_str().unwrap_or("STARTING");
    out.push_str("# HELP fv_streams_build_status The build's status (1 for the current one).\n# TYPE fv_streams_build_status gauge\n");
    for s in ["STARTING", "RUNNING", "STOPPED", "FAILED"] {
        out.push_str(&format!(
            "fv_streams_build_status{{status=\"{s}\"}} {}\n",
            u8::from(s == status)
        ));
    }
    out
}

fn label(s: &str) -> String {
    s.replace('\\', "\\\\").replace('"', "\\\"")
}

#[cfg(test)]
mod tests {
    use super::*;
    use serde_json::json;

    #[test]
    fn renders_counters_ledgers_checkpoint_and_status() {
        let r = json!({
            "status": "RUNNING",
            "metrics": {
                "consumed": 10, "emitted": 9, "dropped": 1,
                "cpuMs": { "fv-task": 1200, "rdk:broker": 30 },
                "taskCpuMs": { "sources": 900, "operators": 300 },
                "checkpoint": { "epoch": 7, "objects": 4, "files": 0, "fileBytes": 0, "tookMs": 3, "atMs": chrono::Utc::now().timestamp_millis() }
            }
        });
        let text = render(&r);
        for needle in [
            "fv_streams_consumed_total 10",
            "fv_streams_emitted_total 9",
            "fv_streams_dropped_total 1",
            "fv_streams_cpu_ms_total{family=\"fv-task\"} 1200",
            "fv_streams_task_cpu_ms_total{role=\"sources\"} 900",
            "fv_streams_checkpoint_epoch 7",
            "fv_streams_checkpoint_took_ms 3",
            "fv_streams_build_status{status=\"RUNNING\"} 1",
            "fv_streams_build_status{status=\"FAILED\"} 0",
        ] {
            assert!(text.contains(needle), "missing {needle} in:\n{text}");
        }
        assert!(text.contains("fv_streams_checkpoint_age_seconds 0."), "{text}");
    }

    #[tokio::test]
    async fn the_endpoints_answer_from_the_live_record() {
        let (tx, rx) = watch::channel(json!({ "status": "STARTING", "metrics": {} }));
        let addr = start("127.0.0.1:0", rx).await.unwrap();
        let get = |path: &str| {
            let url = format!("http://{addr}{path}");
            async move {
                let resp = tokio::task::spawn_blocking(move || ureq::get(&url).call())
                    .await
                    .unwrap();
                match resp {
                    Ok(mut r) => (r.status().as_u16(), r.body_mut().read_to_string().unwrap_or_default()),
                    Err(ureq::Error::StatusCode(c)) => (c, String::new()),
                    Err(e) => panic!("{e}"),
                }
            }
        };
        assert_eq!(get("/health").await.0, 200);
        assert_eq!(get("/healthz").await.0, 200, "the alias");
        assert_eq!(get("/ready").await.0, 503, "not RUNNING yet");
        tx.send(json!({ "status": "RUNNING", "metrics": { "consumed": 5 } }))
            .unwrap();
        assert_eq!(get("/ready").await.0, 200);
        assert_eq!(get("/readyz").await.0, 200, "the alias");
        let (code, body) = get("/status").await;
        assert_eq!(code, 200);
        assert!(body.contains("\"consumed\":5"), "{body}");
        let (code, body) = get("/metrics").await;
        assert_eq!(code, 200);
        assert!(body.contains("fv_streams_consumed_total 5"), "{body}");
    }
}