use std::sync::Arc;
use axum::extract::State;
use axum::http::StatusCode;
use axum::response::IntoResponse;
use axum::routing::get;
use axum::{Json, Router};
use serde_json::Value as J;
use tokio::sync::watch;
pub type Live = watch::Receiver<J>;
pub async fn start(addr: &str, live: Live) -> Result<std::net::SocketAddr, String> {
let app = Router::new()
.route("/health", get(|| async { "ok" }))
.route("/healthz", get(|| async { "ok" }))
.route("/ready", get(readyz))
.route("/readyz", get(readyz))
.route("/status", get(status))
.route("/metrics", get(metrics))
.with_state(Arc::new(live));
let listener = tokio::net::TcpListener::bind(addr)
.await
.map_err(|e| format!("--serve {addr}: {e}"))?;
let bound = listener.local_addr().map_err(|e| e.to_string())?;
tokio::spawn(async move {
let _ = axum::serve(listener, app).await;
});
Ok(bound)
}
async fn readyz(State(live): State<Arc<Live>>) -> impl IntoResponse {
let r = live.borrow().clone();
let status = r["status"].as_str().unwrap_or("STARTING").to_string();
if status == "RUNNING" {
(StatusCode::OK, status)
} else {
(StatusCode::SERVICE_UNAVAILABLE, status)
}
}
async fn status(State(live): State<Arc<Live>>) -> Json<J> {
Json(live.borrow().clone())
}
async fn metrics(State(live): State<Arc<Live>>) -> impl IntoResponse {
let r = live.borrow().clone();
([("content-type", "text/plain; version=0.0.4")], render(&r))
}
pub fn render(r: &J) -> String {
let m = &r["metrics"];
let mut out = String::with_capacity(2048);
let mut counter = |name: &str, help: &str, v: u64| {
out.push_str(&format!("# HELP {name} {help}\n# TYPE {name} counter\n{name} {v}\n"));
};
counter(
"fv_streams_consumed_total",
"Rows the sources produced.",
m["consumed"].as_u64().unwrap_or(0),
);
counter(
"fv_streams_emitted_total",
"Rows the sinks wrote.",
m["emitted"].as_u64().unwrap_or(0),
);
counter(
"fv_streams_dropped_total",
"Rows dropped as bad data.",
m["dropped"].as_u64().unwrap_or(0),
);
if let Some(fam) = m["cpuMs"].as_object() {
out.push_str("# HELP fv_streams_cpu_ms_total CPU milliseconds by thread family.\n# TYPE fv_streams_cpu_ms_total counter\n");
for (k, v) in fam {
out.push_str(&format!(
"fv_streams_cpu_ms_total{{family=\"{}\"}} {}\n",
label(k),
v.as_u64().unwrap_or(0)
));
}
}
if let Some(roles) = m["taskCpuMs"].as_object() {
out.push_str("# HELP fv_streams_task_cpu_ms_total CPU milliseconds by task role.\n# TYPE fv_streams_task_cpu_ms_total counter\n");
for (k, v) in roles {
out.push_str(&format!(
"fv_streams_task_cpu_ms_total{{role=\"{}\"}} {}\n",
label(k),
v.as_u64().unwrap_or(0)
));
}
}
let c = &m["checkpoint"];
if let Some(epoch) = c["epoch"].as_u64() {
let at_ms = c["atMs"].as_i64().unwrap_or(0);
let age = (chrono::Utc::now().timestamp_millis() - at_ms).max(0) as f64 / 1000.0;
out.push_str(&format!("# HELP fv_streams_checkpoint_epoch The last checkpoint's epoch.\n# TYPE fv_streams_checkpoint_epoch gauge\nfv_streams_checkpoint_epoch {epoch}\n"));
out.push_str(&format!("# HELP fv_streams_checkpoint_age_seconds Seconds since the last checkpoint.\n# TYPE fv_streams_checkpoint_age_seconds gauge\nfv_streams_checkpoint_age_seconds {age:.3}\n"));
out.push_str(&format!("# HELP fv_streams_checkpoint_file_bytes State file bytes in the last checkpoint.\n# TYPE fv_streams_checkpoint_file_bytes gauge\nfv_streams_checkpoint_file_bytes {}\n", c["fileBytes"].as_u64().unwrap_or(0)));
out.push_str(&format!("# HELP fv_streams_checkpoint_took_ms How long writing the last checkpoint took.\n# TYPE fv_streams_checkpoint_took_ms gauge\nfv_streams_checkpoint_took_ms {}\n", c["tookMs"].as_u64().unwrap_or(0)));
if let Some(ms) = c["durationMs"].as_u64() {
out.push_str(&format!("# HELP fv_streams_checkpoint_duration_ms How long the last checkpoint took from its barrier to durable.\n# TYPE fv_streams_checkpoint_duration_ms gauge\nfv_streams_checkpoint_duration_ms {ms}\n"));
out.push_str(&format!("# HELP fv_streams_checkpoint_over_interval 1 when the last checkpoint took longer than its interval, so the next one waited for it.\n# TYPE fv_streams_checkpoint_over_interval gauge\nfv_streams_checkpoint_over_interval {}\n", u8::from(c["overInterval"].as_bool().unwrap_or(false))));
}
}
let status = r["status"].as_str().unwrap_or("STARTING");
out.push_str("# HELP fv_streams_build_status The build's status (1 for the current one).\n# TYPE fv_streams_build_status gauge\n");
for s in ["STARTING", "RUNNING", "STOPPED", "FAILED"] {
out.push_str(&format!(
"fv_streams_build_status{{status=\"{s}\"}} {}\n",
u8::from(s == status)
));
}
out
}
fn label(s: &str) -> String {
s.replace('\\', "\\\\").replace('"', "\\\"")
}
#[cfg(test)]
mod tests {
use super::*;
use serde_json::json;
#[test]
fn renders_counters_ledgers_checkpoint_and_status() {
let r = json!({
"status": "RUNNING",
"metrics": {
"consumed": 10, "emitted": 9, "dropped": 1,
"cpuMs": { "fv-task": 1200, "rdk:broker": 30 },
"taskCpuMs": { "sources": 900, "operators": 300 },
"checkpoint": { "epoch": 7, "objects": 4, "files": 0, "fileBytes": 0, "tookMs": 3, "atMs": chrono::Utc::now().timestamp_millis(), "durationMs": 2400, "overInterval": true }
}
});
let text = render(&r);
for needle in [
"fv_streams_consumed_total 10",
"fv_streams_emitted_total 9",
"fv_streams_dropped_total 1",
"fv_streams_cpu_ms_total{family=\"fv-task\"} 1200",
"fv_streams_task_cpu_ms_total{role=\"sources\"} 900",
"fv_streams_checkpoint_epoch 7",
"fv_streams_checkpoint_took_ms 3",
"fv_streams_checkpoint_duration_ms 2400",
"fv_streams_checkpoint_over_interval 1",
"fv_streams_build_status{status=\"RUNNING\"} 1",
"fv_streams_build_status{status=\"FAILED\"} 0",
] {
assert!(text.contains(needle), "missing {needle} in:\n{text}");
}
assert!(text.contains("fv_streams_checkpoint_age_seconds 0."), "{text}");
let mut r = r;
r["metrics"]["checkpoint"]["durationMs"] = serde_json::Value::Null;
let text = render(&r);
assert!(text.contains("fv_streams_checkpoint_epoch 7"), "{text}");
assert!(!text.contains("fv_streams_checkpoint_duration_ms"), "{text}");
}
#[tokio::test]
async fn the_endpoints_answer_from_the_live_record() {
let (tx, rx) = watch::channel(json!({ "status": "STARTING", "metrics": {} }));
let addr = start("127.0.0.1:0", rx).await.unwrap();
let get = |path: &str| {
let url = format!("http://{addr}{path}");
async move {
let resp = tokio::task::spawn_blocking(move || ureq::get(&url).call())
.await
.unwrap();
match resp {
Ok(mut r) => (r.status().as_u16(), r.body_mut().read_to_string().unwrap_or_default()),
Err(ureq::Error::StatusCode(c)) => (c, String::new()),
Err(e) => panic!("{e}"),
}
}
};
assert_eq!(get("/health").await.0, 200);
assert_eq!(get("/healthz").await.0, 200, "the alias");
assert_eq!(get("/ready").await.0, 503, "not RUNNING yet");
tx.send(json!({ "status": "RUNNING", "metrics": { "consumed": 5 } }))
.unwrap();
assert_eq!(get("/ready").await.0, 200);
assert_eq!(get("/readyz").await.0, 200, "the alias");
let (code, body) = get("/status").await;
assert_eq!(code, 200);
assert!(body.contains("\"consumed\":5"), "{body}");
let (code, body) = get("/metrics").await;
assert_eq!(code, 200);
assert!(body.contains("fv_streams_consumed_total 5"), "{body}");
}
}