use std::collections::BTreeMap;
use serde_json::Value as J;
use crate::inline::apply_steps;
use crate::reshape::Input;
use crate::row::Row;
use fv_value::{compile, json_to_value, Value};
#[derive(Debug, Clone, PartialEq)]
pub struct ExpResult {
pub dataset: String,
pub kind: String,
pub ok: bool,
}
#[derive(Debug, Clone)]
pub struct BuildRecord {
pub status: String, pub outputs: BTreeMap<String, Vec<Row>>,
pub expectations: Vec<ExpResult>,
pub error: Option<String>,
}
#[async_trait::async_trait]
pub trait ComputeRunner: Sync {
async fn run(&self, step: &J, inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String>;
}
pub struct NoComputeRuntime;
#[async_trait::async_trait]
impl ComputeRunner for NoComputeRuntime {
async fn run(&self, step: &J, _inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String> {
Err(format!(
"compute step '{}' needs a compute runtime (none provided to run_build)",
step["ref"].as_str().unwrap_or("?")
))
}
}
pub struct Build<'a> {
pipeline: &'a J,
datasets: &'a [J],
inputs: BTreeMap<String, Vec<Row>>,
compute: Option<&'a dyn ComputeRunner>,
sql: Option<&'a dyn crate::reshape::SqlRunner>,
}
impl<'a> Build<'a> {
pub fn new(pipeline: &'a J, datasets: &'a [J]) -> Self {
Self {
pipeline,
datasets,
inputs: BTreeMap::new(),
compute: None,
sql: None,
}
}
pub fn input(mut self, dataset: impl Into<String>, rows: Vec<Row>) -> Self {
self.inputs.insert(dataset.into(), rows);
self
}
pub fn inputs(mut self, inputs: BTreeMap<String, Vec<Row>>) -> Self {
self.inputs.extend(inputs);
self
}
pub fn compute(mut self, runner: &'a dyn ComputeRunner) -> Self {
self.compute = Some(runner);
self
}
pub fn sql(mut self, runner: &'a dyn crate::reshape::SqlRunner) -> Self {
self.sql = Some(runner);
self
}
pub async fn run(self) -> BuildRecord {
let compute = self.compute.unwrap_or(&NoComputeRuntime);
let sql = self.sql.unwrap_or(&crate::reshape::LocalSqlRunner);
run_build_with_native(self.pipeline, self.datasets, self.inputs, compute, sql).await
}
}
pub async fn run_build(pipeline: &J, datasets: &[J], inputs: BTreeMap<String, Vec<Row>>) -> BuildRecord {
run_build_with(pipeline, datasets, inputs, &NoComputeRuntime).await
}
pub async fn run_build_with(
pipeline: &J,
datasets: &[J],
inputs: BTreeMap<String, Vec<Row>>,
compute: &dyn ComputeRunner,
) -> BuildRecord {
run_build_with_native(pipeline, datasets, inputs, compute, &crate::reshape::LocalSqlRunner).await
}
pub async fn run_build_with_native(
pipeline: &J,
datasets: &[J],
inputs: BTreeMap<String, Vec<Row>>,
compute: &dyn ComputeRunner,
native: &dyn crate::reshape::SqlRunner,
) -> BuildRecord {
let mut rec = BuildRecord {
status: "RUNNING".into(),
outputs: BTreeMap::new(),
expectations: Vec::new(),
error: None,
};
match run_inner(pipeline, datasets, inputs, compute, native, &mut rec).await {
Ok(status) => {
rec.status = status;
}
Err(e) => {
rec.status = "FAILED".into();
rec.error = Some(e);
}
}
rec
}
async fn run_inner(
pipeline: &J,
datasets: &[J],
inputs: BTreeMap<String, Vec<Row>>,
compute: &dyn ComputeRunner,
native: &dyn crate::reshape::SqlRunner,
rec: &mut BuildRecord,
) -> Result<String, String> {
let by_name: BTreeMap<&str, &J> = datasets
.iter()
.map(|d| (d["apiName"].as_str().unwrap_or(""), d))
.collect();
let transforms = topo_transforms(pipeline)?;
let mut logical: BTreeMap<String, Vec<Row>> = BTreeMap::new();
for t in &transforms {
let out_name = t["output"].as_str().ok_or("transform missing output")?;
let out_ds = *by_name
.get(out_name)
.ok_or_else(|| format!("output dataset '{out_name}' unknown"))?;
let out_logical = out_ds.get("materialized") == Some(&J::Bool(false));
let mut in_rows: Vec<(String, Vec<Row>)> = Vec::new();
for inp in t["inputs"].as_array().ok_or("transform inputs must be an array")? {
let name = inp.as_str().ok_or("input name must be a string")?;
let rows = logical
.get(name)
.or_else(|| rec.outputs.get(name))
.or_else(|| inputs.get(name))
.cloned()
.ok_or_else(|| format!("input '{name}' has no rows"))?;
in_rows.push((name.to_string(), rows));
}
let steps = t["steps"].as_array().cloned().unwrap_or_default();
if let Some(bad) = steps.iter().find(|s| s["op"].as_str() == Some("sql")) {
return Err(format!(
"step op '{}' needs a connectivity runtime and is not executed here",
bad["op"]
));
}
let out_rows = run_steps(&steps, in_rows, compute, native).await?;
if out_logical {
logical.insert(out_name.to_string(), out_rows);
continue;
}
let expectations = out_ds["expectations"].as_array().cloned().unwrap_or_default();
let mut all_ok = true;
for e in &expectations {
let (kind, ok) = evaluate_expectation(e, &out_rows, now_ms())?;
all_ok &= ok;
rec.expectations.push(ExpResult {
dataset: out_name.to_string(),
kind,
ok,
});
}
if !all_ok {
rec.error = Some(format!(
"expectations failed for '{out_name}' — previous output stays current"
));
return Ok("BLOCKED".into());
}
rec.outputs.insert(out_name.to_string(), out_rows);
}
Ok("SUCCEEDED".into())
}
fn is_whole_input(step: &J) -> bool {
let op = step["op"].as_str().unwrap_or("");
crate::reshape::NATIVE_OPS.contains(&op) || matches!(op, "wasm" | "container")
}
async fn run_steps(
steps: &[J],
mut in_rows: Vec<(String, Vec<Row>)>,
compute: &dyn ComputeRunner,
native: &dyn crate::reshape::SqlRunner,
) -> Result<Vec<Row>, String> {
let parse = |slice: &[J]| slice.iter().map(crate::inline::parse).collect::<Result<Vec<_>, _>>();
let whole: Vec<usize> = (0..steps.len()).filter(|&i| is_whole_input(&steps[i])).collect();
let Some(&at) = whole.first() else {
let rows = in_rows
.first()
.map(|(_, r)| r.as_slice())
.ok_or("a transform needs an input")?;
return apply_steps(&parse(steps)?, rows).map_err(|e| e.to_string());
};
let op = |i: usize| steps[i]["op"].as_str().unwrap_or("");
if whole.len() > 1 {
return Err(format!(
"a transform may contain at most one reshape or compute step (found {}: {}) — split it into chained transforms",
whole.len(),
whole.iter().map(|&i| op(i)).collect::<Vec<_>>().join(", ")
));
}
let (before, after) = (parse(&steps[..at])?, parse(&steps[at + 1..])?);
if !before.is_empty() {
if in_rows.len() != 1 {
return Err(format!(
"a step before '{}' cannot say which of its {} inputs it applies to — transform each input in its own preceding transform",
op(at),
in_rows.len()
));
}
in_rows[0].1 = apply_steps(&before, &in_rows[0].1).map_err(|e| e.to_string())?;
}
let step = &steps[at];
let out = if matches!(op(at), "wasm" | "container") {
compute.run(step, &in_rows).await?
} else {
let inputs: Vec<Input> = in_rows.iter().map(|(n, r)| input_of(n, r)).collect();
crate::reshape::run_with(step, inputs, native).await?
};
if after.is_empty() {
Ok(out)
} else {
apply_steps(&after, &out).map_err(|e| e.to_string())
}
}
fn topo_transforms(pipeline: &J) -> Result<Vec<J>, String> {
let transforms = pipeline["transforms"].as_array().cloned().unwrap_or_default();
let producer: BTreeMap<String, usize> = transforms
.iter()
.enumerate()
.filter_map(|(i, t)| t["output"].as_str().map(|o| (o.to_string(), i)))
.collect();
let mut done = vec![false; transforms.len()];
let mut order = Vec::new();
fn visit(
i: usize,
transforms: &[J],
producer: &BTreeMap<String, usize>,
done: &mut [bool],
path: &mut Vec<usize>,
order: &mut Vec<J>,
) -> Result<(), String> {
if done[i] {
return Ok(());
}
if path.contains(&i) {
return Err("transform cycle".into());
}
path.push(i);
if let Some(inputs) = transforms[i]["inputs"].as_array() {
for inp in inputs {
if let Some(name) = inp.as_str() {
if let Some(&p) = producer.get(name) {
if p != i {
visit(p, transforms, producer, done, path, order)?;
}
}
}
}
}
path.pop();
done[i] = true;
order.push(transforms[i].clone());
Ok(())
}
for i in 0..transforms.len() {
let mut path = Vec::new();
visit(i, &transforms, &producer, &mut done, &mut path, &mut order)?;
}
Ok(order)
}
fn input_of(name: &str, rows: &[Row]) -> Input {
let mut columns: Vec<String> = Vec::new();
for r in rows {
for (k, _) in &r.0 {
if !columns.contains(k) {
columns.push(k.clone());
}
}
}
let _ = name;
Input {
columns,
rows: rows.to_vec(),
}
}
fn str_list(j: &J) -> Vec<String> {
j.as_array()
.map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
.unwrap_or_default()
}
use crate::now_ms;
fn evaluate_expectation(e: &J, rows: &[Row], now: i64) -> Result<(String, bool), String> {
let kind = e["kind"].as_str().unwrap_or("").to_string();
let ok = match kind.as_str() {
"rowCountMin" => rows.len() as i64 >= e["min"].as_i64().unwrap_or(i64::MAX),
"rowCountMax" => rows.len() as i64 <= e["max"].as_i64().unwrap_or(i64::MIN),
"notNull" => {
let col = e["column"].as_str().unwrap_or("");
rows.iter().all(|r| !matches!(r.get(col), Value::Null))
}
"uniquePk" => {
let cols = str_list(&e["columns"]);
let mut seen = std::collections::HashSet::new();
rows.iter()
.all(|r| seen.insert(format!("{:?}", cols.iter().map(|c| r.get(c)).collect::<Vec<_>>())))
}
"enumDomain" => {
let col = e["column"].as_str().unwrap_or("");
let allowed: Vec<Value> = e["values"]
.as_array()
.map(|a| a.iter().map(json_to_value).collect())
.unwrap_or_default();
rows.iter().all(|r| {
let v = r.get(col);
matches!(v, Value::Null) || allowed.iter().any(|a| a == &v)
})
}
"expression" => {
let ast = compile(e["expression"].as_str().unwrap_or("")).map_err(|err| err.to_string())?;
rows.iter()
.all(|r| matches!(ast.eval(r.lookup()), Ok(Value::Bool(true))))
}
"freshness" => {
let col = e["column"].as_str().unwrap_or("");
let ast = compile(&format!("epochMs({col})")).map_err(|err| err.to_string())?;
let mut max_ms: Option<i64> = None;
for r in rows {
if let Ok(Value::Num(ms)) = ast.eval(r.lookup()) {
let m = ms as i64;
max_ms = Some(max_ms.map_or(m, |x| x.max(m)));
}
}
match (max_ms, e["maxAgeMs"].as_i64()) {
(Some(mx), Some(age)) => (now - mx) <= age,
_ => false,
}
}
other => return Err(format!("unknown expectation kind '{other}' (fail-closed)")),
};
Ok((kind, ok))
}
#[cfg(test)]
mod tests {
use super::*;
use crate::row::Row;
#[test]
fn freshness_verdict() {
let base = 1_787_140_800_000i64; let rows = vec![Row(vec![(
"updatedAt".into(),
Value::Str("2026-08-19T12:00:00Z".into()),
)])];
let e = serde_json::json!({"kind":"freshness","column":"updatedAt","maxAgeMs":5000});
assert!(
evaluate_expectation(&e, &rows, base + 1000).unwrap().1,
"age 1000ms <= 5000ms"
);
assert!(
!evaluate_expectation(&e, &rows, base + 9000).unwrap().1,
"age 9000ms > 5000ms"
);
let empty: Vec<Row> = vec![];
assert!(
!evaluate_expectation(&e, &empty, base).unwrap().1,
"no timestamp → fail-closed"
);
}
struct DoublerCompute;
#[async_trait::async_trait]
impl ComputeRunner for DoublerCompute {
async fn run(&self, step: &J, inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String> {
assert_eq!(step["ref"].as_str(), Some("spikeTotal"));
Ok(inputs[0]
.1
.iter()
.map(|r| {
let mut cols = r.0.clone();
let n = match r.get("amount") {
Value::Num(n) => n,
_ => 0.0,
};
cols.push(("total".into(), Value::Num(n * 2.0)));
Row(cols)
})
.collect())
}
}
fn compute_pipeline(op: &str) -> (J, Vec<J>, BTreeMap<String, Vec<Row>>) {
let pipeline = serde_json::json!({
"transforms": [{ "output": "scored", "inputs": ["raw"], "steps": [{ "op": op, "ref": "spikeTotal" }] }]
});
let datasets = vec![serde_json::json!({ "apiName": "scored", "materialized": true, "expectations": [] })];
let inputs = BTreeMap::from([("raw".to_string(), vec![Row(vec![("amount".into(), Value::Num(2.5))])])]);
(pipeline, datasets, inputs)
}
#[tokio::test]
async fn wasm_and_container_steps_both_delegate_to_the_compute_runner() {
for op in ["wasm", "container"] {
let (pipeline, datasets, inputs) = compute_pipeline(op);
let rec = run_build_with(&pipeline, &datasets, inputs, &DoublerCompute).await;
assert_eq!(rec.status, "SUCCEEDED", "op={op}: {:?}", rec.error);
assert_eq!(rec.outputs["scored"][0].get("total"), Value::Num(5.0), "op={op}");
}
}
#[tokio::test]
async fn compute_step_fails_closed_without_a_runtime() {
let (pipeline, datasets, inputs) = compute_pipeline("container");
let rec = run_build(&pipeline, &datasets, inputs).await; assert_eq!(rec.status, "FAILED");
assert!(rec.error.unwrap().contains("needs a compute runtime"));
}
}