fv-plan 0.4.0

The FusionVault transform executor on Apache DataFusion: inline row steps and relational reshapes (join, aggregate, window, union) over Arrow, with the value dialect translated into the plan, spatial SQL, file sources, and build orchestration checked against the published conformance vectors.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
//! Build orchestration — runs a pipeline's transforms in dependency order and records the result.
//!
//! Executes a pipeline's transforms in dependency order using the inline + reshape executors,
//! carries logical (materialized:false) intermediates in memory, computes expectation verdicts
//! and promotes-or-blocks. Builds run over in-memory inputs; reading sources and writing outputs
//! is the host's job. Differential-tested against the published contract's `build` conformance
//! vectors.

use std::collections::BTreeMap;

use serde_json::Value as J;

use crate::inline::apply_steps;
use crate::reshape::Input;
use crate::row::Row;
use fv_value::{compile, json_to_value, Value};

#[derive(Debug, Clone, PartialEq)]
pub struct ExpResult {
    pub dataset: String,
    pub kind: String,
    pub ok: bool,
}

#[derive(Debug, Clone)]
pub struct BuildRecord {
    pub status: String, // SUCCEEDED | BLOCKED | FAILED
    pub outputs: BTreeMap<String, Vec<Row>>,
    pub expectations: Vec<ExpResult>,
    pub error: Option<String>,
}

/// Runs a whole-transform COMPUTE step (`wasm` or `container`) — the registry backends. The
/// step SELECTS a registry transform by `ref` (`id` or `id@version`); the RUNTIME — the `fv-compute`
/// registry + its backends (fv-compute-wasm / fv-compute-container), Arrow in/out — lives in the worker
/// (fv-datafusion), so fv-plan stays pure + IO-free (it never links wasmtime or spawns processes). The
/// registry picks the impl (wasm vs container) from the resolved transform's manifest.
#[async_trait::async_trait]
pub trait ComputeRunner: Sync {
    async fn run(&self, step: &J, inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String>;
}

/// The default (no compute runtime): a `wasm`/`container` step fails closed.
pub struct NoComputeRuntime;

#[async_trait::async_trait]
impl ComputeRunner for NoComputeRuntime {
    async fn run(&self, step: &J, _inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String> {
        Err(format!(
            "compute step '{}' needs a compute runtime (none provided to run_build)",
            step["ref"].as_str().unwrap_or("?")
        ))
    }
}

/// Builds a pipeline: the pipeline and dataset definitions, the input rows, and the optional
/// runtimes for compute steps and distributed reshapes.
///
/// ```no_run
/// use fv_plan::{Build, Row};
/// # let pipeline = serde_json::json!({});
/// # let datasets: Vec<serde_json::Value> = vec![];
/// # let rows: Vec<Row> = vec![];
/// # tokio::runtime::Runtime::new().unwrap().block_on(async {
/// let record = Build::new(&pipeline, &datasets)
///     .input("orders", rows)          // one per external input dataset
///     .run()
///     .await;
/// assert!(matches!(record.status.as_str(), "SUCCEEDED" | "BLOCKED" | "FAILED"));
/// # });
/// ```
///
/// Without `.compute(...)`, a `wasm` / `container` step fails the build; without `.sql(...)`,
/// reshapes run single-node on DataFusion.
pub struct Build<'a> {
    pipeline: &'a J,
    datasets: &'a [J],
    inputs: BTreeMap<String, Vec<Row>>,
    compute: Option<&'a dyn ComputeRunner>,
    sql: Option<&'a dyn crate::reshape::SqlRunner>,
}

impl<'a> Build<'a> {
    pub fn new(pipeline: &'a J, datasets: &'a [J]) -> Self {
        Self {
            pipeline,
            datasets,
            inputs: BTreeMap::new(),
            compute: None,
            sql: None,
        }
    }

    /// Rows for one external input dataset (by `apiName`).
    pub fn input(mut self, dataset: impl Into<String>, rows: Vec<Row>) -> Self {
        self.inputs.insert(dataset.into(), rows);
        self
    }

    /// Rows for several input datasets at once.
    pub fn inputs(mut self, inputs: BTreeMap<String, Vec<Row>>) -> Self {
        self.inputs.extend(inputs);
        self
    }

    /// The runtime that executes `wasm` / `container` steps (see [`crate::compute::KineticsRunner`]).
    pub fn compute(mut self, runner: &'a dyn ComputeRunner) -> Self {
        self.compute = Some(runner);
        self
    }

    /// Where the reshape SQL runs: single-node by default, or your distributed [`crate::reshape::SqlRunner`].
    pub fn sql(mut self, runner: &'a dyn crate::reshape::SqlRunner) -> Self {
        self.sql = Some(runner);
        self
    }

    /// Run the build. Never panics: every outcome is a [`BuildRecord`] whose `status` is
    /// `SUCCEEDED`, `BLOCKED` (an expectation failed; the previous output stays current) or
    /// `FAILED` (with `error`).
    pub async fn run(self) -> BuildRecord {
        let compute = self.compute.unwrap_or(&NoComputeRuntime);
        let sql = self.sql.unwrap_or(&crate::reshape::LocalSqlRunner);
        run_build_with_native(self.pipeline, self.datasets, self.inputs, compute, sql).await
    }
}

/// Execute a pipeline over in-memory inputs (dataset apiName → rows). [`Build`] is the same thing
/// as a builder. Never panics — any failure ends
/// in a FAILED record, mirroring the reference's "a build always ends in a record" discipline.
/// `wasm`/`container` steps fail closed (no runtime); use [`run_build_with`] to supply one.
pub async fn run_build(pipeline: &J, datasets: &[J], inputs: BTreeMap<String, Vec<Row>>) -> BuildRecord {
    run_build_with(pipeline, datasets, inputs, &NoComputeRuntime).await
}

/// [`run_build`] with a [`ComputeRunner`] supplied — the worker passes its registry-backed runtime so
/// `wasm` AND `container` steps execute (the registry picks the impl per transform). Native reshape
/// ops run single-node ([`crate::reshape::LocalSqlRunner`]); use [`run_build_with_native`] to
/// distribute them.
pub async fn run_build_with(
    pipeline: &J,
    datasets: &[J],
    inputs: BTreeMap<String, Vec<Row>>,
    compute: &dyn ComputeRunner,
) -> BuildRecord {
    run_build_with_native(pipeline, datasets, inputs, compute, &crate::reshape::LocalSqlRunner).await
}

/// [`run_build_with`] plus a [`crate::reshape::SqlRunner`] for the native reshape ops — the worker
/// passes a Ballista-backed runner (gated on `FV_BALLISTA_URL`) to distribute join/aggregate/window/
/// union; the default `LocalSqlRunner` keeps them single-node. Results are identical either way.
pub async fn run_build_with_native(
    pipeline: &J,
    datasets: &[J],
    inputs: BTreeMap<String, Vec<Row>>,
    compute: &dyn ComputeRunner,
    native: &dyn crate::reshape::SqlRunner,
) -> BuildRecord {
    let mut rec = BuildRecord {
        status: "RUNNING".into(),
        outputs: BTreeMap::new(),
        expectations: Vec::new(),
        error: None,
    };
    match run_inner(pipeline, datasets, inputs, compute, native, &mut rec).await {
        Ok(status) => {
            rec.status = status;
        }
        Err(e) => {
            rec.status = "FAILED".into();
            rec.error = Some(e);
        }
    }
    rec
}

async fn run_inner(
    pipeline: &J,
    datasets: &[J],
    inputs: BTreeMap<String, Vec<Row>>,
    compute: &dyn ComputeRunner,
    native: &dyn crate::reshape::SqlRunner,
    rec: &mut BuildRecord,
) -> Result<String, String> {
    let by_name: BTreeMap<&str, &J> = datasets
        .iter()
        .map(|d| (d["apiName"].as_str().unwrap_or(""), d))
        .collect();
    let transforms = topo_transforms(pipeline)?;

    let mut logical: BTreeMap<String, Vec<Row>> = BTreeMap::new();

    for t in &transforms {
        let out_name = t["output"].as_str().ok_or("transform missing output")?;
        let out_ds = *by_name
            .get(out_name)
            .ok_or_else(|| format!("output dataset '{out_name}' unknown"))?;
        let out_logical = out_ds.get("materialized") == Some(&J::Bool(false));

        // Resolve each input's rows (external map / promoted output / logical buffer) + its columns.
        let mut in_rows: Vec<(String, Vec<Row>)> = Vec::new();
        for inp in t["inputs"].as_array().ok_or("transform inputs must be an array")? {
            let name = inp.as_str().ok_or("input name must be a string")?;
            let rows = logical
                .get(name)
                .or_else(|| rec.outputs.get(name))
                .or_else(|| inputs.get(name))
                .cloned()
                .ok_or_else(|| format!("input '{name}' has no rows"))?;
            in_rows.push((name.to_string(), rows));
        }

        let steps = t["steps"].as_array().cloned().unwrap_or_default();
        if let Some(bad) = steps.iter().find(|s| s["op"].as_str() == Some("sql")) {
            return Err(format!(
                "step op '{}' needs a connectivity runtime and is not executed here",
                bad["op"]
            ));
        }
        let out_rows = run_steps(&steps, in_rows, compute, native).await?;

        if out_logical {
            logical.insert(out_name.to_string(), out_rows);
            continue;
        }

        // Materialized output: evaluate expectations, then promote-or-block.
        let expectations = out_ds["expectations"].as_array().cloned().unwrap_or_default();
        let mut all_ok = true;
        for e in &expectations {
            let (kind, ok) = evaluate_expectation(e, &out_rows, now_ms())?;
            all_ok &= ok;
            rec.expectations.push(ExpResult {
                dataset: out_name.to_string(),
                kind,
                ok,
            });
        }
        if !all_ok {
            rec.error = Some(format!(
                "expectations failed for '{out_name}' — previous output stays current"
            ));
            return Ok("BLOCKED".into());
        }
        rec.outputs.insert(out_name.to_string(), out_rows);
    }

    Ok("SUCCEEDED".into())
}

/// Whether `step` runs over whole inputs rather than row by row: a native reshape (join,
/// aggregate, window, union) or a registry-backed compute (`wasm`, `container`).
fn is_whole_input(step: &J) -> bool {
    let op = step["op"].as_str().unwrap_or("");
    crate::reshape::NATIVE_OPS.contains(&op) || matches!(op, "wasm" | "container")
}

/// One transform's steps, as FusionVault Streams runs a stage: row-by-row steps, then at most one
/// reshape or compute step, then row-by-row steps on its output. A second reshape or compute step
/// is refused, and so is a row-by-row step before one that reads several inputs — it could not say
/// which input it applies to.
async fn run_steps(
    steps: &[J],
    mut in_rows: Vec<(String, Vec<Row>)>,
    compute: &dyn ComputeRunner,
    native: &dyn crate::reshape::SqlRunner,
) -> Result<Vec<Row>, String> {
    let parse = |slice: &[J]| slice.iter().map(crate::inline::parse).collect::<Result<Vec<_>, _>>();
    let whole: Vec<usize> = (0..steps.len()).filter(|&i| is_whole_input(&steps[i])).collect();
    let Some(&at) = whole.first() else {
        let rows = in_rows
            .first()
            .map(|(_, r)| r.as_slice())
            .ok_or("a transform needs an input")?;
        return apply_steps(&parse(steps)?, rows).map_err(|e| e.to_string());
    };
    let op = |i: usize| steps[i]["op"].as_str().unwrap_or("");
    if whole.len() > 1 {
        return Err(format!(
            "a transform may contain at most one reshape or compute step (found {}: {}) — split it into chained transforms",
            whole.len(),
            whole.iter().map(|&i| op(i)).collect::<Vec<_>>().join(", ")
        ));
    }
    let (before, after) = (parse(&steps[..at])?, parse(&steps[at + 1..])?);
    if !before.is_empty() {
        if in_rows.len() != 1 {
            return Err(format!(
                "a step before '{}' cannot say which of its {} inputs it applies to — transform each input in its own preceding transform",
                op(at),
                in_rows.len()
            ));
        }
        in_rows[0].1 = apply_steps(&before, &in_rows[0].1).map_err(|e| e.to_string())?;
    }
    let step = &steps[at];
    let out = if matches!(op(at), "wasm" | "container") {
        compute.run(step, &in_rows).await?
    } else {
        let inputs: Vec<Input> = in_rows.iter().map(|(n, r)| input_of(n, r)).collect();
        crate::reshape::run_with(step, inputs, native).await?
    };
    if after.is_empty() {
        Ok(out)
    } else {
        apply_steps(&after, &out).map_err(|e| e.to_string())
    }
}

/// Producers before consumers (the control-plane gate already rejects cycles; we fail closed anyway).
fn topo_transforms(pipeline: &J) -> Result<Vec<J>, String> {
    let transforms = pipeline["transforms"].as_array().cloned().unwrap_or_default();
    let producer: BTreeMap<String, usize> = transforms
        .iter()
        .enumerate()
        .filter_map(|(i, t)| t["output"].as_str().map(|o| (o.to_string(), i)))
        .collect();
    let mut done = vec![false; transforms.len()];
    let mut order = Vec::new();

    fn visit(
        i: usize,
        transforms: &[J],
        producer: &BTreeMap<String, usize>,
        done: &mut [bool],
        path: &mut Vec<usize>,
        order: &mut Vec<J>,
    ) -> Result<(), String> {
        if done[i] {
            return Ok(());
        }
        if path.contains(&i) {
            return Err("transform cycle".into());
        }
        path.push(i);
        if let Some(inputs) = transforms[i]["inputs"].as_array() {
            for inp in inputs {
                if let Some(name) = inp.as_str() {
                    if let Some(&p) = producer.get(name) {
                        if p != i {
                            visit(p, transforms, producer, done, path, order)?;
                        }
                    }
                }
            }
        }
        path.pop();
        done[i] = true;
        order.push(transforms[i].clone());
        Ok(())
    }

    for i in 0..transforms.len() {
        let mut path = Vec::new();
        visit(i, &transforms, &producer, &mut done, &mut path, &mut order)?;
    }
    Ok(order)
}

fn input_of(name: &str, rows: &[Row]) -> Input {
    // Columns = first-seen union of keys across the rows.
    let mut columns: Vec<String> = Vec::new();
    for r in rows {
        for (k, _) in &r.0 {
            if !columns.contains(k) {
                columns.push(k.clone());
            }
        }
    }
    let _ = name;
    Input {
        columns,
        rows: rows.to_vec(),
    }
}

fn str_list(j: &J) -> Vec<String> {
    j.as_array()
        .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
        .unwrap_or_default()
}

// Wall-clock in epoch-ms (injected into `evaluate_expectation` for the freshness verdict).
use crate::now_ms;

/// Port of `expectations.py::evaluate_expectation` (verdict only).
fn evaluate_expectation(e: &J, rows: &[Row], now: i64) -> Result<(String, bool), String> {
    let kind = e["kind"].as_str().unwrap_or("").to_string();
    let ok = match kind.as_str() {
        "rowCountMin" => rows.len() as i64 >= e["min"].as_i64().unwrap_or(i64::MAX),
        "rowCountMax" => rows.len() as i64 <= e["max"].as_i64().unwrap_or(i64::MIN),
        "notNull" => {
            let col = e["column"].as_str().unwrap_or("");
            rows.iter().all(|r| !matches!(r.get(col), Value::Null))
        }
        "uniquePk" => {
            let cols = str_list(&e["columns"]);
            let mut seen = std::collections::HashSet::new();
            rows.iter()
                .all(|r| seen.insert(format!("{:?}", cols.iter().map(|c| r.get(c)).collect::<Vec<_>>())))
        }
        "enumDomain" => {
            let col = e["column"].as_str().unwrap_or("");
            let allowed: Vec<Value> = e["values"]
                .as_array()
                .map(|a| a.iter().map(json_to_value).collect())
                .unwrap_or_default();
            rows.iter().all(|r| {
                let v = r.get(col);
                matches!(v, Value::Null) || allowed.iter().any(|a| a == &v)
            })
        }
        "expression" => {
            let ast = compile(e["expression"].as_str().unwrap_or("")).map_err(|err| err.to_string())?;
            rows.iter()
                .all(|r| matches!(ast.eval(r.lookup()), Ok(Value::Bool(true))))
        }
        // freshness: newest timestamp in `column` must be no older than maxAgeMs. Timestamps parse via
        // the value dialect's epochMs; no parseable timestamp (or no maxAgeMs) → fail-closed.
        "freshness" => {
            let col = e["column"].as_str().unwrap_or("");
            let ast = compile(&format!("epochMs({col})")).map_err(|err| err.to_string())?;
            let mut max_ms: Option<i64> = None;
            for r in rows {
                if let Ok(Value::Num(ms)) = ast.eval(r.lookup()) {
                    let m = ms as i64;
                    max_ms = Some(max_ms.map_or(m, |x| x.max(m)));
                }
            }
            match (max_ms, e["maxAgeMs"].as_i64()) {
                (Some(mx), Some(age)) => (now - mx) <= age,
                _ => false,
            }
        }
        other => return Err(format!("unknown expectation kind '{other}' (fail-closed)")),
    };
    Ok((kind, ok))
}

#[cfg(test)]
mod tests {
    use super::*;
    use crate::row::Row;

    #[test]
    fn freshness_verdict() {
        let base = 1_787_140_800_000i64; // epochMs("2026-08-19T12:00:00Z")
        let rows = vec![Row(vec![(
            "updatedAt".into(),
            Value::Str("2026-08-19T12:00:00Z".into()),
        )])];
        let e = serde_json::json!({"kind":"freshness","column":"updatedAt","maxAgeMs":5000});
        assert!(
            evaluate_expectation(&e, &rows, base + 1000).unwrap().1,
            "age 1000ms <= 5000ms"
        );
        assert!(
            !evaluate_expectation(&e, &rows, base + 9000).unwrap().1,
            "age 9000ms > 5000ms"
        );
        let empty: Vec<Row> = vec![];
        assert!(
            !evaluate_expectation(&e, &empty, base).unwrap().1,
            "no timestamp → fail-closed"
        );
    }

    /// A mock compute runtime that appends a `total` column — proves `run_build_with` delegates a
    /// `wasm`/`container` step to the ComputeRunner and uses its output (the real runtime = the
    /// fv-compute registry + backends in the worker).
    struct DoublerCompute;

    #[async_trait::async_trait]
    impl ComputeRunner for DoublerCompute {
        async fn run(&self, step: &J, inputs: &[(String, Vec<Row>)]) -> Result<Vec<Row>, String> {
            assert_eq!(step["ref"].as_str(), Some("spikeTotal"));
            Ok(inputs[0]
                .1
                .iter()
                .map(|r| {
                    let mut cols = r.0.clone();
                    let n = match r.get("amount") {
                        Value::Num(n) => n,
                        _ => 0.0,
                    };
                    cols.push(("total".into(), Value::Num(n * 2.0)));
                    Row(cols)
                })
                .collect())
        }
    }

    fn compute_pipeline(op: &str) -> (J, Vec<J>, BTreeMap<String, Vec<Row>>) {
        let pipeline = serde_json::json!({
            "transforms": [{ "output": "scored", "inputs": ["raw"], "steps": [{ "op": op, "ref": "spikeTotal" }] }]
        });
        let datasets = vec![serde_json::json!({ "apiName": "scored", "materialized": true, "expectations": [] })];
        let inputs = BTreeMap::from([("raw".to_string(), vec![Row(vec![("amount".into(), Value::Num(2.5))])])]);
        (pipeline, datasets, inputs)
    }

    #[tokio::test]
    async fn wasm_and_container_steps_both_delegate_to_the_compute_runner() {
        for op in ["wasm", "container"] {
            let (pipeline, datasets, inputs) = compute_pipeline(op);
            let rec = run_build_with(&pipeline, &datasets, inputs, &DoublerCompute).await;
            assert_eq!(rec.status, "SUCCEEDED", "op={op}: {:?}", rec.error);
            assert_eq!(rec.outputs["scored"][0].get("total"), Value::Num(5.0), "op={op}");
        }
    }

    #[tokio::test]
    async fn compute_step_fails_closed_without_a_runtime() {
        let (pipeline, datasets, inputs) = compute_pipeline("container");
        let rec = run_build(&pipeline, &datasets, inputs).await; // NoComputeRuntime
        assert_eq!(rec.status, "FAILED");
        assert!(rec.error.unwrap().contains("needs a compute runtime"));
    }
}