fv-plan 0.3.0

The FusionVault transform executor on Apache DataFusion: inline row steps and relational reshapes (join, aggregate, window, union) over Arrow, with the value dialect translated into the plan, spatial SQL, file sources, and build orchestration checked against the published conformance vectors.
Documentation
//! Inline row-op executor. Executes a chain of row ops (select/rename/drop/filter/applyExpression)
//! row-wise, delegating value compute to `fv-value`.
//! Vectored against the published contract's `step-kind` conformance vectors.

use crate::row::Row;
use fv_value::{compile, ExprError, Value};

/// A pipeline step as decoded JSON (op + its params), matching the control-plane spec.
#[derive(Debug, Clone)]
pub enum Step {
    Select { columns: Vec<String> },
    Rename { mapping: Vec<(String, String)> },
    Drop { columns: Vec<String> },
    Filter { expression: String },
    ApplyExpression { column: String, expression: String },
}

#[derive(Debug)]
pub enum StepError {
    Expr(ExprError),
    Message(String),
}

impl std::fmt::Display for StepError {
    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
        match self {
            StepError::Expr(e) => write!(f, "{e}"),
            StepError::Message(m) => write!(f, "{m}"),
        }
    }
}

impl From<ExprError> for StepError {
    fn from(e: ExprError) -> Self {
        StepError::Expr(e)
    }
}

/// Apply one step to a set of rows, producing the transformed rows.
pub fn apply_step(step: &Step, rows: &[Row]) -> Result<Vec<Row>, StepError> {
    match step {
        // Project to exactly `columns`, in order; a missing column lands null.
        Step::Select { columns } => Ok(rows
            .iter()
            .map(|r| Row(columns.iter().map(|c| (c.clone(), r.get(c))).collect()))
            .collect()),

        // Rename mapped keys (others pass through), preserving column order.
        Step::Rename { mapping } => Ok(rows
            .iter()
            .map(|r| {
                Row(r
                    .0
                    .iter()
                    .map(|(k, v)| {
                        let nk = mapping
                            .iter()
                            .find(|(from, _)| from == k)
                            .map(|(_, to)| to.clone())
                            .unwrap_or_else(|| k.clone());
                        (nk, v.clone())
                    })
                    .collect())
            })
            .collect()),

        // Drop the named columns.
        Step::Drop { columns } => Ok(rows
            .iter()
            .map(|r| Row(r.0.iter().filter(|(k, _)| !columns.contains(k)).cloned().collect()))
            .collect()),

        // Keep rows whose predicate is strictly `true`; a non-boolean result is an error (fail-closed).
        Step::Filter { expression } => {
            let ast = compile(expression)?;
            let mut out = Vec::new();
            for r in rows {
                match ast.eval(r.lookup())? {
                    Value::Bool(true) => out.push(r.clone()),
                    Value::Bool(false) => {}
                    other => {
                        return Err(StepError::Message(format!(
                            "filter: predicate must yield a boolean, got {other:?}"
                        )))
                    }
                }
            }
            Ok(out)
        }

        // Add or overwrite a column from an expression over the row's identifiers.
        Step::ApplyExpression { column, expression } => {
            let ast = compile(expression)?;
            let mut out = Vec::with_capacity(rows.len());
            for r in rows {
                let v = ast.eval(r.lookup())?;
                let mut nr = r.clone();
                nr.set(column, v);
                out.push(nr);
            }
            Ok(out)
        }
    }
}

/// Apply a chain of inline steps left-to-right.
pub fn apply_steps(steps: &[Step], rows: &[Row]) -> Result<Vec<Row>, StepError> {
    let mut current = rows.to_vec();
    for step in steps {
        current = apply_step(step, &current)?;
    }
    Ok(current)
}

/// Apply `steps` with PER-ROW error isolation — for stream builds, where a single poison row
/// (dirty data hitting a per-row `eval`/`filter` error) must NOT sink the whole batch. Fast
/// path applies to the whole batch; only if that errors does it fall back to applying per row,
/// keeping the survivors and returning `(survivors, dropped_row_count)`.
///
/// CONFIG vs DATA: callers must validate the steps COMPILE first (see [`validate_steps`]) so a bad
/// expression fails LOUDLY at build start instead of silently dropping every row here — this helper
/// is for *data* errors (a bad row), not authoring errors (a bad step).
pub fn apply_steps_isolating(steps: &[Step], rows: &[Row]) -> (Vec<Row>, usize) {
    match apply_steps(steps, rows) {
        Ok(out) => (out, 0),
        Err(_) => {
            let mut survived = Vec::new();
            let mut dropped = 0usize;
            for row in rows {
                match apply_steps(steps, std::slice::from_ref(row)) {
                    Ok(mut o) => survived.append(&mut o),
                    Err(_) => dropped += 1,
                }
            }
            (survived, dropped)
        }
    }
}

/// Validate that every step's expression COMPILES (an authoring/config check), independent of data —
/// run once at stream-build start so a bad expression fails the build instead of per-row-dropping
/// forever. Compiling with zero rows exercises `compile()` without evaluating any row.
pub fn validate_steps(steps: &[Step]) -> Result<(), StepError> {
    apply_steps(steps, &[]).map(|_| ())
}

/// The inline (row-wise) step ops. Adding an inline step = a `Step` variant + an `apply_step` arm +
/// a `parse` arm + one entry here — all in THIS file.
pub const INLINE_OPS: &[&str] = &["select", "rename", "drop", "filter", "applyExpression"];

/// Decode an inline step from its JSON spec (used by the build loop + the worker's preview tracer).
pub fn parse(step: &serde_json::Value) -> Result<Step, String> {
    let strs = |k: &str| {
        step[k]
            .as_array()
            .map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
            .unwrap_or_default()
    };
    Ok(match step["op"].as_str().unwrap_or("") {
        "select" => Step::Select {
            columns: strs("columns"),
        },
        "drop" => Step::Drop {
            columns: strs("columns"),
        },
        "rename" => Step::Rename {
            mapping: step["mapping"]
                .as_object()
                .map(|m| {
                    m.iter()
                        .map(|(k, v)| (k.clone(), v.as_str().unwrap_or("").to_string()))
                        .collect()
                })
                .unwrap_or_default(),
        },
        "filter" => Step::Filter {
            expression: step["expression"].as_str().unwrap_or("").to_string(),
        },
        "applyExpression" => Step::ApplyExpression {
            column: step["column"].as_str().unwrap_or("").to_string(),
            expression: step["expression"].as_str().unwrap_or("").to_string(),
        },
        other => return Err(format!("unknown inline step op '{other}'")),
    })
}

#[cfg(test)]
mod isolation_tests {
    use super::*;
    use crate::row::Row;
    use fv_value::Value;

    fn row(pairs: &[(&str, Value)]) -> Row {
        Row(pairs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect())
    }

    #[test]
    fn isolating_drops_only_the_poison_row() {
        // filter on `keep`: a non-boolean predicate is a per-row (fail-closed) error, so the whole
        // batch errors — but isolation keeps the good rows and drops just the poison one.
        let steps = vec![Step::Filter {
            expression: "keep".into(),
        }];
        let rows = vec![
            row(&[("keep", Value::Bool(true)), ("id", Value::Num(1.0))]),
            row(&[("keep", Value::Num(9.0)), ("id", Value::Num(2.0))]), // poison: non-boolean
            row(&[("keep", Value::Bool(true)), ("id", Value::Num(3.0))]),
        ];
        assert!(apply_steps(&steps, &rows).is_err(), "whole-batch apply is fail-closed");
        let (out, dropped) = apply_steps_isolating(&steps, &rows);
        assert_eq!(dropped, 1);
        assert_eq!(out.len(), 2);
        assert_eq!(out[0].get("id"), Value::Num(1.0));
        assert_eq!(out[1].get("id"), Value::Num(3.0));
    }

    #[test]
    fn isolating_fast_path_when_all_rows_ok() {
        let steps = vec![Step::Filter {
            expression: "keep".into(),
        }];
        let rows = vec![
            row(&[("keep", Value::Bool(true)), ("id", Value::Num(1.0))]),
            row(&[("keep", Value::Bool(false)), ("id", Value::Num(2.0))]),
        ];
        let (out, dropped) = apply_steps_isolating(&steps, &rows);
        assert_eq!(dropped, 0);
        assert_eq!(out.len(), 1, "filter keeps only the true row");
        assert_eq!(out[0].get("id"), Value::Num(1.0));
    }

    #[test]
    fn validate_rejects_bad_expression_accepts_good() {
        assert!(validate_steps(&[Step::ApplyExpression {
            column: "x".into(),
            expression: "1 +".into()
        }])
        .is_err());
        assert!(validate_steps(&[Step::ApplyExpression {
            column: "x".into(),
            expression: "1 + 2".into()
        }])
        .is_ok());
    }
}