use crate::row::Row;
use fv_value::{compile, ExprError, Value};
#[derive(Debug, Clone)]
pub enum Step {
Select { columns: Vec<String> },
Rename { mapping: Vec<(String, String)> },
Drop { columns: Vec<String> },
Filter { expression: String },
ApplyExpression { column: String, expression: String },
}
#[derive(Debug)]
pub enum StepError {
Expr(ExprError),
Message(String),
}
impl std::fmt::Display for StepError {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
match self {
StepError::Expr(e) => write!(f, "{e}"),
StepError::Message(m) => write!(f, "{m}"),
}
}
}
impl From<ExprError> for StepError {
fn from(e: ExprError) -> Self {
StepError::Expr(e)
}
}
pub fn apply_step(step: &Step, rows: &[Row]) -> Result<Vec<Row>, StepError> {
match step {
Step::Select { columns } => Ok(rows
.iter()
.map(|r| Row(columns.iter().map(|c| (c.clone(), r.get(c))).collect()))
.collect()),
Step::Rename { mapping } => Ok(rows
.iter()
.map(|r| {
Row(r
.0
.iter()
.map(|(k, v)| {
let nk = mapping
.iter()
.find(|(from, _)| from == k)
.map(|(_, to)| to.clone())
.unwrap_or_else(|| k.clone());
(nk, v.clone())
})
.collect())
})
.collect()),
Step::Drop { columns } => Ok(rows
.iter()
.map(|r| Row(r.0.iter().filter(|(k, _)| !columns.contains(k)).cloned().collect()))
.collect()),
Step::Filter { expression } => {
let ast = compile(expression)?;
let mut out = Vec::new();
for r in rows {
match ast.eval(r.lookup())? {
Value::Bool(true) => out.push(r.clone()),
Value::Bool(false) => {}
other => {
return Err(StepError::Message(format!(
"filter: predicate must yield a boolean, got {other:?}"
)))
}
}
}
Ok(out)
}
Step::ApplyExpression { column, expression } => {
let ast = compile(expression)?;
let mut out = Vec::with_capacity(rows.len());
for r in rows {
let v = ast.eval(r.lookup())?;
let mut nr = r.clone();
nr.set(column, v);
out.push(nr);
}
Ok(out)
}
}
}
pub fn apply_steps(steps: &[Step], rows: &[Row]) -> Result<Vec<Row>, StepError> {
let mut current = rows.to_vec();
for step in steps {
current = apply_step(step, ¤t)?;
}
Ok(current)
}
pub fn apply_steps_isolating(steps: &[Step], rows: &[Row]) -> (Vec<Row>, usize) {
match apply_steps(steps, rows) {
Ok(out) => (out, 0),
Err(_) => {
let mut survived = Vec::new();
let mut dropped = 0usize;
for row in rows {
match apply_steps(steps, std::slice::from_ref(row)) {
Ok(mut o) => survived.append(&mut o),
Err(_) => dropped += 1,
}
}
(survived, dropped)
}
}
}
pub fn validate_steps(steps: &[Step]) -> Result<(), StepError> {
apply_steps(steps, &[]).map(|_| ())
}
pub const INLINE_OPS: &[&str] = &["select", "rename", "drop", "filter", "applyExpression"];
pub fn parse(step: &serde_json::Value) -> Result<Step, String> {
let strs = |k: &str| {
step[k]
.as_array()
.map(|a| a.iter().filter_map(|v| v.as_str().map(String::from)).collect())
.unwrap_or_default()
};
Ok(match step["op"].as_str().unwrap_or("") {
"select" => Step::Select {
columns: strs("columns"),
},
"drop" => Step::Drop {
columns: strs("columns"),
},
"rename" => Step::Rename {
mapping: step["mapping"]
.as_object()
.map(|m| {
m.iter()
.map(|(k, v)| (k.clone(), v.as_str().unwrap_or("").to_string()))
.collect()
})
.unwrap_or_default(),
},
"filter" => Step::Filter {
expression: step["expression"].as_str().unwrap_or("").to_string(),
},
"applyExpression" => Step::ApplyExpression {
column: step["column"].as_str().unwrap_or("").to_string(),
expression: step["expression"].as_str().unwrap_or("").to_string(),
},
other => return Err(format!("unknown inline step op '{other}'")),
})
}
#[cfg(test)]
mod isolation_tests {
use super::*;
use crate::row::Row;
use fv_value::Value;
fn row(pairs: &[(&str, Value)]) -> Row {
Row(pairs.iter().map(|(k, v)| (k.to_string(), v.clone())).collect())
}
#[test]
fn isolating_drops_only_the_poison_row() {
let steps = vec![Step::Filter {
expression: "keep".into(),
}];
let rows = vec![
row(&[("keep", Value::Bool(true)), ("id", Value::Num(1.0))]),
row(&[("keep", Value::Num(9.0)), ("id", Value::Num(2.0))]), row(&[("keep", Value::Bool(true)), ("id", Value::Num(3.0))]),
];
assert!(apply_steps(&steps, &rows).is_err(), "whole-batch apply is fail-closed");
let (out, dropped) = apply_steps_isolating(&steps, &rows);
assert_eq!(dropped, 1);
assert_eq!(out.len(), 2);
assert_eq!(out[0].get("id"), Value::Num(1.0));
assert_eq!(out[1].get("id"), Value::Num(3.0));
}
#[test]
fn isolating_fast_path_when_all_rows_ok() {
let steps = vec![Step::Filter {
expression: "keep".into(),
}];
let rows = vec![
row(&[("keep", Value::Bool(true)), ("id", Value::Num(1.0))]),
row(&[("keep", Value::Bool(false)), ("id", Value::Num(2.0))]),
];
let (out, dropped) = apply_steps_isolating(&steps, &rows);
assert_eq!(dropped, 0);
assert_eq!(out.len(), 1, "filter keeps only the true row");
assert_eq!(out[0].get("id"), Value::Num(1.0));
}
#[test]
fn validate_rejects_bad_expression_accepts_good() {
assert!(validate_steps(&[Step::ApplyExpression {
column: "x".into(),
expression: "1 +".into()
}])
.is_err());
assert!(validate_steps(&[Step::ApplyExpression {
column: "x".into(),
expression: "1 + 2".into()
}])
.is_ok());
}
}