use std::sync::Arc;
use datafusion::arrow::array::{Float64Array, StringArray};
use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::prelude::*;
use fv_value::{compile, Value};
use fv_value_datafusion::{cell_to_value, Translator};
fn table(xs: &[Option<f64>], names: &[Option<&str>]) -> (SessionContext, Arc<Schema>) {
let schema = Arc::new(Schema::new(vec![
Field::new("x", DataType::Float64, true),
Field::new("name", DataType::Utf8, true),
]));
let batch = RecordBatch::try_new(
schema.clone(),
vec![
Arc::new(Float64Array::from(xs.to_vec())),
Arc::new(StringArray::from(names.to_vec())),
],
)
.unwrap();
let ctx = SessionContext::new();
ctx.register_batch("t", batch).unwrap();
(ctx, schema)
}
async fn run(ctx: &SessionContext, schema: Arc<Schema>, src: &str, exact_expected: bool) -> Result<Vec<Value>, String> {
let expr = compile(src).unwrap();
let t = Translator::new(schema)
.translate_or_fallback(&expr)
.map_err(|e| e.to_string())?;
assert_eq!(t.exact, exact_expected, "exactness of `{src}`: {}", t.expr);
let df = ctx
.table("t")
.await
.unwrap()
.select(vec![t.expr.alias("v")])
.map_err(|e| e.to_string())?;
let batches = df.collect().await.map_err(|e| e.to_string())?;
Ok(batches
.iter()
.flat_map(|b| {
(0..b.num_rows())
.map(|i| cell_to_value(b.column(0), i).unwrap())
.collect::<Vec<_>>()
})
.collect())
}
#[tokio::test]
async fn errors_fire_only_for_the_rows_that_reach_them() {
let (ctx, schema) = table(&[Some(0.0), Some(2.0)], &[None, None]);
let v = run(&ctx, schema.clone(), "x == 0 ? 0 : 10 / x", true).await.unwrap();
assert_eq!(v, vec![Value::Num(0.0), Value::Num(5.0)]);
let v = run(&ctx, schema.clone(), "x == 0 || 10 / x > 1", true).await.unwrap();
assert_eq!(v, vec![Value::Bool(true), Value::Bool(true)]);
let v = run(&ctx, schema.clone(), "x != 0 && 10 / x > 1", true).await.unwrap();
assert_eq!(v, vec![Value::Bool(false), Value::Bool(true)]);
let err = run(&ctx, schema.clone(), "10 / x", true).await.unwrap_err();
assert!(err.contains("division by zero"), "{err}");
let v = run(&ctx, schema, "false && (1 / 0 == 1)", true).await.unwrap();
assert_eq!(v, vec![Value::Bool(false), Value::Bool(false)]);
}
#[tokio::test]
async fn domain_guards_and_nulls() {
let (ctx, schema) = table(&[Some(4.0), Some(-1.0), None], &[None, None, None]);
let err = run(&ctx, schema.clone(), "sqrt(x)", true).await.unwrap_err();
assert!(err.contains("negative argument"), "{err}");
let v = run(&ctx, schema.clone(), "x < 0 ? null : sqrt(x)", true).await.unwrap();
assert_eq!(v, vec![Value::Num(2.0), Value::Null, Value::Null]);
let err = run(&ctx, schema.clone(), "(x > 0) && (x < 0 ? true : null)", true)
.await
.unwrap_err();
assert!(err.contains("expected a boolean"), "{err}");
let v = run(&ctx, schema.clone(), "x + 1", true).await.unwrap();
assert_eq!(v, vec![Value::Num(5.0), Value::Num(0.0), Value::Null]);
let v = run(&ctx, schema, "x < 1", true).await.unwrap();
assert_eq!(v, vec![Value::Bool(false), Value::Bool(true), Value::Bool(false)]);
}
#[tokio::test]
async fn missing_columns_read_as_null_and_fallbacks_mix_in() {
let (ctx, schema) = table(&[Some(1.0), Some(2.0)], &[Some("ab"), None]);
let v = run(&ctx, schema.clone(), "ghost == null", true).await.unwrap();
assert_eq!(v, vec![Value::Bool(true), Value::Bool(true)]);
let v = run(&ctx, schema.clone(), "coalesce(ghost, x)", true).await.unwrap();
assert_eq!(v, vec![Value::Num(1.0), Value::Num(2.0)]);
let v = run(&ctx, schema.clone(), "capitalize(name) == 'Ab'", false)
.await
.unwrap();
assert_eq!(v, vec![Value::Bool(true), Value::Bool(false)]);
let v = run(&ctx, schema, "x * 2 > 2 && capitalize(name) != 'zz'", false)
.await
.unwrap();
assert_eq!(v, vec![Value::Bool(false), Value::Bool(true)]);
}
#[tokio::test]
async fn exact_translation_refuses_what_it_cannot_reproduce() {
let (_ctx, schema) = table(&[], &[]);
let t = Translator::new(schema);
assert!(t.translate(&compile("capitalize(name)").unwrap()).is_err());
assert!(t.translate(&compile("name.first").unwrap()).is_err());
assert!(
t.translate(&compile("x > 1 ? 'a' : 2").unwrap()).is_err(),
"branches of different kinds"
);
assert!(
t.translate(&compile("x + name").unwrap()).is_err(),
"a type error at run time is not translatable"
);
assert!(t.translate(&compile("upper(name) == 'A' && x >= 2").unwrap()).is_ok());
}
#[tokio::test]
async fn predicate_mode_emits_plain_pushable_filters() {
use fv_value_datafusion::Options;
let (ctx, schema) = table(
&[Some(60.0), Some(30.0), None],
&[Some("ACTIVE"), Some("ACTIVE"), Some("DOWN")],
);
let t = Translator::new(schema.clone()).with_options(Options { assume_finite: true });
let p = t
.translate_predicate(&compile("name == 'ACTIVE' && x > 40").unwrap())
.unwrap();
assert_eq!(p.expr.to_string(), "name = Utf8(\"ACTIVE\") AND x > Float64(40)");
let rows: Vec<Value> = ctx
.table("t")
.await
.unwrap()
.filter(p.expr)
.unwrap()
.collect()
.await
.unwrap()
.iter()
.flat_map(|b| {
(0..b.num_rows())
.map(|i| cell_to_value(b.column(0), i).unwrap())
.collect::<Vec<_>>()
})
.collect();
assert_eq!(rows, vec![Value::Num(60.0)]);
let ne = t.translate_predicate(&compile("name != 'ACTIVE'").unwrap()).unwrap();
assert!(ne.expr.to_string().contains("IS NOT DISTINCT FROM"), "{}", ne.expr);
let not = t.translate_predicate(&compile("!(x > 40)").unwrap()).unwrap();
assert!(not.expr.to_string().contains("CASE"), "{}", not.expr);
let lazy = t
.translate_predicate(&compile("x == 0 || 10 / x > 1").unwrap())
.unwrap();
assert!(lazy.expr.to_string().contains("CASE"), "{}", lazy.expr);
let strict = Translator::new(schema)
.translate_predicate(&compile("x > 40").unwrap())
.unwrap();
assert_eq!(strict.expr.to_string(), "x > Float64(40) AND x <= Float64(inf)");
assert!(t.translate_predicate(&compile("x + 1").unwrap()).is_err());
}
#[tokio::test]
async fn the_string_family_translates_exactly_where_datafusion_agrees() {
let (ctx, schema) = table(
&[Some(12.0), Some(1.5), None],
&[
Some("https://www.nexmark.com/a/b/c/item.htm?query=1&channel_id=7"),
Some("ab"),
None,
],
);
let s = |v: &str| Value::Str(v.to_string());
assert_eq!(
run(&ctx, schema.clone(), r#"splitPart(name, "/", 4)"#, true)
.await
.unwrap(),
vec![s("a"), s(""), Value::Null]
);
assert_eq!(
run(
&ctx,
schema.clone(),
r#"regexExtract(name, "(?:&|^)channel_id=[^&]*")"#,
true
)
.await
.unwrap(),
vec![s("&channel_id=7"), Value::Null, Value::Null]
);
assert_eq!(
run(&ctx, schema.clone(), r#"left(name, 5)"#, true).await.unwrap(),
vec![s("https"), s("ab"), Value::Null]
);
assert_eq!(
run(&ctx, schema.clone(), r#"substr(name, 9, 3)"#, true).await.unwrap(),
vec![s("www"), s(""), Value::Null]
);
assert_eq!(
run(&ctx, schema.clone(), r#"padStart(toString(x), 4, "0")"#, false)
.await
.unwrap(),
vec![s("0012"), s("01.5"), Value::Null]
);
assert_eq!(
run(&ctx, schema.clone(), "epochToIso(x * 1000)", true).await.unwrap(),
vec![
s("1970-01-01T00:00:12.000Z"),
s("1970-01-01T00:00:01.500Z"),
Value::Null
]
);
assert_eq!(
run(
&ctx,
schema.clone(),
r#"regexExtract(name, "channel_id=([^&]*)")"#,
true
)
.await
.unwrap(),
vec![s("7"), Value::Null, Value::Null]
);
assert_eq!(
run(&ctx, schema.clone(), "left(name, -1)", true).await.unwrap(),
vec![
s("https://www.nexmark.com/a/b/c/item.htm?query=1&channel_id="),
s("a"),
Value::Null
]
);
let t = Translator::new(schema.clone());
for src in [
"left(name, x)",
"splitPart(name, '/', x)",
"padStart(name, x, '0')",
"roundTo(x, x)",
] {
assert!(!t.translate_or_fallback(&compile(src).unwrap()).unwrap().exact, "{src}");
}
}
#[test]
fn arrow_float_text_is_the_dialects_inside_the_vectorised_band() {
use datafusion::arrow::array::{Array, StringArray};
use datafusion::arrow::compute::cast;
use fv_value_datafusion::translate::PLAIN_FLOAT_TEXT;
let mut seed = 0x9E37_79B9_7F4A_7C15_u64;
let mut next = move || {
seed ^= seed << 13;
seed ^= seed >> 7;
seed ^= seed << 17;
seed
};
let (lo, hi) = (PLAIN_FLOAT_TEXT.start, PLAIN_FLOAT_TEXT.end);
let mut values = vec![lo, -lo, f64::from_bits(hi.to_bits() - 1), lo + f64::EPSILON * lo];
for exp in lo.log10() as i32..hi.log10() as i32 {
for _ in 0..2000 {
let v = (1.0 + (next() % 1_000_000_000) as f64 / 1e9) * 10f64.powi(exp);
if v.fract() != 0.0 && PLAIN_FLOAT_TEXT.contains(&v) {
values.push(if next() % 2 == 0 { v } else { -v });
}
}
}
let arrow = cast(
&(Arc::new(Float64Array::from(values.clone())) as Arc<dyn Array>),
&DataType::Utf8,
)
.unwrap();
let arrow = arrow.as_any().downcast_ref::<StringArray>().unwrap();
let wrong: Vec<String> = values
.iter()
.enumerate()
.filter_map(|(i, v)| {
let scope: fv_value::Scope = [("x".to_string(), Value::Num(*v))].into_iter().collect();
let ours = fv_value::evaluate("toString(x)", &scope).unwrap();
(ours.as_str() != Some(arrow.value(i))).then(|| format!("{v:e}: dialect {ours}, Arrow {}", arrow.value(i)))
})
.collect();
assert!(
wrong.is_empty(),
"{} of {} differ:\n{}",
wrong.len(),
values.len(),
wrong.join("\n")
);
}
#[tokio::test]
async fn to_string_of_an_integer_column_is_exact_at_the_extremes() {
use datafusion::arrow::array::Int64Array;
let values = vec![
Some(i64::MAX),
Some(i64::MIN),
Some((1 << 53) + 1),
Some(-7),
Some(0),
None,
];
let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, true)]));
let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(values.clone()))]).unwrap();
let expr = compile("toString(id)").unwrap();
let t = Translator::new(schema)
.translate(&expr)
.expect("exact for an integer column");
let ctx = SessionContext::new();
let out = ctx
.read_batch(batch)
.unwrap()
.select(vec![t.expr.alias("v")])
.unwrap()
.collect()
.await
.unwrap();
for (i, v) in values.iter().enumerate() {
let scope: fv_value::Scope = [("id".to_string(), v.map_or(Value::Null, |n| Value::Num(n as f64)))]
.into_iter()
.collect();
assert_eq!(
cell_to_value(out[0].column(0), i).unwrap(),
expr.eval(&scope).unwrap(),
"id = {v:?}"
);
}
}
#[test]
fn translate_refuses_a_tree_with_a_per_row_step() {
let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Float64, true)]));
let t = Translator::new(schema);
let e = compile("toString(x)").unwrap();
assert!(t.translate(&e).is_err());
assert!(t.translate_predicate(&compile("toString(x) == '1'").unwrap()).is_err());
assert!(!t.translate_or_fallback(&e).unwrap().exact);
}
mod column_types {
use super::*;
use datafusion::arrow::array::{Array, BinaryArray, DictionaryArray, Int64Array, StructArray};
use datafusion::arrow::datatypes::Int32Type;
async fn rows_kept(
schema: Arc<Schema>,
columns: Vec<datafusion::arrow::array::ArrayRef>,
src: &str,
) -> Result<usize, String> {
let batch = RecordBatch::try_new(schema.clone(), columns).unwrap();
let t = Translator::new(schema)
.translate_predicate_or_fallback(&compile(src).unwrap())
.map_err(|e| e.to_string())?;
let ctx = SessionContext::new();
let out = ctx
.read_batch(batch)
.unwrap()
.filter(t.expr)
.map_err(|e| e.to_string())?
.collect()
.await
.map_err(|e| e.to_string())?;
Ok(out.iter().map(|b| b.num_rows()).sum())
}
#[tokio::test]
async fn a_dictionary_string_column_reads_its_values() {
let names: DictionaryArray<Int32Type> = vec!["gold", "silver", "gold"].into_iter().collect();
let schema = Arc::new(Schema::new(vec![Field::new("tier", names.data_type().clone(), true)]));
for src in [
"tier == 'gold'",
"upper(tier) == 'GOLD'",
"startsWith(tier, 'go')",
"length(tier) == 4",
] {
assert_eq!(
rows_kept(schema.clone(), vec![Arc::new(names.clone())], src).await,
Ok(2),
"{src}"
);
}
}
#[tokio::test]
async fn a_struct_column_reads_as_an_object() {
let inner = StructArray::from(vec![(
Arc::new(Field::new("x", DataType::Int64, true)),
Arc::new(Int64Array::from(vec![1, 5])) as datafusion::arrow::array::ArrayRef,
)]);
let schema = Arc::new(Schema::new(vec![Field::new("s", inner.data_type().clone(), true)]));
assert_eq!(rows_kept(schema, vec![Arc::new(inner)], "s.x > 2").await, Ok(1));
}
#[tokio::test]
async fn a_binary_column_is_refused_by_name_when_the_plan_is_built() {
let schema = Arc::new(Schema::new(vec![Field::new("geom", DataType::Binary, true)]));
let geom = BinaryArray::from(vec![Some(b"\x01\x01".as_ref())]);
let err = rows_kept(schema, vec![Arc::new(geom)], "geom != null")
.await
.unwrap_err();
assert!(err.contains("geom") && err.contains("Binary"), "{err}");
}
}
#[tokio::test]
async fn integer_arithmetic_is_checked_only_where_it_can_overflow() {
use datafusion::arrow::array::Int64Array;
let schema = Arc::new(Schema::new(vec![Field::new("t", DataType::Int64, true)]));
let t = Translator::new(schema.clone());
for src in ["t - t % 10000", "t * t * t", "(t + t) * 1000 - t", "abs(t) * t"] {
let e = t.translate(&compile(src).unwrap()).unwrap();
assert!(
!e.expr.to_string().contains("fv_finite"),
"`{src}` needs no check: {}",
e.expr
);
}
let huge = vec!["t"; 17].join(" * ");
let e = t.translate(&compile(&huge).unwrap()).unwrap();
assert!(e.expr.to_string().contains("fv_finite"), "{}", e.expr);
let batch = RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![i64::MAX]))]).unwrap();
let err = SessionContext::new()
.read_batch(batch)
.unwrap()
.select(vec![e.expr.alias("v")])
.unwrap()
.collect()
.await
.unwrap_err();
assert!(err.to_string().contains("'*': result is not a finite number"), "{err}");
}