fv-value-datafusion 0.3.0

The fv-value expression dialect as DataFusion expressions: exact translation for pushdown and vectorized evaluation, with a per-row UDF fallback.
Documentation
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
//! The semantics the translator must reproduce through a real DataFusion query — the cases the
//! conformance vectors cannot exercise because they use constants: errors that must fire only for
//! the rows that reach them, missing columns, domain guards, and the fallback over columns.

use std::sync::Arc;

use datafusion::arrow::array::{Float64Array, StringArray};
use datafusion::arrow::datatypes::{DataType, Field, Schema};
use datafusion::arrow::record_batch::RecordBatch;
use datafusion::prelude::*;
use fv_value::{compile, Value};
use fv_value_datafusion::{cell_to_value, Translator};

fn table(xs: &[Option<f64>], names: &[Option<&str>]) -> (SessionContext, Arc<Schema>) {
    let schema = Arc::new(Schema::new(vec![
        Field::new("x", DataType::Float64, true),
        Field::new("name", DataType::Utf8, true),
    ]));
    let batch = RecordBatch::try_new(
        schema.clone(),
        vec![
            Arc::new(Float64Array::from(xs.to_vec())),
            Arc::new(StringArray::from(names.to_vec())),
        ],
    )
    .unwrap();
    let ctx = SessionContext::new();
    ctx.register_batch("t", batch).unwrap();
    (ctx, schema)
}

async fn run(ctx: &SessionContext, schema: Arc<Schema>, src: &str, exact_expected: bool) -> Result<Vec<Value>, String> {
    let expr = compile(src).unwrap();
    let t = Translator::new(schema)
        .translate_or_fallback(&expr)
        .map_err(|e| e.to_string())?;
    assert_eq!(t.exact, exact_expected, "exactness of `{src}`: {}", t.expr);
    let df = ctx
        .table("t")
        .await
        .unwrap()
        .select(vec![t.expr.alias("v")])
        .map_err(|e| e.to_string())?;
    let batches = df.collect().await.map_err(|e| e.to_string())?;
    Ok(batches
        .iter()
        .flat_map(|b| {
            (0..b.num_rows())
                .map(|i| cell_to_value(b.column(0), i).unwrap())
                .collect::<Vec<_>>()
        })
        .collect())
}

#[tokio::test]
async fn errors_fire_only_for_the_rows_that_reach_them() {
    let (ctx, schema) = table(&[Some(0.0), Some(2.0)], &[None, None]);
    // A division that never happens on the zero row.
    let v = run(&ctx, schema.clone(), "x == 0 ? 0 : 10 / x", true).await.unwrap();
    assert_eq!(v, vec![Value::Num(0.0), Value::Num(5.0)]);
    // Short-circuit: the right side is not evaluated when the left decides.
    let v = run(&ctx, schema.clone(), "x == 0 || 10 / x > 1", true).await.unwrap();
    assert_eq!(v, vec![Value::Bool(true), Value::Bool(true)]);
    let v = run(&ctx, schema.clone(), "x != 0 && 10 / x > 1", true).await.unwrap();
    assert_eq!(v, vec![Value::Bool(false), Value::Bool(true)]);
    // ...but a division the zero row does reach is the dialect's error.
    let err = run(&ctx, schema.clone(), "10 / x", true).await.unwrap_err();
    assert!(err.contains("division by zero"), "{err}");
    // Constant folding of an erroring constant stays lazy inside a branch nobody takes.
    let v = run(&ctx, schema, "false && (1 / 0 == 1)", true).await.unwrap();
    assert_eq!(v, vec![Value::Bool(false), Value::Bool(false)]);
}

#[tokio::test]
async fn domain_guards_and_nulls() {
    let (ctx, schema) = table(&[Some(4.0), Some(-1.0), None], &[None, None, None]);
    let err = run(&ctx, schema.clone(), "sqrt(x)", true).await.unwrap_err();
    assert!(err.contains("negative argument"), "{err}");
    let v = run(&ctx, schema.clone(), "x < 0 ? null : sqrt(x)", true).await.unwrap();
    assert_eq!(v, vec![Value::Num(2.0), Value::Null, Value::Null]);
    // Strict booleans: a null where a boolean is required is an error, as in the dialect.
    let err = run(&ctx, schema.clone(), "(x > 0) && (x < 0 ? true : null)", true)
        .await
        .unwrap_err();
    assert!(err.contains("expected a boolean"), "{err}");
    // Null propagation and ordering with nulls.
    let v = run(&ctx, schema.clone(), "x + 1", true).await.unwrap();
    assert_eq!(v, vec![Value::Num(5.0), Value::Num(0.0), Value::Null]);
    let v = run(&ctx, schema, "x < 1", true).await.unwrap();
    assert_eq!(v, vec![Value::Bool(false), Value::Bool(true), Value::Bool(false)]);
}

#[tokio::test]
async fn missing_columns_read_as_null_and_fallbacks_mix_in() {
    let (ctx, schema) = table(&[Some(1.0), Some(2.0)], &[Some("ab"), None]);
    let v = run(&ctx, schema.clone(), "ghost == null", true).await.unwrap();
    assert_eq!(v, vec![Value::Bool(true), Value::Bool(true)]);
    let v = run(&ctx, schema.clone(), "coalesce(ghost, x)", true).await.unwrap();
    assert_eq!(v, vec![Value::Num(1.0), Value::Num(2.0)]);
    // `capitalize` is not mapped: it runs as a per-row fallback over `name`, inside an otherwise
    // exact expression — the comparison around it still translates.
    let v = run(&ctx, schema.clone(), "capitalize(name) == 'Ab'", false)
        .await
        .unwrap();
    assert_eq!(v, vec![Value::Bool(true), Value::Bool(false)]);
    let v = run(&ctx, schema, "x * 2 > 2 && capitalize(name) != 'zz'", false)
        .await
        .unwrap();
    assert_eq!(v, vec![Value::Bool(false), Value::Bool(true)]);
}

#[tokio::test]
async fn exact_translation_refuses_what_it_cannot_reproduce() {
    let (_ctx, schema) = table(&[], &[]);
    let t = Translator::new(schema);
    assert!(t.translate(&compile("capitalize(name)").unwrap()).is_err());
    assert!(t.translate(&compile("name.first").unwrap()).is_err());
    assert!(
        t.translate(&compile("x > 1 ? 'a' : 2").unwrap()).is_err(),
        "branches of different kinds"
    );
    assert!(
        t.translate(&compile("x + name").unwrap()).is_err(),
        "a type error at run time is not translatable"
    );
    assert!(t.translate(&compile("upper(name) == 'A' && x >= 2").unwrap()).is_ok());
}

#[tokio::test]
async fn predicate_mode_emits_plain_pushable_filters() {
    use fv_value_datafusion::Options;
    let (ctx, schema) = table(
        &[Some(60.0), Some(30.0), None],
        &[Some("ACTIVE"), Some("ACTIVE"), Some("DOWN")],
    );
    let t = Translator::new(schema.clone()).with_options(Options { assume_finite: true });
    // The common filter shape becomes exactly what a source's pushdown recognises.
    let p = t
        .translate_predicate(&compile("name == 'ACTIVE' && x > 40").unwrap())
        .unwrap();
    assert_eq!(p.expr.to_string(), "name = Utf8(\"ACTIVE\") AND x > Float64(40)");
    let rows: Vec<Value> = ctx
        .table("t")
        .await
        .unwrap()
        .filter(p.expr)
        .unwrap()
        .collect()
        .await
        .unwrap()
        .iter()
        .flat_map(|b| {
            (0..b.num_rows())
                .map(|i| cell_to_value(b.column(0), i).unwrap())
                .collect::<Vec<_>>()
        })
        .collect();
    assert_eq!(rows, vec![Value::Num(60.0)]);
    // `!=` and `!` keep null-safety even in a filter: the dialect keeps a null row for `!=`.
    let ne = t.translate_predicate(&compile("name != 'ACTIVE'").unwrap()).unwrap();
    assert!(ne.expr.to_string().contains("IS NOT DISTINCT FROM"), "{}", ne.expr);
    let not = t.translate_predicate(&compile("!(x > 40)").unwrap()).unwrap();
    assert!(not.expr.to_string().contains("CASE"), "{}", not.expr);
    // A right side that can fail still uses the lazy form, so the error only fires where reached.
    let lazy = t
        .translate_predicate(&compile("x == 0 || 10 / x > 1").unwrap())
        .unwrap();
    assert!(lazy.expr.to_string().contains("CASE"), "{}", lazy.expr);
    // Without assume_finite the comparison is bounded against a NaN, which Arrow's total order
    // ranks above every number: still a plain conjunct of range comparisons a source can prune
    // on, with no CASE and no isnan.
    let strict = Translator::new(schema)
        .translate_predicate(&compile("x > 40").unwrap())
        .unwrap();
    assert_eq!(strict.expr.to_string(), "x > Float64(40) AND x <= Float64(inf)");
    // A non-boolean is refused as a predicate.
    assert!(t.translate_predicate(&compile("x + 1").unwrap()).is_err());
}

#[tokio::test]
async fn the_string_family_translates_exactly_where_datafusion_agrees() {
    // 2026-09-16: these ran per row until a streaming stage measured 20k events/s per core on a
    // regex; each is exact only under the guard that makes DataFusion the dialect's twin.
    let (ctx, schema) = table(
        &[Some(12.0), Some(1.5), None],
        &[
            Some("https://www.nexmark.com/a/b/c/item.htm?query=1&channel_id=7"),
            Some("ab"),
            None,
        ],
    );
    let s = |v: &str| Value::Str(v.to_string());
    assert_eq!(
        run(&ctx, schema.clone(), r#"splitPart(name, "/", 4)"#, true)
            .await
            .unwrap(),
        vec![s("a"), s(""), Value::Null]
    );
    assert_eq!(
        run(
            &ctx,
            schema.clone(),
            r#"regexExtract(name, "(?:&|^)channel_id=[^&]*")"#,
            true
        )
        .await
        .unwrap(),
        vec![s("&channel_id=7"), Value::Null, Value::Null]
    );
    assert_eq!(
        run(&ctx, schema.clone(), r#"left(name, 5)"#, true).await.unwrap(),
        vec![s("https"), s("ab"), Value::Null]
    );
    assert_eq!(
        run(&ctx, schema.clone(), r#"substr(name, 9, 3)"#, true).await.unwrap(),
        vec![s("www"), s(""), Value::Null]
    );
    // toString of a number is vectorised for whole numbers and ordinary fractions; the rest
    // (a tiny fraction, NaN) print per row, so the translation is not exact as a whole
    assert_eq!(
        run(&ctx, schema.clone(), r#"padStart(toString(x), 4, "0")"#, false)
            .await
            .unwrap(),
        vec![s("0012"), s("01.5"), Value::Null]
    );
    assert_eq!(
        run(&ctx, schema.clone(), "epochToIso(x * 1000)", true).await.unwrap(),
        vec![
            s("1970-01-01T00:00:12.000Z"),
            s("1970-01-01T00:00:01.500Z"),
            Value::Null
        ]
    );
    // a capturing group gives its match, and a negative count drops from the other end, in both
    assert_eq!(
        run(
            &ctx,
            schema.clone(),
            r#"regexExtract(name, "channel_id=([^&]*)")"#,
            true
        )
        .await
        .unwrap(),
        vec![s("7"), Value::Null, Value::Null]
    );
    assert_eq!(
        run(&ctx, schema.clone(), "left(name, -1)", true).await.unwrap(),
        vec![
            s("https://www.nexmark.com/a/b/c/item.htm?query=1&channel_id="),
            s("a"),
            Value::Null
        ]
    );
    // a count or a position read from a column stays per row: the dialect checks it
    let t = Translator::new(schema.clone());
    for src in [
        "left(name, x)",
        "splitPart(name, '/', x)",
        "padStart(name, x, '0')",
        "roundTo(x, x)",
    ] {
        assert!(!t.translate_or_fallback(&compile(src).unwrap()).unwrap().exact, "{src}");
    }
}

/// The band where `toString` of a fraction runs as Arrow's cast must be one where Arrow writes
/// exactly what the dialect writes; random fractions across every decade of it, and its edges.
#[test]
fn arrow_float_text_is_the_dialects_inside_the_vectorised_band() {
    use datafusion::arrow::array::{Array, StringArray};
    use datafusion::arrow::compute::cast;
    use fv_value_datafusion::translate::PLAIN_FLOAT_TEXT;
    let mut seed = 0x9E37_79B9_7F4A_7C15_u64;
    let mut next = move || {
        seed ^= seed << 13;
        seed ^= seed >> 7;
        seed ^= seed << 17;
        seed
    };
    let (lo, hi) = (PLAIN_FLOAT_TEXT.start, PLAIN_FLOAT_TEXT.end);
    let mut values = vec![lo, -lo, f64::from_bits(hi.to_bits() - 1), lo + f64::EPSILON * lo];
    for exp in lo.log10() as i32..hi.log10() as i32 {
        for _ in 0..2000 {
            let v = (1.0 + (next() % 1_000_000_000) as f64 / 1e9) * 10f64.powi(exp);
            if v.fract() != 0.0 && PLAIN_FLOAT_TEXT.contains(&v) {
                values.push(if next() % 2 == 0 { v } else { -v });
            }
        }
    }
    let arrow = cast(
        &(Arc::new(Float64Array::from(values.clone())) as Arc<dyn Array>),
        &DataType::Utf8,
    )
    .unwrap();
    let arrow = arrow.as_any().downcast_ref::<StringArray>().unwrap();
    let wrong: Vec<String> = values
        .iter()
        .enumerate()
        .filter_map(|(i, v)| {
            let scope: fv_value::Scope = [("x".to_string(), Value::Num(*v))].into_iter().collect();
            let ours = fv_value::evaluate("toString(x)", &scope).unwrap();
            (ours.as_str() != Some(arrow.value(i))).then(|| format!("{v:e}: dialect {ours}, Arrow {}", arrow.value(i)))
        })
        .collect();
    assert!(
        wrong.is_empty(),
        "{} of {} differ:\n{}",
        wrong.len(),
        values.len(),
        wrong.join("\n")
    );
}

/// An integer column prints through the 64-bit cast, exactly as the dialect prints the number the
/// integer becomes — including one near i64::MAX, which as a number is 2^63.
#[tokio::test]
async fn to_string_of_an_integer_column_is_exact_at_the_extremes() {
    use datafusion::arrow::array::Int64Array;
    let values = vec![
        Some(i64::MAX),
        Some(i64::MIN),
        Some((1 << 53) + 1),
        Some(-7),
        Some(0),
        None,
    ];
    let schema = Arc::new(Schema::new(vec![Field::new("id", DataType::Int64, true)]));
    let batch = RecordBatch::try_new(schema.clone(), vec![Arc::new(Int64Array::from(values.clone()))]).unwrap();
    let expr = compile("toString(id)").unwrap();
    let t = Translator::new(schema)
        .translate(&expr)
        .expect("exact for an integer column");
    let ctx = SessionContext::new();
    let out = ctx
        .read_batch(batch)
        .unwrap()
        .select(vec![t.expr.alias("v")])
        .unwrap()
        .collect()
        .await
        .unwrap();
    for (i, v) in values.iter().enumerate() {
        let scope: fv_value::Scope = [("id".to_string(), v.map_or(Value::Null, |n| Value::Num(n as f64)))]
            .into_iter()
            .collect();
        assert_eq!(
            cell_to_value(out[0].column(0), i).unwrap(),
            expr.eval(&scope).unwrap(),
            "id = {v:?}"
        );
    }
}

/// `translate` promises a tree with no per-row step: one that needs the dialect for some rows is
/// refused there, and `translate_or_fallback` takes it.
#[test]
fn translate_refuses_a_tree_with_a_per_row_step() {
    let schema = Arc::new(Schema::new(vec![Field::new("x", DataType::Float64, true)]));
    let t = Translator::new(schema);
    let e = compile("toString(x)").unwrap();
    assert!(t.translate(&e).is_err());
    assert!(t.translate_predicate(&compile("toString(x) == '1'").unwrap()).is_err());
    assert!(!t.translate_or_fallback(&e).unwrap().exact);
}

/// A COLUMN TYPE THE DIALECT HAS A VALUE FOR CONVERTS EXACTLY; ANY OTHER IS REFUSED BY NAME. It
/// used to read as null in the per-row path, so a filter over a binary or dictionary column
/// quietly matched nothing.
mod column_types {
    use super::*;
    use datafusion::arrow::array::{Array, BinaryArray, DictionaryArray, Int64Array, StructArray};
    use datafusion::arrow::datatypes::Int32Type;

    async fn rows_kept(
        schema: Arc<Schema>,
        columns: Vec<datafusion::arrow::array::ArrayRef>,
        src: &str,
    ) -> Result<usize, String> {
        let batch = RecordBatch::try_new(schema.clone(), columns).unwrap();
        let t = Translator::new(schema)
            .translate_predicate_or_fallback(&compile(src).unwrap())
            .map_err(|e| e.to_string())?;
        let ctx = SessionContext::new();
        let out = ctx
            .read_batch(batch)
            .unwrap()
            .filter(t.expr)
            .map_err(|e| e.to_string())?
            .collect()
            .await
            .map_err(|e| e.to_string())?;
        Ok(out.iter().map(|b| b.num_rows()).sum())
    }

    #[tokio::test]
    async fn a_dictionary_string_column_reads_its_values() {
        let names: DictionaryArray<Int32Type> = vec!["gold", "silver", "gold"].into_iter().collect();
        let schema = Arc::new(Schema::new(vec![Field::new("tier", names.data_type().clone(), true)]));
        for src in [
            "tier == 'gold'",
            "upper(tier) == 'GOLD'",
            "startsWith(tier, 'go')",
            "length(tier) == 4",
        ] {
            assert_eq!(
                rows_kept(schema.clone(), vec![Arc::new(names.clone())], src).await,
                Ok(2),
                "{src}"
            );
        }
    }

    #[tokio::test]
    async fn a_struct_column_reads_as_an_object() {
        let inner = StructArray::from(vec![(
            Arc::new(Field::new("x", DataType::Int64, true)),
            Arc::new(Int64Array::from(vec![1, 5])) as datafusion::arrow::array::ArrayRef,
        )]);
        let schema = Arc::new(Schema::new(vec![Field::new("s", inner.data_type().clone(), true)]));
        assert_eq!(rows_kept(schema, vec![Arc::new(inner)], "s.x > 2").await, Ok(1));
    }

    #[tokio::test]
    async fn a_binary_column_is_refused_by_name_when_the_plan_is_built() {
        let schema = Arc::new(Schema::new(vec![Field::new("geom", DataType::Binary, true)]));
        let geom = BinaryArray::from(vec![Some(b"\x01\x01".as_ref())]);
        let err = rows_kept(schema, vec![Arc::new(geom)], "geom != null")
            .await
            .unwrap_err();
        assert!(err.contains("geom") && err.contains("Binary"), "{err}");
    }
}

/// THE OVERFLOW CHECK RESTS ON A BOUND, NOT ON THE OPERATOR. Arithmetic over integer columns
/// (|x| ≤ 2^63) cannot reach the largest number in a few steps, so it carries no finiteness check
/// and costs DataFusion's bare kernels; an expression whose bound can pass it keeps the check, and
/// gets the dialect's error.
#[tokio::test]
async fn integer_arithmetic_is_checked_only_where_it_can_overflow() {
    use datafusion::arrow::array::Int64Array;
    let schema = Arc::new(Schema::new(vec![Field::new("t", DataType::Int64, true)]));
    let t = Translator::new(schema.clone());
    for src in ["t - t % 10000", "t * t * t", "(t + t) * 1000 - t", "abs(t) * t"] {
        let e = t.translate(&compile(src).unwrap()).unwrap();
        assert!(
            !e.expr.to_string().contains("fv_finite"),
            "`{src}` needs no check: {}",
            e.expr
        );
    }
    // seventeen factors of 2^63 can pass the largest number
    let huge = vec!["t"; 17].join(" * ");
    let e = t.translate(&compile(&huge).unwrap()).unwrap();
    assert!(e.expr.to_string().contains("fv_finite"), "{}", e.expr);
    let batch = RecordBatch::try_new(schema, vec![Arc::new(Int64Array::from(vec![i64::MAX]))]).unwrap();
    let err = SessionContext::new()
        .read_batch(batch)
        .unwrap()
        .select(vec![e.expr.alias("v")])
        .unwrap()
        .collect()
        .await
        .unwrap_err();
    assert!(err.to_string().contains("'*': result is not a finite number"), "{err}");
}