use std::fmt;
use datafusion::arrow::datatypes::{DataType, SchemaRef};
use datafusion::common::Column;
use datafusion::functions::{core, datetime, math, regex, string, unicode};
use datafusion::functions_nested::expr_fn::array_element;
use datafusion::logical_expr::{cast, lit, when, BinaryExpr, Expr as DfExpr, Operator, ScalarUDF};
use datafusion::scalar::ScalarValue;
use fv_value::{Ast, BinaryOp, Expr, UnaryOp, Value};
use crate::convert::{data_type_for, Kind};
use crate::udf::{fallback, ErrorUdf, FiniteUdf};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Unsupported {
pub reason: String,
}
impl fmt::Display for Unsupported {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "not translatable: {}", self.reason)
}
}
impl std::error::Error for Unsupported {}
fn no<T>(reason: impl Into<String>) -> Result<T, Unsupported> {
Err(Unsupported { reason: reason.into() })
}
#[derive(Debug, Clone)]
pub struct Translation {
pub expr: DfExpr,
pub kind: Kind,
pub exact: bool,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Options {
pub assume_finite: bool,
}
#[derive(Debug, Clone)]
pub struct Translator {
schema: SchemaRef,
options: Options,
}
#[derive(Debug, Clone)]
struct Node {
expr: DfExpr,
kind: Kind,
nullable: bool,
maybe_nonfinite: bool,
may_error: bool,
exact: bool,
}
impl Node {
fn new(expr: DfExpr, kind: Kind) -> Self {
Node {
expr,
kind,
nullable: true,
maybe_nonfinite: kind == Kind::Num,
may_error: false,
exact: true,
}
}
fn certain(mut self) -> Self {
self.nullable = false;
self
}
fn finite(mut self) -> Self {
self.maybe_nonfinite = false;
self
}
fn fallible(mut self) -> Self {
self.may_error = true;
self
}
fn null(kind: Kind) -> Self {
Node::new(null_of(kind), kind).finite()
}
}
fn f64_lit(x: f64) -> DfExpr {
DfExpr::Literal(ScalarValue::Float64(Some(x)), None)
}
fn null_of(kind: Kind) -> DfExpr {
DfExpr::Literal(
match kind {
Kind::Num => ScalarValue::Float64(None),
Kind::Str => ScalarValue::Utf8(None),
Kind::Bool => ScalarValue::Boolean(None),
Kind::Null => ScalarValue::Null,
},
None,
)
}
fn binary(l: DfExpr, op: Operator, r: DfExpr) -> DfExpr {
DfExpr::BinaryExpr(BinaryExpr::new(Box::new(l), op, Box::new(r)))
}
fn any_null(nodes: &[&Node]) -> Option<DfExpr> {
let checks: Vec<DfExpr> = nodes
.iter()
.filter(|n| n.nullable)
.map(|n| n.expr.clone().is_null())
.collect();
(!checks.is_empty()).then(|| or_all(checks))
}
fn ordered_numbers(op: Operator, a: &Node, b: &Node) -> DfExpr {
let greater = matches!(op, Operator::Gt | Operator::GtEq);
let inclusive = matches!(op, Operator::LtEq | Operator::GtEq);
let zero = |right: bool| if (inclusive == greater) == right { -0.0 } else { 0.0 };
let side = |n: &Node, right: bool, other: &Node| match f64_literal(&n.expr) {
Some(0.0) => f64_lit(zero(right)),
Some(_) => n.expr.clone(),
None if f64_literal(&other.expr).is_some() => n.expr.clone(),
None => binary(n.expr.clone(), Operator::Plus, f64_lit(0.0)),
};
let mut conj = binary(side(a, false, b), op, side(b, true, a));
let (high, low) = if greater { (a, b) } else { (b, a) };
if high.maybe_nonfinite {
conj = binary(
conj,
Operator::And,
binary(high.expr.clone(), Operator::LtEq, f64_lit(f64::INFINITY)),
);
}
if low.maybe_nonfinite {
conj = binary(
conj,
Operator::And,
binary(low.expr.clone(), Operator::GtEq, f64_lit(f64::NEG_INFINITY)),
);
}
conj
}
fn or_all(mut items: Vec<DfExpr>) -> DfExpr {
let first = items.remove(0);
items.into_iter().fold(first, |acc, e| binary(acc, Operator::Or, e))
}
fn case(clauses: Vec<(DfExpr, DfExpr)>, otherwise: DfExpr) -> DfExpr {
let mut it = clauses.into_iter();
let (c, t) = it.next().expect("at least one WHEN");
let mut b = when(c, t);
for (c, t) in it {
b = b.when(c, t);
}
b.otherwise(otherwise).expect("a CASE with clauses is well-formed")
}
fn lazy_bool_case(clauses: Vec<(DfExpr, DfExpr)>, otherwise: DfExpr) -> DfExpr {
let num = |b: DfExpr| match b {
DfExpr::Literal(ScalarValue::Boolean(Some(true)), _) => f64_lit(1.0),
DfExpr::Literal(ScalarValue::Boolean(Some(false)), _) => f64_lit(0.0),
other => cast(other, DataType::Float64),
};
let clauses = clauses.into_iter().map(|(c, t)| (c, num(t))).collect();
binary(case(clauses, num(otherwise)), Operator::Eq, f64_lit(1.0))
}
fn error(kind: Kind, message: &str) -> DfExpr {
ErrorUdf::call_expr(data_type_for(kind), message)
}
fn isnan(e: DfExpr) -> DfExpr {
math::isnan().call(vec![e])
}
fn non_negative_literal(e: &DfExpr) -> bool {
matches!(e, DfExpr::Literal(ScalarValue::Float64(Some(n)), _) if *n >= 0.0 && n.fract() == 0.0)
}
const TWO_POW_63: f64 = 9_223_372_036_854_775_808.0;
fn whole_literal(e: &DfExpr) -> bool {
f64_literal(e).is_some_and(|n| n.fract() == 0.0 && n.abs() < TWO_POW_63)
}
pub const PLAIN_FLOAT_TEXT: std::ops::Range<f64> = 1e-5..1e15;
fn epoch_millis(a: &Node) -> DfExpr {
datetime::expr_fn::to_timestamp_millis(vec![cast(a.expr.clone(), DataType::Int64)])
}
fn to_f64(e: DfExpr) -> DfExpr {
cast(e, DataType::Float64)
}
fn f64_literal(e: &DfExpr) -> Option<f64> {
match e {
DfExpr::Literal(ScalarValue::Float64(Some(x)), _) => Some(*x),
_ => None,
}
}
fn finite_checked(mut node: Node, what: &str, possible: bool) -> Node {
if node.kind == Kind::Num && possible {
node.expr = FiniteUdf::call_expr(node.expr, what);
node.may_error = true;
}
node.maybe_nonfinite = false;
node
}
fn literal_rules_out_overflow(op: BinaryOp, a: &Node, b: &Node) -> bool {
let lit = |n: &Node| f64_literal(&n.expr).map(f64::abs);
let small = |n: &Node| lit(n).is_some_and(|c| c < 2f64.powi(970));
match op {
BinaryOp::Add | BinaryOp::Sub => small(a) || small(b),
BinaryOp::Mul => lit(a).is_some_and(|c| c <= 1.0) || lit(b).is_some_and(|c| c <= 1.0),
BinaryOp::Div => lit(b).is_some_and(|c| c >= 1.0),
_ => true,
}
}
fn is_nonzero_literal(e: &DfExpr) -> bool {
matches!(e, DfExpr::Literal(ScalarValue::Float64(Some(x)), _) if *x != 0.0)
}
fn retype(expr: DfExpr, from: Kind, to: Kind) -> DfExpr {
if from == Kind::Null && to != Kind::Null {
null_of(to)
} else {
expr
}
}
fn common_kind(kinds: impl Iterator<Item = Option<Kind>>) -> Option<Kind> {
let mut found: Option<Kind> = None;
for k in kinds {
match k? {
Kind::Null => {}
k => match found {
None => found = Some(k),
Some(f) if f == k => {}
Some(_) => return None,
},
}
}
Some(found.unwrap_or(Kind::Null))
}
fn builtin_kind(name: &str, args: &[Option<Kind>]) -> Option<Kind> {
const NUM: &[&str] = &[
"abs",
"floor",
"ceil",
"trunc",
"round",
"exp",
"sqrt",
"ln",
"log10",
"log2",
"cbrt",
"sign",
"pow",
"hypot",
"degToRad",
"radToDeg",
"min",
"max",
"clamp",
"roundTo",
"toNumber",
"sin",
"cos",
"tan",
"asin",
"acos",
"atan",
"atan2",
"sinh",
"cosh",
"tanh",
"log",
"pi",
"factorial",
"gcd",
"lcm",
"length",
"indexOf",
"ascii",
"epochMs",
"yearOf",
"monthOf",
"dayOf",
"hourOf",
"minuteOf",
"dayOfWeekOf",
"dayOfYear",
"arraySum",
"arrayMin",
"arrayMax",
"haversineKm",
"bearingDeg",
"distance3dM",
];
const STR: &[&str] = &[
"upper",
"lower",
"trim",
"trimStart",
"trimEnd",
"capitalize",
"reverse",
"concat",
"replace",
"repeat",
"substr",
"left",
"right",
"splitPart",
"concatWs",
"chr",
"toHex",
"regexReplace",
"substring",
"padStart",
"padEnd",
"toString",
"regexExtract",
"arrayJoin",
"epochToIso",
"typeOf",
"groundStrike",
];
const BOOL: &[&str] = &[
"contains",
"startsWith",
"endsWith",
"regexMatch",
"isNull",
"isNumber",
"isString",
"isBoolean",
"arrayContains",
"geoContains",
"geoWithin",
"geoIntersects",
"altitudeBetween",
"geoWithinVolume",
];
if NUM.contains(&name) {
Some(Kind::Num)
} else if STR.contains(&name) {
Some(Kind::Str)
} else if BOOL.contains(&name) {
Some(Kind::Bool)
} else if name == "coalesce" || name == "nullIf" {
common_kind(args.iter().copied())
} else {
None
}
}
#[derive(Clone, Copy)]
enum Guard {
NonNegative,
Positive,
UnitInterval,
}
fn guarded(node: Node, arg: &Node, guard: Guard, message: &str) -> Node {
if arg.kind != Kind::Num {
return node;
}
let x = arg.expr.clone();
let below = |op: Operator, bound: f64| {
let c = binary(x.clone(), op, f64_lit(bound));
if arg.maybe_nonfinite {
binary(
c,
Operator::And,
binary(x.clone(), Operator::GtEq, f64_lit(f64::NEG_INFINITY)),
)
} else {
c
}
};
let out_of_domain = match guard {
Guard::NonNegative => below(Operator::Lt, -0.0),
Guard::Positive => below(Operator::LtEq, 0.0),
Guard::UnitInterval => binary(
binary(x.clone(), Operator::Lt, f64_lit(-1.0)),
Operator::Or,
binary(x, Operator::Gt, f64_lit(1.0)),
),
};
let mut clauses = Vec::new();
if arg.nullable {
clauses.push((arg.expr.clone().is_null(), null_of(Kind::Num)));
}
clauses.push((out_of_domain, error(Kind::Num, message)));
Node {
expr: case(clauses, node.expr),
may_error: true,
..node
}
}
fn largest_finite_multiplicand(factor: f64) -> f64 {
if (f64::MAX * factor).is_finite() {
return f64::MAX;
}
let (mut lo, mut hi) = (0u64, f64::MAX.to_bits());
while hi - lo > 1 {
let mid = lo + (hi - lo) / 2;
if (f64::from_bits(mid) * factor).is_finite() {
lo = mid;
} else {
hi = mid;
}
}
f64::from_bits(lo)
}
type Udf = fn() -> std::sync::Arc<ScalarUDF>;
impl Translator {
pub fn new(schema: SchemaRef) -> Self {
Translator {
schema,
options: Options::default(),
}
}
pub fn with_options(mut self, options: Options) -> Self {
self.options = options;
self
}
pub fn translate(&self, expr: &Expr) -> Result<Translation, Unsupported> {
let n = self.node(expr.ast(), false, false)?;
if !n.exact {
return no("some rows need the dialect's own evaluation");
}
Ok(Translation {
expr: n.expr,
kind: n.kind,
exact: true,
})
}
pub fn translate_or_fallback(&self, expr: &Expr) -> Result<Translation, Unsupported> {
let n = self.node(expr.ast(), true, false)?;
Ok(Translation {
expr: n.expr,
kind: n.kind,
exact: n.exact,
})
}
pub fn translate_predicate(&self, expr: &Expr) -> Result<Translation, Unsupported> {
let n = self.node(expr.ast(), false, true)?;
if n.kind != Kind::Bool {
return no(format!("a filter must be a boolean, this is a {:?}", n.kind));
}
if !n.exact {
return no("some rows need the dialect's own evaluation");
}
Ok(Translation {
expr: n.expr,
kind: n.kind,
exact: true,
})
}
pub fn translate_predicate_or_fallback(&self, expr: &Expr) -> Result<Translation, Unsupported> {
let n = self.node(expr.ast(), true, true)?;
if n.kind != Kind::Bool {
return no(format!("a filter must be a boolean, this is a {:?}", n.kind));
}
Ok(Translation {
expr: n.expr,
kind: n.kind,
exact: n.exact,
})
}
pub fn kind_of(&self, expr: &Expr) -> Option<Kind> {
self.kind(expr.ast())
}
fn column_kind(&self, name: &str) -> Option<Kind> {
self.schema
.field_with_name(name)
.ok()
.and_then(|f| Kind::of_data_type(f.data_type()))
}
fn kind(&self, ast: &Ast) -> Option<Kind> {
Some(match ast {
Ast::Lit(v) => match v {
Value::Null => Kind::Null,
Value::Bool(_) => Kind::Bool,
Value::Num(_) => Kind::Num,
Value::Str(_) => Kind::Str,
_ => return None,
},
Ast::Id(name) => match self.schema.field_with_name(name) {
Ok(_) => self.column_kind(name)?,
Err(_) => Kind::Null, },
Ast::Member(..) | Ast::Index(..) | Ast::List(_) => return None,
Ast::Unary(UnaryOp::Not, _) => Kind::Bool,
Ast::Unary(UnaryOp::Neg, _) => Kind::Num,
Ast::Binary(op, l, r) => match op {
BinaryOp::Or
| BinaryOp::And
| BinaryOp::Eq
| BinaryOp::Ne
| BinaryOp::In
| BinaryOp::Lt
| BinaryOp::Le
| BinaryOp::Gt
| BinaryOp::Ge => Kind::Bool,
BinaryOp::Add => common_kind([self.kind(l), self.kind(r)].into_iter())?,
BinaryOp::Sub | BinaryOp::Mul | BinaryOp::Div | BinaryOp::Rem => Kind::Num,
_ => return None,
},
Ast::Ternary(_, t, e) => common_kind([self.kind(t), self.kind(e)].into_iter())?,
Ast::Call(f, args) => {
let kinds: Vec<Option<Kind>> = args.iter().map(|a| self.kind(a)).collect();
builtin_kind(f.name(), &kinds)?
}
_ => return None,
})
}
fn node(&self, ast: &Ast, fb: bool, pred: bool) -> Result<Node, Unsupported> {
match self.node_inner(ast, fb, pred) {
Ok(n) => Ok(n),
Err(u) if fb => match self.kind(ast) {
Some(kind) => {
let expr = Expr::from_ast(ast.clone());
let mut n = Node::new(fallback(&expr, data_type_for(kind), &self.schema)?, kind).fallible();
n.exact = false;
Ok(n)
}
None => Err(u),
},
Err(u) => Err(u),
}
}
fn node_inner(&self, ast: &Ast, fb: bool, pred: bool) -> Result<Node, Unsupported> {
if !matches!(ast, Ast::Lit(_)) && ast.identifiers().is_empty() {
return self.constant(ast);
}
match ast {
Ast::Lit(v) => Ok(match v {
Value::Null => Node::null(Kind::Null),
Value::Bool(b) => Node::new(lit(*b), Kind::Bool).certain().finite(),
Value::Num(n) => {
let node = Node::new(f64_lit(*n), Kind::Num).certain();
if n.is_nan() {
node
} else {
node.finite()
}
}
Value::Str(s) => Node::new(lit(s.as_str()), Kind::Str).certain().finite(),
Value::List(_) | Value::Obj(_) => return no("list and object literals"),
}),
Ast::Id(name) => {
let Ok(field) = self.schema.field_with_name(name) else {
return Ok(Node::null(Kind::Null));
};
let kind = Kind::of_data_type(field.data_type()).ok_or_else(|| Unsupported {
reason: format!("column '{name}' has type {}", field.data_type()),
})?;
let column = DfExpr::Column(Column::from_name(name));
let expr = match (kind, field.data_type()) {
(Kind::Num, DataType::Float64) => column,
(Kind::Num, _) => to_f64(column),
(_, DataType::Dictionary(_, values)) => cast(column, values.as_ref().clone()),
_ => column,
};
let mut n = Node::new(expr, kind);
n.nullable = field.is_nullable();
n.maybe_nonfinite =
!self.options.assume_finite && matches!(field.data_type(), DataType::Float32 | DataType::Float64);
Ok(n)
}
Ast::Member(..) => no("member access on objects"),
Ast::Index(..) => no("indexing"),
Ast::List(_) => no("list values"),
Ast::Unary(UnaryOp::Not, a) => {
let a = self.must_bool(self.node(a, fb, false)?, "'!'")?;
Ok(Node {
expr: DfExpr::Not(Box::new(a.expr)),
kind: Kind::Bool,
nullable: false,
maybe_nonfinite: false,
may_error: a.may_error,
exact: a.exact,
})
}
Ast::Unary(UnaryOp::Neg, a) => {
let a = self.node(a, fb, false)?;
match a.kind {
Kind::Null => Ok(Node::null(Kind::Num)),
Kind::Num => {
let possible = a.maybe_nonfinite;
Ok(finite_checked(
Node {
expr: DfExpr::Negative(Box::new(a.expr)),
..a
},
"'-'",
possible,
))
}
k => no(format!("unary '-' on a {k:?}")),
}
}
Ast::Ternary(c, t, e) => {
let c = self.must_bool(self.node(c, fb, false)?, "'?:' condition")?;
let t = self.node(t, fb, false)?;
let e = self.node(e, fb, false)?;
let kind = common_kind([Some(t.kind), Some(e.kind)].into_iter()).ok_or_else(|| Unsupported {
reason: "'?:' branches of different kinds".into(),
})?;
let (te, ee) = (retype(t.expr, t.kind, kind), retype(e.expr, e.kind, kind));
let expr = if kind == Kind::Bool {
lazy_bool_case(vec![(c.expr, te)], ee)
} else {
case(vec![(c.expr, te)], ee)
};
Ok(Node {
expr,
kind,
nullable: t.nullable || e.nullable,
maybe_nonfinite: t.maybe_nonfinite || e.maybe_nonfinite,
may_error: c.may_error || t.may_error || e.may_error,
exact: c.exact && t.exact && e.exact,
})
}
Ast::Binary(op, l, r) => self.binary_node(*op, l, r, fb, pred),
Ast::Call(f, args) => {
let nodes = args
.iter()
.map(|a| self.node(a, fb, false))
.collect::<Result<Vec<_>, _>>()?;
self.call(f.name(), nodes, ast)
}
_ => no("unknown node"),
}
}
fn constant(&self, ast: &Ast) -> Result<Node, Unsupported> {
let expr = Expr::from_ast(ast.clone());
match expr.eval(&fv_value::Empty) {
Ok(Value::Null) => Ok(Node::null(Kind::Null)),
Ok(Value::Bool(b)) => Ok(Node::new(lit(b), Kind::Bool).certain().finite()),
Ok(Value::Num(n)) => {
let node = Node::new(f64_lit(n), Kind::Num).certain();
Ok(if n.is_nan() { node } else { node.finite() })
}
Ok(Value::Str(s)) => Ok(Node::new(lit(s.as_str()), Kind::Str).certain().finite()),
Ok(other) => no(format!("constant {} values", other.type_name())),
Err(e) => match self.kind(ast) {
Some(kind) if kind != Kind::Null => Ok(Node::new(error(kind, &e.to_string()), kind)
.certain()
.finite()
.fallible()),
_ => no(format!("a constant that errors ({e}) with no known kind")),
},
}
}
fn must_bool(&self, n: Node, where_: &str) -> Result<Node, Unsupported> {
match n.kind {
Kind::Bool if !n.nullable => Ok(n),
Kind::Bool => Ok(Node {
expr: lazy_bool_case(
vec![(
n.expr.clone().is_null(),
error(Kind::Bool, &format!("{where_}: expected a boolean, got null")),
)],
n.expr,
),
nullable: false,
may_error: true,
..n
}),
k => no(format!("{where_} on a {k:?}")),
}
}
fn binary_node(&self, op: BinaryOp, l: &Ast, r: &Ast, fb: bool, pred: bool) -> Result<Node, Unsupported> {
match op {
BinaryOp::And | BinaryOp::Or => {
let a = self.must_bool(self.node(l, fb, pred)?, op.symbol())?;
let b = self.must_bool(self.node(r, fb, pred)?, op.symbol())?;
let may_error = a.may_error || b.may_error;
let expr = match (op, b.may_error) {
(BinaryOp::And, false) => binary(a.expr, Operator::And, b.expr),
(BinaryOp::And, true) => lazy_bool_case(vec![(a.expr, b.expr)], lit(false)),
(_, false) => binary(a.expr, Operator::Or, b.expr),
(_, true) => lazy_bool_case(vec![(a.expr, lit(true))], b.expr),
};
Ok(Node {
expr,
kind: Kind::Bool,
nullable: false,
maybe_nonfinite: false,
may_error,
exact: a.exact && b.exact,
})
}
BinaryOp::In => {
let a = self.node(l, fb, false)?;
let Ast::List(items) = r else {
return no("'in' with a non-literal list");
};
if items.is_empty() {
return Ok(Node::new(lit(false), Kind::Bool).certain().finite());
}
let mut exact = a.exact;
let mut may_error = a.may_error;
let mut clauses = Vec::with_capacity(items.len());
for it in items {
let b = self.node(it, fb, false)?;
exact &= b.exact;
may_error |= b.may_error;
clauses.push(self.eq(&a, &b, pred));
}
Ok(Node {
expr: or_all(clauses),
kind: Kind::Bool,
nullable: false,
maybe_nonfinite: false,
may_error,
exact,
})
}
_ => {
let a = self.node(l, fb, false)?;
let b = self.node(r, fb, false)?;
let exact = a.exact && b.exact;
let may_error = a.may_error || b.may_error;
let mut n = match op {
BinaryOp::Eq => Node::new(self.eq(&a, &b, pred), Kind::Bool).certain().finite(),
BinaryOp::Ne => Node::new(DfExpr::Not(Box::new(self.eq(&a, &b, false))), Kind::Bool)
.certain()
.finite(),
BinaryOp::Lt | BinaryOp::Le | BinaryOp::Gt | BinaryOp::Ge => self.ordered(op, &a, &b, pred)?,
BinaryOp::Add => self.add(&a, &b, l, r)?,
BinaryOp::Sub | BinaryOp::Mul | BinaryOp::Div | BinaryOp::Rem => self.arith(op, &a, &b, l, r)?,
BinaryOp::And | BinaryOp::Or | BinaryOp::In => unreachable!(),
_ => return no("unknown operator"),
};
n.exact = exact;
n.may_error |= may_error;
Ok(n)
}
}
}
fn eq(&self, a: &Node, b: &Node, pred: bool) -> DfExpr {
let both_null = binary(a.expr.clone().is_null(), Operator::And, b.expr.clone().is_null());
match (a.kind, b.kind) {
(Kind::Null, Kind::Null) => lit(true),
(Kind::Null, _) => b.expr.clone().is_null(),
(_, Kind::Null) => a.expr.clone().is_null(),
(x, y) if x != y => {
if a.nullable && b.nullable {
both_null
} else {
lit(false)
}
}
(Kind::Num, Kind::Num) => {
let (x, y) = (a.expr.clone(), b.expr.clone());
let mut clauses = Vec::new();
if a.nullable && b.nullable {
clauses.push((both_null, lit(true)));
}
if !pred {
if let Some(null) = any_null(&[a, b]) {
clauses.push((null, lit(false)));
}
}
if a.maybe_nonfinite && b.maybe_nonfinite {
clauses.push((isnan(x.clone()), lit(false)));
}
let plain = if is_nonzero_literal(&x) || is_nonzero_literal(&y) {
binary(x, Operator::Eq, y)
} else {
binary(
binary(x, Operator::Plus, f64_lit(0.0)),
Operator::Eq,
binary(y, Operator::Plus, f64_lit(0.0)),
)
};
if clauses.is_empty() {
plain
} else {
case(clauses, plain)
}
}
_ => {
if pred && (!a.nullable || !b.nullable) {
binary(a.expr.clone(), Operator::Eq, b.expr.clone())
} else {
binary(a.expr.clone(), Operator::IsNotDistinctFrom, b.expr.clone())
}
}
}
}
fn ordered(&self, op: BinaryOp, a: &Node, b: &Node, pred: bool) -> Result<Node, Unsupported> {
let dfop = match op {
BinaryOp::Lt => Operator::Lt,
BinaryOp::Le => Operator::LtEq,
BinaryOp::Gt => Operator::Gt,
_ => Operator::GtEq,
};
match (a.kind, b.kind) {
(Kind::Null, _) | (_, Kind::Null) => Ok(Node::new(lit(false), Kind::Bool).certain().finite()),
(Kind::Num, Kind::Num) | (Kind::Str, Kind::Str) => {
let mut clauses = Vec::new();
if !pred {
if let Some(null) = any_null(&[a, b]) {
clauses.push((null, lit(false)));
}
}
let plain = if a.kind == Kind::Num {
ordered_numbers(dfop, a, b)
} else {
binary(a.expr.clone(), dfop, b.expr.clone())
};
let expr = if clauses.is_empty() {
plain
} else {
case(clauses, plain)
};
Ok(Node::new(expr, Kind::Bool).certain().finite())
}
(x, y) => no(format!("'{}' between a {x:?} and a {y:?}", op.symbol())),
}
}
fn add(&self, a: &Node, b: &Node, l: &Ast, r: &Ast) -> Result<Node, Unsupported> {
match (a.kind, b.kind) {
(Kind::Null, Kind::Null) => Ok(Node::null(Kind::Null)),
(Kind::Null, k) | (k, Kind::Null) if matches!(k, Kind::Num | Kind::Str) => Ok(Node::null(k)),
(Kind::Num, Kind::Num) => Ok(finite_checked(
Node {
expr: binary(a.expr.clone(), Operator::Plus, b.expr.clone()),
kind: Kind::Num,
nullable: a.nullable || b.nullable,
maybe_nonfinite: true,
may_error: false,
exact: true,
},
"'+'",
a.maybe_nonfinite || b.maybe_nonfinite || self.can_overflow(BinaryOp::Add, a, b, l, r),
)),
(Kind::Str, Kind::Str) => Ok(Node {
expr: binary(a.expr.clone(), Operator::StringConcat, b.expr.clone()),
kind: Kind::Str,
nullable: a.nullable || b.nullable,
maybe_nonfinite: false,
may_error: false,
exact: true,
}),
(x, y) => no(format!("'+' between a {x:?} and a {y:?}")),
}
}
fn arith(&self, op: BinaryOp, a: &Node, b: &Node, l: &Ast, r: &Ast) -> Result<Node, Unsupported> {
match (a.kind, b.kind) {
(Kind::Null, _) | (_, Kind::Null)
if matches!(a.kind, Kind::Null | Kind::Num) && matches!(b.kind, Kind::Null | Kind::Num) =>
{
Ok(Node::null(Kind::Num))
}
(Kind::Num, Kind::Num) => {
let (x, y) = (a.expr.clone(), b.expr.clone());
let dfop = match op {
BinaryOp::Sub => Operator::Minus,
BinaryOp::Mul => Operator::Multiply,
BinaryOp::Div => Operator::Divide,
_ => Operator::Modulo,
};
let guarded = matches!(op, BinaryOp::Div | BinaryOp::Rem) && !is_nonzero_literal(&b.expr);
let plain = binary(x, dfop, y.clone());
let expr = if guarded {
let mut clauses = Vec::new();
if let Some(null) = any_null(&[a, b]) {
clauses.push((null, null_of(Kind::Num)));
}
clauses.push((
binary(math::abs().call(vec![y]), Operator::Eq, f64_lit(0.0)),
error(Kind::Num, "division by zero"),
));
case(clauses, plain)
} else {
plain
};
let node = Node {
expr,
kind: Kind::Num,
nullable: a.nullable || b.nullable,
maybe_nonfinite: true,
may_error: guarded,
exact: true,
};
let possible = a.maybe_nonfinite || b.maybe_nonfinite || self.can_overflow(op, a, b, l, r);
Ok(finite_checked(node, &format!("'{}'", op.symbol()), possible))
}
(x, y) => no(format!("'{}' between a {x:?} and a {y:?}", op.symbol())),
}
}
fn can_overflow(&self, op: BinaryOp, a: &Node, b: &Node, l: &Ast, r: &Ast) -> bool {
if literal_rules_out_overflow(op, a, b) {
return false;
}
let (x, y) = (self.bound(l), self.bound(r));
let worst = match op {
BinaryOp::Add | BinaryOp::Sub => x + y,
BinaryOp::Mul => x * y,
BinaryOp::Div => match f64_literal(&b.expr) {
Some(c) if c != 0.0 => x / c.abs(),
_ => f64::INFINITY,
},
_ => return false,
};
worst >= f64::MAX / 2.0
}
fn bound(&self, ast: &Ast) -> f64 {
match ast {
Ast::Lit(Value::Num(n)) => n.abs(),
Ast::Id(name) => match self.schema.field_with_name(name).map(|f| f.data_type()) {
Ok(DataType::Int8 | DataType::UInt8) => 256.0,
Ok(DataType::Int16 | DataType::UInt16) => 65_536.0,
Ok(DataType::Int32 | DataType::UInt32) => 4_294_967_296.0,
Ok(DataType::Int64 | DataType::UInt64) => 18_446_744_073_709_551_616.0,
_ => f64::INFINITY,
},
Ast::Unary(UnaryOp::Neg, e) => self.bound(e),
Ast::Binary(op, l, r) => {
let (x, y) = (self.bound(l), self.bound(r));
match op {
BinaryOp::Add | BinaryOp::Sub => x + y,
BinaryOp::Mul => x * y,
BinaryOp::Rem => x,
BinaryOp::Div => match r.as_ref() {
Ast::Lit(Value::Num(c)) if *c != 0.0 => x / c.abs(),
_ => f64::INFINITY,
},
_ => f64::INFINITY,
}
}
Ast::Ternary(_, t, e) => self.bound(t).max(self.bound(e)),
Ast::Call(f, args) => match (f.name(), args.as_slice()) {
("abs" | "floor" | "ceil" | "trunc" | "round", [a]) => self.bound(a) + 1.0,
("sign" | "sin" | "cos" | "tanh", _) => 1.0,
("atan" | "atan2", _) => std::f64::consts::PI,
("min" | "max" | "coalesce", args) => args.iter().map(|a| self.bound(a)).fold(0.0, f64::max),
_ => f64::INFINITY,
},
_ => f64::INFINITY,
}
}
fn extreme(&self, name: &str, args: &[Node]) -> Result<Node, Unsupported> {
let mut exprs: Vec<DfExpr> = args
.iter()
.filter(|a| a.kind != Kind::Null)
.map(|a| a.expr.clone())
.collect();
let expr = if exprs.len() == 1 {
exprs.remove(0)
} else if name == "min" {
core::least().call(exprs)
} else {
core::greatest().call(exprs)
};
let node = Node {
expr,
kind: Kind::Num,
nullable: args.iter().all(|a| a.nullable),
maybe_nonfinite: true,
may_error: args.iter().any(|a| a.may_error),
exact: args.iter().all(|a| a.exact),
};
Ok(finite_checked(node, name, args.iter().any(|a| a.maybe_nonfinite)))
}
fn round_to(&self, a: &Node, d: &Node) -> Result<Node, Unsupported> {
let Some(d) = f64_literal(&d.expr).filter(|d| d.fract() == 0.0) else {
return no("roundTo with digits that are not a whole-number literal");
};
let factor = 10_f64.powi(d as i32);
let x = a.expr.clone();
let limit = if factor.is_finite() && factor != 0.0 {
largest_finite_multiplicand(factor)
} else {
f64::MAX
};
let expr = if factor == 0.0 {
if a.nullable {
case(vec![(x.is_null(), null_of(Kind::Num))], f64_lit(0.0))
} else {
f64_lit(0.0)
}
} else if factor.is_infinite() {
x
} else {
let scaled = binary(x.clone(), Operator::Multiply, f64_lit(factor));
let rounded = binary(math::round().call(vec![scaled]), Operator::Divide, f64_lit(factor));
match limit {
f64::MAX => rounded,
limit => case(
vec![(
binary(math::abs().call(vec![x.clone()]), Operator::Gt, f64_lit(limit)),
x,
)],
rounded,
),
}
};
let overflow = factor.is_finite() && factor != 0.0 && !((limit * factor).round() / factor).is_finite();
let node = Node {
expr,
kind: Kind::Num,
nullable: a.nullable,
maybe_nonfinite: true,
may_error: a.may_error,
exact: a.exact,
};
Ok(finite_checked(
node,
"roundTo",
factor != 0.0 && (a.maybe_nonfinite || overflow),
))
}
fn within_dates(&self, a: &Node, expr: DfExpr, ast: &Ast, kind: Kind) -> Result<DfExpr, Unsupported> {
use chrono::{DateTime, Utc};
let (min, max) = (
DateTime::<Utc>::MIN_UTC.timestamp_millis() as f64,
DateTime::<Utc>::MAX_UTC.timestamp_millis() as f64,
);
let x = a.expr.clone();
let has_date = binary(
binary(x.clone(), Operator::Gt, f64_lit(min - 1.0)),
Operator::And,
binary(x.clone(), Operator::Lt, f64_lit(max + 1.0)),
);
let mut clauses = Vec::new();
if a.nullable {
clauses.push((x.is_null(), null_of(kind)));
}
clauses.push((has_date, expr));
Ok(case(clauses, self.native(ast, kind)?))
}
fn integer_argument(&self, ast: &Ast) -> bool {
let Ast::Call(_, args) = ast else { return false };
let [Ast::Id(name)] = args.as_slice() else { return false };
self.schema.field_with_name(name).is_ok_and(|f| {
matches!(
f.data_type(),
DataType::Int8
| DataType::Int16
| DataType::Int32
| DataType::Int64
| DataType::UInt8
| DataType::UInt16
| DataType::UInt32
)
})
}
fn native(&self, ast: &Ast, kind: Kind) -> Result<DfExpr, Unsupported> {
fallback(&Expr::from_ast(ast.clone()), data_type_for(kind), &self.schema)
}
fn call(&self, name: &str, args: Vec<Node>, ast: &Ast) -> Result<Node, Unsupported> {
let exact = args.iter().all(|a| a.exact);
let nullable = args.iter().any(|a| a.nullable);
let may_error = args.iter().any(|a| a.may_error);
let num = |expr: DfExpr, nullable: bool| Node {
expr,
kind: Kind::Num,
nullable,
maybe_nonfinite: true,
may_error,
exact,
};
let num1 = |f: Udf, a: &Node| -> Result<Node, Unsupported> {
match a.kind {
Kind::Num => Ok(num(f().call(vec![a.expr.clone()]), a.nullable)),
Kind::Null => Ok(Node::null(Kind::Num)),
k => no(format!("{name} on a {k:?}")),
}
};
let str1 = |f: Udf, a: &Node, kind: Kind| -> Result<Node, Unsupported> {
match a.kind {
Kind::Str => Ok(Node {
expr: f().call(vec![a.expr.clone()]),
kind,
nullable: a.nullable,
maybe_nonfinite: false,
may_error,
exact,
}),
Kind::Null => Ok(Node::null(kind)),
k => no(format!("{name} on a {k:?}")),
}
};
let keeps = |n: Node, a: &Node| finite_checked(n, name, a.maybe_nonfinite);
let grows = |n: Node| finite_checked(n, name, true);
let all_num = |args: &[Node]| args.iter().all(|a| a.kind == Kind::Num);
let all_str = |args: &[Node]| args.iter().all(|a| a.kind == Kind::Str);
let any_null_kind = |args: &[Node]| args.iter().any(|a| a.kind == Kind::Null);
let all_num_or_null = |args: &[Node]| args.iter().all(|a| matches!(a.kind, Kind::Num | Kind::Null));
let all_str_or_null = |args: &[Node]| args.iter().all(|a| matches!(a.kind, Kind::Str | Kind::Null));
match (name, args.as_slice()) {
("abs", [a]) => Ok(keeps(num1(math::abs, a)?, a)),
("floor", [a]) => Ok(keeps(num1(math::floor, a)?, a)),
("ceil", [a]) => Ok(keeps(num1(math::ceil, a)?, a)),
("trunc", [a]) => Ok(keeps(num1(math::trunc, a)?, a)),
("round", [a]) => Ok(keeps(num1(math::round, a)?, a)),
("sign", [a]) => Ok(keeps(num1(math::signum, a)?, a)),
("cbrt", [a]) => Ok(keeps(num1(math::cbrt, a)?, a)),
("sin", [a]) => Ok(keeps(num1(math::sin, a)?, a)),
("cos", [a]) => Ok(keeps(num1(math::cos, a)?, a)),
("tan", [a]) => Ok(keeps(num1(math::tan, a)?, a)),
("atan", [a]) => Ok(keeps(num1(math::atan, a)?, a)),
("tanh", [a]) => Ok(keeps(num1(math::tanh, a)?, a)),
("degToRad", [a]) => Ok(keeps(num1(math::radians, a)?, a)),
("exp", [a]) => Ok(grows(num1(math::exp, a)?)),
("sinh", [a]) => Ok(grows(num1(math::sinh, a)?)),
("cosh", [a]) => Ok(grows(num1(math::cosh, a)?)),
("radToDeg", [a]) => Ok(grows(num1(math::degrees, a)?)),
("sqrt", [a]) => Ok(keeps(
guarded(num1(math::sqrt, a)?, a, Guard::NonNegative, "sqrt: negative argument"),
a,
)),
("ln", [a]) => Ok(keeps(
guarded(num1(math::ln, a)?, a, Guard::Positive, "ln: argument must be positive"),
a,
)),
("log10", [a]) => Ok(keeps(
guarded(
num1(math::log10, a)?,
a,
Guard::Positive,
"log10: argument must be positive",
),
a,
)),
("log2", [a]) => Ok(keeps(
guarded(
num1(math::log2, a)?,
a,
Guard::Positive,
"log2: argument must be positive",
),
a,
)),
("asin", [a]) => Ok(keeps(
guarded(
num1(math::asin, a)?,
a,
Guard::UnitInterval,
"asin: argument must be in [-1, 1]",
),
a,
)),
("acos", [a]) => Ok(keeps(
guarded(
num1(math::acos, a)?,
a,
Guard::UnitInterval,
"acos: argument must be in [-1, 1]",
),
a,
)),
("roundTo" | "pow" | "atan2" | "min" | "max", _) if any_null_kind(&args) && all_num_or_null(&args) => {
match name {
"min" | "max" if !args.iter().all(|a| a.kind == Kind::Null) => self.extreme(name, &args),
_ => Ok(Node::null(Kind::Num)),
}
}
("roundTo", [a, d]) if all_num(&args) => self.round_to(a, d),
("pow", [a, b]) if all_num(&args) => {
let zero_to_negative = binary(
binary(math::abs().call(vec![a.expr.clone()]), Operator::Eq, f64_lit(0.0)),
Operator::And,
binary(b.expr.clone(), Operator::Lt, f64_lit(-0.0)),
);
let power = math::power().call(vec![a.expr.clone(), b.expr.clone()]);
Ok(grows(num(
case(
vec![(zero_to_negative, error(Kind::Num, "pow: result is not a finite number"))],
power,
),
nullable,
)))
}
("atan2", [a, b]) if all_num(&args) => Ok(finite_checked(
num(math::atan2().call(vec![a.expr.clone(), b.expr.clone()]), nullable),
name,
a.maybe_nonfinite || b.maybe_nonfinite,
)),
("min" | "max", _) if !args.is_empty() && all_num(&args) => self.extreme(name, &args),
("pi", []) => Ok(Node::new(f64_lit(std::f64::consts::PI), Kind::Num).certain().finite()),
("coalesce", _) => {
let kind = common_kind(args.iter().map(|a| Some(a.kind))).ok_or_else(|| Unsupported {
reason: "coalesce over mixed kinds".into(),
})?;
let exprs: Vec<DfExpr> = args.iter().map(|a| retype(a.expr.clone(), a.kind, kind)).collect();
Ok(Node {
expr: core::coalesce().call(exprs),
kind,
nullable: args.iter().all(|a| a.nullable),
maybe_nonfinite: args.iter().any(|a| a.maybe_nonfinite),
may_error,
exact,
})
}
("isNull", [a]) => Ok(Node {
expr: a.expr.clone().is_null(),
kind: Kind::Bool,
nullable: false,
maybe_nonfinite: false,
may_error,
exact,
}),
("isNumber" | "isString" | "isBoolean", [a]) => {
let want = match name {
"isNumber" => Kind::Num,
"isString" => Kind::Str,
_ => Kind::Bool,
};
let expr = if a.kind == want {
a.expr.clone().is_not_null()
} else {
lit(false)
};
Ok(Node {
expr,
kind: Kind::Bool,
nullable: false,
maybe_nonfinite: false,
may_error,
exact,
})
}
("typeOf", [a]) => {
let type_name = match a.kind {
Kind::Num => "number",
Kind::Str => "string",
Kind::Bool => "boolean",
Kind::Null => "null",
};
let expr = if a.nullable && a.kind != Kind::Null {
case(vec![(a.expr.clone().is_null(), lit("null"))], lit(type_name))
} else {
lit(type_name)
};
Ok(Node {
expr,
kind: Kind::Str,
nullable: false,
maybe_nonfinite: false,
may_error,
exact,
})
}
("nullIf", [a, b]) => {
let kind = common_kind([Some(a.kind), Some(b.kind)].into_iter()).ok_or_else(|| Unsupported {
reason: "nullIf over mixed kinds".into(),
})?;
Ok(Node {
expr: case(
vec![(self.eq(a, b, false), null_of(kind))],
retype(a.expr.clone(), a.kind, kind),
),
kind,
nullable: true,
maybe_nonfinite: a.maybe_nonfinite,
may_error,
exact,
})
}
("upper", [a]) => str1(string::upper, a, Kind::Str),
("lower", [a]) => str1(string::lower, a, Kind::Str),
("reverse", [a]) => str1(unicode::reverse, a, Kind::Str),
("length", [a]) => match a.kind {
Kind::Str => Ok(Node {
expr: to_f64(unicode::character_length().call(vec![a.expr.clone()])),
kind: Kind::Num,
nullable: a.nullable,
maybe_nonfinite: false,
may_error,
exact,
}),
Kind::Null => Ok(Node::null(Kind::Num)),
k => no(format!("length on a {k:?}")),
},
("startsWith" | "endsWith" | "contains" | "regexMatch", _)
if any_null_kind(&args) && all_str_or_null(&args) =>
{
Ok(Node::null(Kind::Bool))
}
("startsWith" | "endsWith" | "contains", [a, b]) if all_str(&args) => {
let f = match name {
"startsWith" => string::starts_with(),
"endsWith" => string::ends_with(),
_ => string::contains(),
};
Ok(Node {
expr: f.call(vec![a.expr.clone(), b.expr.clone()]),
kind: Kind::Bool,
nullable,
maybe_nonfinite: false,
may_error,
exact,
})
}
("regexMatch", [a, b]) if a.kind == Kind::Str && valid_pattern(&b.expr) => Ok(Node {
expr: regex::regexp_like().call(vec![a.expr.clone(), b.expr.clone()]),
kind: Kind::Bool,
nullable,
maybe_nonfinite: false,
may_error,
exact,
}),
("replace" | "indexOf", _) if any_null_kind(&args) && all_str_or_null(&args) => {
Ok(Node::null(if name == "replace" { Kind::Str } else { Kind::Num }))
}
("splitPart", [a, b, c])
if a.kind == Kind::Str
&& b.kind == Kind::Str
&& f64_literal(&c.expr).is_some_and(|n| n != 0.0 && n.abs() < TWO_POW_63) =>
{
Ok(Node {
expr: string::split_part().call(vec![
a.expr.clone(),
b.expr.clone(),
cast(c.expr.clone(), DataType::Int64),
]),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact,
})
}
("left" | "right", [a, b]) if a.kind == Kind::Str && whole_literal(&b.expr) => Ok(Node {
expr: (if name == "left" {
unicode::left()
} else {
unicode::right()
})
.call(vec![a.expr.clone(), cast(b.expr.clone(), DataType::Int64)]),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact,
}),
("substr" | "substring", [a, b]) if a.kind == Kind::Str && whole_literal(&b.expr) => Ok(Node {
expr: unicode::substr().call(vec![a.expr.clone(), cast(b.expr.clone(), DataType::Int64)]),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact,
}),
("substr" | "substring", [a, b, c])
if a.kind == Kind::Str && whole_literal(&b.expr) && non_negative_literal(&c.expr) =>
{
Ok(Node {
expr: unicode::substr().call(vec![
a.expr.clone(),
cast(b.expr.clone(), DataType::Int64),
cast(c.expr.clone(), DataType::Int64),
]),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact,
})
}
("padStart" | "padEnd", [a, b, c])
if a.kind == Kind::Str
&& f64_literal(&b.expr)
.is_some_and(|n| n.trunc().abs() <= (fv_value::MAX_BUILT_BYTES / 4) as f64)
&& c.kind == Kind::Str =>
{
let pad = (if name == "padStart" {
unicode::lpad()
} else {
unicode::rpad()
})
.call(vec![
a.expr.clone(),
cast(b.expr.clone(), DataType::Int64),
c.expr.clone(),
]);
let too_short = unicode::character_length()
.call(vec![a.expr.clone()])
.lt(cast(b.expr.clone(), DataType::Int32));
let has_pad = unicode::character_length().call(vec![c.expr.clone()]).gt(lit(0i32));
Ok(Node {
expr: when(too_short.and(has_pad), pad)
.otherwise(a.expr.clone())
.expect("case"),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact,
})
}
("regexExtract", [a, b]) if a.kind == Kind::Str && valid_pattern(&b.expr) => Ok(Node {
expr: array_element(
regex::regexp_match().call(vec![a.expr.clone(), b.expr.clone()]),
lit(1i64),
),
kind: Kind::Str,
nullable: true,
maybe_nonfinite: false,
may_error,
exact,
}),
("epochToIso", [a]) if a.kind == Kind::Num => Ok(Node {
expr: self.within_dates(
a,
datetime::expr_fn::to_char(epoch_millis(a), lit("%Y-%m-%dT%H:%M:%S%.3fZ")),
ast,
Kind::Str,
)?,
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error: true,
exact,
}),
("toString", [a]) if a.kind == Kind::Str => Ok(a.clone()),
("toString", [a]) if a.kind == Kind::Bool => Ok(Node {
expr: when(a.expr.clone(), lit("true")).otherwise(lit("false")).expect("case"),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact,
}),
("toString", [a]) if a.kind == Kind::Num && self.integer_argument(ast) => {
let x = a.expr.clone();
Ok(Node {
expr: case(
vec![(
binary(x.clone(), Operator::Eq, f64_lit(TWO_POW_63)),
lit("9223372036854775808"),
)],
cast(cast(x, DataType::Int64), DataType::Utf8),
),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact,
})
}
("toString", [a]) if a.kind == Kind::Num => {
let x = a.expr.clone();
let magnitude = math::abs().call(vec![x.clone()]);
let whole = binary(
binary(math::floor().call(vec![x.clone()]), Operator::Eq, x.clone()),
Operator::And,
binary(magnitude.clone(), Operator::Lt, f64_lit(TWO_POW_63)),
);
let plain = binary(
binary(magnitude.clone(), Operator::GtEq, f64_lit(PLAIN_FLOAT_TEXT.start)),
Operator::And,
binary(magnitude, Operator::Lt, f64_lit(PLAIN_FLOAT_TEXT.end)),
);
let mut clauses = Vec::new();
if a.nullable {
clauses.push((x.clone().is_null(), null_of(Kind::Str)));
}
clauses.push((whole, cast(cast(x.clone(), DataType::Int64), DataType::Utf8)));
clauses.push((plain, cast(x, DataType::Utf8)));
Ok(Node {
expr: case(clauses, self.native(ast, Kind::Str)?),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact: false,
})
}
("replace", [a, b, c]) if all_str(&args) => Ok(Node {
expr: string::replace().call(vec![a.expr.clone(), b.expr.clone(), c.expr.clone()]),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact,
}),
("indexOf", [a, b]) if all_str(&args) => {
let pos = to_f64(unicode::strpos().call(vec![a.expr.clone(), b.expr.clone()]));
let mut clauses = Vec::new();
if let Some(null) = any_null(&[a, b]) {
clauses.push((null, null_of(Kind::Num)));
}
clauses.push((binary(pos.clone(), Operator::Eq, f64_lit(0.0)), f64_lit(-1.0)));
Ok(Node {
expr: case(clauses, binary(pos, Operator::Minus, f64_lit(1.0))),
kind: Kind::Num,
nullable,
maybe_nonfinite: false,
may_error,
exact,
})
}
("yearOf" | "monthOf" | "dayOf" | "hourOf" | "minuteOf" | "dayOfWeekOf" | "dayOfYear", [a]) => match a.kind
{
Kind::Num => {
let ts = epoch_millis(a);
let part = |p: &str| to_f64(datetime::date_part().call(vec![lit(p), ts.clone()]));
let expr = match name {
"yearOf" => part("year"),
"monthOf" => part("month"),
"dayOf" => part("day"),
"hourOf" => part("hour"),
"minuteOf" => part("minute"),
"dayOfYear" => part("doy"),
_ => {
let dow = part("dow");
case(
vec![(binary(dow.clone(), Operator::Eq, f64_lit(0.0)), f64_lit(7.0))],
dow,
)
}
};
Ok(Node {
expr: self.within_dates(a, expr, ast, Kind::Num)?,
kind: Kind::Num,
nullable: a.nullable,
maybe_nonfinite: false,
may_error: true,
exact,
})
}
Kind::Null => Ok(Node::null(Kind::Num)),
k => no(format!("{name} on a {k:?}")),
},
("concat", _) if any_null_kind(&args) && all_str_or_null(&args) => Ok(Node::null(Kind::Str)),
("concatWs" | "trim" | "trimStart" | "trimEnd" | "repeat" | "regexReplace", [first, ..])
if first.kind == Kind::Null =>
{
Ok(Node::null(Kind::Str))
}
("concat", [first, rest @ ..]) if all_str(&args) => Ok(Node {
expr: rest.iter().fold(first.expr.clone(), |l, r| {
binary(l, Operator::StringConcat, r.expr.clone())
}),
kind: Kind::Str,
nullable,
maybe_nonfinite: false,
may_error,
exact,
}),
("concatWs", [sep, parts @ ..])
if sep.kind == Kind::Str
&& !parts.is_empty()
&& parts.iter().all(|a| matches!(a.kind, Kind::Str | Kind::Null)) =>
{
let mut exprs = vec![sep.expr.clone()];
exprs.extend(parts.iter().map(|a| retype(a.expr.clone(), a.kind, Kind::Str)));
Ok(Node {
expr: string::concat_ws().call(exprs),
kind: Kind::Str,
nullable: sep.nullable,
maybe_nonfinite: false,
may_error,
exact,
})
}
("trim" | "trimStart" | "trimEnd", [a]) if a.kind == Kind::Str => Ok(Node {
expr: match name {
"trim" => string::btrim(),
"trimStart" => string::ltrim(),
_ => string::rtrim(),
}
.call(vec![a.expr.clone(), lit(unicode_whitespace())]),
kind: Kind::Str,
nullable: a.nullable,
maybe_nonfinite: false,
may_error,
exact,
}),
("repeat", [a, n])
if a.kind == Kind::Str
&& f64_literal(&n.expr).is_some_and(|c| c.trunc() <= fv_value::MAX_BUILT_BYTES as f64) =>
{
let count = f64_literal(&n.expr).unwrap_or(0.0).trunc().max(0.0);
let bytes = binary(
to_f64(string::octet_length().call(vec![a.expr.clone()])),
Operator::Multiply,
f64_lit(count),
);
Ok(Node {
expr: case(
vec![(
binary(bytes, Operator::Gt, f64_lit(fv_value::MAX_BUILT_BYTES as f64)),
self.native(ast, Kind::Str)?,
)],
string::repeat().call(vec![a.expr.clone(), lit(count as i64)]),
),
kind: Kind::Str,
nullable: a.nullable,
maybe_nonfinite: false,
may_error: true,
exact,
})
}
("regexReplace", [a, p, r]) if a.kind == Kind::Str && valid_pattern(&p.expr) => {
let DfExpr::Literal(ScalarValue::Utf8(Some(rep)), _) = &r.expr else {
return no("regexReplace with a replacement that is not a literal");
};
let rep = fv_value::regex_replacement(rep);
if rep.contains('\\') {
return no("regexReplace with a backslash in the replacement");
}
Ok(Node {
expr: regex::regexp_replace().call(vec![
a.expr.clone(),
p.expr.clone(),
lit(rep.into_owned()),
lit("g"),
]),
kind: Kind::Str,
nullable: a.nullable,
maybe_nonfinite: false,
may_error,
exact,
})
}
("clamp", [x, lo, hi]) if all_num(&args) => {
let (Some(l), Some(h)) = (f64_literal(&lo.expr), f64_literal(&hi.expr)) else {
return no("clamp with bounds that are not literals");
};
let mut clauses = Vec::new();
if x.nullable {
clauses.push((x.expr.clone().is_null(), null_of(Kind::Num)));
}
let otherwise = if l > h {
error(Kind::Num, "clamp: lo must not be greater than hi")
} else {
if x.maybe_nonfinite {
clauses.push((isnan(x.expr.clone()), f64_lit(l)));
}
core::least().call(vec![
core::greatest().call(vec![x.expr.clone(), f64_lit(l)]),
f64_lit(h),
])
};
Ok(Node {
expr: if clauses.is_empty() {
otherwise
} else {
case(clauses, otherwise)
},
kind: Kind::Num,
nullable: x.nullable,
maybe_nonfinite: false,
may_error: may_error || l > h,
exact,
})
}
_ => no(format!("function {name}")),
}
}
}
fn unicode_whitespace() -> &'static str {
static WS: std::sync::LazyLock<String> = std::sync::LazyLock::new(|| {
(0..=0x10FFFF)
.filter_map(char::from_u32)
.filter(|c| c.is_whitespace())
.collect()
});
&WS
}
fn valid_pattern(e: &DfExpr) -> bool {
matches!(e, DfExpr::Literal(ScalarValue::Utf8(Some(p)), _) if ::regex::Regex::new(p).is_ok())
}