use std::fmt;
use datafusion::arrow::datatypes::{DataType, SchemaRef};
use datafusion::common::Column;
use datafusion::config::ConfigOptions;
use datafusion::functions::{core, datetime, math, regex, string, unicode};
use datafusion::functions_nested::expr_fn::array_element;
use datafusion::logical_expr::{cast, lit, when, BinaryExpr, Expr as DfExpr, Operator, ScalarUDF};
use datafusion::scalar::ScalarValue;
use fv_value::{Ast, BinaryOp, Expr, UnaryOp, Value};
use crate::convert::{data_type_for, Kind};
use crate::udf::{fallback, ErrorUdf};
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct Unsupported {
pub reason: String,
}
impl fmt::Display for Unsupported {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
write!(f, "not translatable: {}", self.reason)
}
}
impl std::error::Error for Unsupported {}
fn no<T>(reason: impl Into<String>) -> Result<T, Unsupported> {
Err(Unsupported { reason: reason.into() })
}
#[derive(Debug, Clone)]
pub struct Translation {
pub expr: DfExpr,
pub kind: Kind,
pub exact: bool,
}
#[derive(Debug, Clone, Copy, Default, PartialEq, Eq)]
pub struct Options {
pub assume_no_nan: bool,
}
#[derive(Debug, Clone)]
pub struct Translator {
schema: SchemaRef,
options: Options,
}
#[derive(Debug, Clone)]
struct Node {
expr: DfExpr,
kind: Kind,
nullable: bool,
maybe_nan: bool,
may_error: bool,
exact: bool,
}
impl Node {
fn new(expr: DfExpr, kind: Kind) -> Self {
Node {
expr,
kind,
nullable: true,
maybe_nan: kind == Kind::Num,
may_error: false,
exact: true,
}
}
fn certain(mut self) -> Self {
self.nullable = false;
self
}
fn finite(mut self) -> Self {
self.maybe_nan = false;
self
}
fn fallible(mut self) -> Self {
self.may_error = true;
self
}
fn null(kind: Kind) -> Self {
Node::new(null_of(kind), kind).finite()
}
}
fn f64_lit(x: f64) -> DfExpr {
DfExpr::Literal(ScalarValue::Float64(Some(x)), None)
}
fn null_of(kind: Kind) -> DfExpr {
DfExpr::Literal(
match kind {
Kind::Num => ScalarValue::Float64(None),
Kind::Str => ScalarValue::Utf8(None),
Kind::Bool => ScalarValue::Boolean(None),
Kind::Null => ScalarValue::Null,
},
None,
)
}
fn binary(l: DfExpr, op: Operator, r: DfExpr) -> DfExpr {
DfExpr::BinaryExpr(BinaryExpr::new(Box::new(l), op, Box::new(r)))
}
fn any_null(nodes: &[&Node]) -> Option<DfExpr> {
let checks: Vec<DfExpr> = nodes
.iter()
.filter(|n| n.nullable)
.map(|n| n.expr.clone().is_null())
.collect();
(!checks.is_empty()).then(|| or_all(checks))
}
fn any_nan(nodes: &[&Node]) -> Option<DfExpr> {
let checks: Vec<DfExpr> = nodes
.iter()
.filter(|n| n.maybe_nan)
.map(|n| isnan(n.expr.clone()))
.collect();
(!checks.is_empty()).then(|| or_all(checks))
}
fn or_all(mut items: Vec<DfExpr>) -> DfExpr {
let first = items.remove(0);
items.into_iter().fold(first, |acc, e| binary(acc, Operator::Or, e))
}
fn case(clauses: Vec<(DfExpr, DfExpr)>, otherwise: DfExpr) -> DfExpr {
let mut it = clauses.into_iter();
let (c, t) = it.next().expect("at least one WHEN");
let mut b = when(c, t);
for (c, t) in it {
b = b.when(c, t);
}
b.otherwise(otherwise).expect("a CASE with clauses is well-formed")
}
fn lazy_bool_case(clauses: Vec<(DfExpr, DfExpr)>, otherwise: DfExpr) -> DfExpr {
let num = |b: DfExpr| match b {
DfExpr::Literal(ScalarValue::Boolean(Some(true)), _) => f64_lit(1.0),
DfExpr::Literal(ScalarValue::Boolean(Some(false)), _) => f64_lit(0.0),
other => cast(other, DataType::Float64),
};
let clauses = clauses.into_iter().map(|(c, t)| (c, num(t))).collect();
binary(case(clauses, num(otherwise)), Operator::Eq, f64_lit(1.0))
}
fn error(kind: Kind, message: &str) -> DfExpr {
ErrorUdf::call_expr(data_type_for(kind), message)
}
fn isnan(e: DfExpr) -> DfExpr {
math::isnan().call(vec![e])
}
fn non_negative_literal(e: &DfExpr) -> bool {
matches!(e, DfExpr::Literal(ScalarValue::Float64(Some(n)), _) if *n >= 0.0 && n.fract() == 0.0)
}
fn positive_literal(e: &DfExpr) -> bool {
matches!(e, DfExpr::Literal(ScalarValue::Float64(Some(n)), _) if *n >= 1.0 && n.fract() == 0.0)
}
fn groupless_pattern(e: &DfExpr) -> bool {
let DfExpr::Literal(ScalarValue::Utf8(Some(p)), _) = e else {
return false;
};
let b = p.as_bytes();
let mut i = 0;
while i < b.len() {
match b[i] {
b'\\' => i += 2,
b'(' => {
if !p[i..].starts_with("(?") {
return false;
}
i += 1;
}
_ => i += 1,
}
}
true
}
fn to_f64(e: DfExpr) -> DfExpr {
cast(e, DataType::Float64)
}
fn is_nonzero_literal(e: &DfExpr) -> bool {
matches!(e, DfExpr::Literal(ScalarValue::Float64(Some(x)), _) if *x != 0.0)
}
fn retype(expr: DfExpr, from: Kind, to: Kind) -> DfExpr {
if from == Kind::Null && to != Kind::Null {
null_of(to)
} else {
expr
}
}
fn common_kind(kinds: impl Iterator<Item = Option<Kind>>) -> Option<Kind> {
let mut found: Option<Kind> = None;
for k in kinds {
match k? {
Kind::Null => {}
k => match found {
None => found = Some(k),
Some(f) if f == k => {}
Some(_) => return None,
},
}
}
Some(found.unwrap_or(Kind::Null))
}
fn builtin_kind(name: &str, args: &[Option<Kind>]) -> Option<Kind> {
const NUM: &[&str] = &[
"abs",
"floor",
"ceil",
"trunc",
"round",
"exp",
"sqrt",
"ln",
"log10",
"log2",
"cbrt",
"sign",
"pow",
"hypot",
"degToRad",
"radToDeg",
"min",
"max",
"clamp",
"roundTo",
"toNumber",
"sin",
"cos",
"tan",
"asin",
"acos",
"atan",
"atan2",
"sinh",
"cosh",
"tanh",
"log",
"pi",
"factorial",
"gcd",
"lcm",
"length",
"indexOf",
"ascii",
"epochMs",
"yearOf",
"monthOf",
"dayOf",
"hourOf",
"minuteOf",
"dayOfWeekOf",
"dayOfYear",
"arraySum",
"arrayMin",
"arrayMax",
"haversineKm",
"bearingDeg",
"distance3dM",
];
const STR: &[&str] = &[
"upper",
"lower",
"trim",
"trimStart",
"trimEnd",
"capitalize",
"reverse",
"concat",
"replace",
"repeat",
"substr",
"left",
"right",
"splitPart",
"concatWs",
"chr",
"toHex",
"regexReplace",
"substring",
"padStart",
"padEnd",
"toString",
"regexExtract",
"arrayJoin",
"epochToIso",
"typeOf",
"groundStrike",
];
const BOOL: &[&str] = &[
"contains",
"startsWith",
"endsWith",
"regexMatch",
"isNull",
"isNumber",
"isString",
"isBoolean",
"arrayContains",
"geoContains",
"geoWithin",
"geoIntersects",
"altitudeBetween",
"geoWithinVolume",
];
if NUM.contains(&name) {
Some(Kind::Num)
} else if STR.contains(&name) {
Some(Kind::Str)
} else if BOOL.contains(&name) {
Some(Kind::Bool)
} else if name == "coalesce" || name == "nullIf" {
common_kind(args.iter().copied())
} else {
None
}
}
#[derive(Clone, Copy)]
enum Guard {
NonNegative,
Positive,
UnitInterval,
}
fn guarded(node: Node, arg: &Node, guard: Guard, message: &str) -> Node {
if arg.kind != Kind::Num {
return node;
}
let x = arg.expr.clone();
let out_of_domain = match guard {
Guard::NonNegative => binary(x, Operator::Lt, f64_lit(0.0)),
Guard::Positive => binary(x, Operator::LtEq, f64_lit(0.0)),
Guard::UnitInterval => binary(
binary(x.clone(), Operator::Lt, f64_lit(-1.0)),
Operator::Or,
binary(x, Operator::Gt, f64_lit(1.0)),
),
};
let mut clauses = Vec::new();
if arg.nullable {
clauses.push((arg.expr.clone().is_null(), null_of(Kind::Num)));
}
clauses.push((out_of_domain, error(Kind::Num, message)));
Node {
expr: case(clauses, node.expr),
may_error: true,
..node
}
}
type Udf = fn() -> std::sync::Arc<ScalarUDF>;
impl Translator {
pub fn new(schema: SchemaRef) -> Self {
Translator {
schema,
options: Options::default(),
}
}
pub fn with_options(mut self, options: Options) -> Self {
self.options = options;
self
}
pub fn translate(&self, expr: &Expr) -> Result<Translation, Unsupported> {
let n = self.node(expr.ast(), false, false)?;
Ok(Translation {
expr: n.expr,
kind: n.kind,
exact: true,
})
}
pub fn translate_or_fallback(&self, expr: &Expr) -> Result<Translation, Unsupported> {
let n = self.node(expr.ast(), true, false)?;
Ok(Translation {
expr: n.expr,
kind: n.kind,
exact: n.exact,
})
}
pub fn translate_predicate(&self, expr: &Expr) -> Result<Translation, Unsupported> {
let n = self.node(expr.ast(), false, true)?;
if n.kind != Kind::Bool {
return no(format!("a filter must be a boolean, this is a {:?}", n.kind));
}
Ok(Translation {
expr: n.expr,
kind: n.kind,
exact: true,
})
}
pub fn translate_predicate_or_fallback(&self, expr: &Expr) -> Result<Translation, Unsupported> {
let n = self.node(expr.ast(), true, true)?;
if n.kind != Kind::Bool {
return no(format!("a filter must be a boolean, this is a {:?}", n.kind));
}
Ok(Translation {
expr: n.expr,
kind: n.kind,
exact: n.exact,
})
}
pub fn kind_of(&self, expr: &Expr) -> Option<Kind> {
self.kind(expr.ast())
}
fn column_kind(&self, name: &str) -> Option<Kind> {
self.schema
.field_with_name(name)
.ok()
.and_then(|f| Kind::of_data_type(f.data_type()))
}
fn kind(&self, ast: &Ast) -> Option<Kind> {
Some(match ast {
Ast::Lit(v) => match v {
Value::Null => Kind::Null,
Value::Bool(_) => Kind::Bool,
Value::Num(_) => Kind::Num,
Value::Str(_) => Kind::Str,
_ => return None,
},
Ast::Id(name) => match self.schema.field_with_name(name) {
Ok(_) => self.column_kind(name)?,
Err(_) => Kind::Null, },
Ast::Member(..) | Ast::Index(..) | Ast::List(_) => return None,
Ast::Unary(UnaryOp::Not, _) => Kind::Bool,
Ast::Unary(UnaryOp::Neg, _) => Kind::Num,
Ast::Binary(op, l, r) => match op {
BinaryOp::Or
| BinaryOp::And
| BinaryOp::Eq
| BinaryOp::Ne
| BinaryOp::In
| BinaryOp::Lt
| BinaryOp::Le
| BinaryOp::Gt
| BinaryOp::Ge => Kind::Bool,
BinaryOp::Add => common_kind([self.kind(l), self.kind(r)].into_iter())?,
BinaryOp::Sub | BinaryOp::Mul | BinaryOp::Div | BinaryOp::Rem => Kind::Num,
_ => return None,
},
Ast::Ternary(_, t, e) => common_kind([self.kind(t), self.kind(e)].into_iter())?,
Ast::Call(f, args) => {
let kinds: Vec<Option<Kind>> = args.iter().map(|a| self.kind(a)).collect();
builtin_kind(f.name(), &kinds)?
}
_ => return None,
})
}
fn node(&self, ast: &Ast, fb: bool, pred: bool) -> Result<Node, Unsupported> {
match self.node_inner(ast, fb, pred) {
Ok(n) => Ok(n),
Err(u) if fb => match self.kind(ast) {
Some(kind) => {
let expr = Expr::from_ast(ast.clone());
let mut n = Node::new(fallback(&expr, data_type_for(kind), &self.schema), kind).fallible();
n.exact = false;
Ok(n)
}
None => Err(u),
},
Err(u) => Err(u),
}
}
fn node_inner(&self, ast: &Ast, fb: bool, pred: bool) -> Result<Node, Unsupported> {
if !matches!(ast, Ast::Lit(_)) && ast.identifiers().is_empty() {
return self.constant(ast);
}
match ast {
Ast::Lit(v) => Ok(match v {
Value::Null => Node::null(Kind::Null),
Value::Bool(b) => Node::new(lit(*b), Kind::Bool).certain().finite(),
Value::Num(n) => {
let node = Node::new(f64_lit(*n), Kind::Num).certain();
if n.is_nan() {
node
} else {
node.finite()
}
}
Value::Str(s) => Node::new(lit(s.as_str()), Kind::Str).certain().finite(),
Value::List(_) | Value::Obj(_) => return no("list and object literals"),
}),
Ast::Id(name) => {
let Ok(field) = self.schema.field_with_name(name) else {
return Ok(Node::null(Kind::Null));
};
let kind = Kind::of_data_type(field.data_type()).ok_or_else(|| Unsupported {
reason: format!("column '{name}' has type {}", field.data_type()),
})?;
let column = DfExpr::Column(Column::from_name(name));
let expr = match (kind, field.data_type()) {
(Kind::Num, DataType::Float64) => column,
(Kind::Num, _) => to_f64(column),
_ => column,
};
let mut n = Node::new(expr, kind);
n.nullable = field.is_nullable();
n.maybe_nan =
!self.options.assume_no_nan && matches!(field.data_type(), DataType::Float32 | DataType::Float64);
Ok(n)
}
Ast::Member(..) => no("member access on objects"),
Ast::Index(..) => no("indexing"),
Ast::List(_) => no("list values"),
Ast::Unary(UnaryOp::Not, a) => {
let a = self.must_bool(self.node(a, fb, false)?, "'!'")?;
Ok(Node {
expr: DfExpr::Not(Box::new(a.expr)),
kind: Kind::Bool,
nullable: false,
maybe_nan: false,
may_error: a.may_error,
exact: a.exact,
})
}
Ast::Unary(UnaryOp::Neg, a) => {
let a = self.node(a, fb, false)?;
match a.kind {
Kind::Null => Ok(Node::null(Kind::Num)),
Kind::Num => Ok(Node {
expr: DfExpr::Negative(Box::new(a.expr)),
..a
}),
k => no(format!("unary '-' on a {k:?}")),
}
}
Ast::Ternary(c, t, e) => {
let c = self.must_bool(self.node(c, fb, false)?, "'?:' condition")?;
let t = self.node(t, fb, false)?;
let e = self.node(e, fb, false)?;
let kind = common_kind([Some(t.kind), Some(e.kind)].into_iter()).ok_or_else(|| Unsupported {
reason: "'?:' branches of different kinds".into(),
})?;
let (te, ee) = (retype(t.expr, t.kind, kind), retype(e.expr, e.kind, kind));
let expr = if kind == Kind::Bool {
lazy_bool_case(vec![(c.expr, te)], ee)
} else {
case(vec![(c.expr, te)], ee)
};
Ok(Node {
expr,
kind,
nullable: t.nullable || e.nullable,
maybe_nan: t.maybe_nan || e.maybe_nan,
may_error: c.may_error || t.may_error || e.may_error,
exact: c.exact && t.exact && e.exact,
})
}
Ast::Binary(op, l, r) => self.binary_node(*op, l, r, fb, pred),
Ast::Call(f, args) => {
let nodes = args
.iter()
.map(|a| self.node(a, fb, false))
.collect::<Result<Vec<_>, _>>()?;
self.call(f.name(), nodes)
}
_ => no("unknown node"),
}
}
fn constant(&self, ast: &Ast) -> Result<Node, Unsupported> {
let expr = Expr::from_ast(ast.clone());
match expr.eval(&fv_value::Empty) {
Ok(Value::Null) => Ok(Node::null(Kind::Null)),
Ok(Value::Bool(b)) => Ok(Node::new(lit(b), Kind::Bool).certain().finite()),
Ok(Value::Num(n)) => {
let node = Node::new(f64_lit(n), Kind::Num).certain();
Ok(if n.is_nan() { node } else { node.finite() })
}
Ok(Value::Str(s)) => Ok(Node::new(lit(s.as_str()), Kind::Str).certain().finite()),
Ok(other) => no(format!("constant {} values", other.type_name())),
Err(e) => match self.kind(ast) {
Some(kind) if kind != Kind::Null => Ok(Node::new(error(kind, &e.to_string()), kind)
.certain()
.finite()
.fallible()),
_ => no(format!("a constant that errors ({e}) with no known kind")),
},
}
}
fn must_bool(&self, n: Node, where_: &str) -> Result<Node, Unsupported> {
match n.kind {
Kind::Bool if !n.nullable => Ok(n),
Kind::Bool => Ok(Node {
expr: lazy_bool_case(
vec![(
n.expr.clone().is_null(),
error(Kind::Bool, &format!("{where_}: expected a boolean, got null")),
)],
n.expr,
),
nullable: false,
may_error: true,
..n
}),
k => no(format!("{where_} on a {k:?}")),
}
}
fn binary_node(&self, op: BinaryOp, l: &Ast, r: &Ast, fb: bool, pred: bool) -> Result<Node, Unsupported> {
match op {
BinaryOp::And | BinaryOp::Or => {
let a = self.must_bool(self.node(l, fb, pred)?, op.symbol())?;
let b = self.must_bool(self.node(r, fb, pred)?, op.symbol())?;
let may_error = a.may_error || b.may_error;
let expr = match (op, b.may_error) {
(BinaryOp::And, false) => binary(a.expr, Operator::And, b.expr),
(BinaryOp::And, true) => lazy_bool_case(vec![(a.expr, b.expr)], lit(false)),
(_, false) => binary(a.expr, Operator::Or, b.expr),
(_, true) => lazy_bool_case(vec![(a.expr, lit(true))], b.expr),
};
Ok(Node {
expr,
kind: Kind::Bool,
nullable: false,
maybe_nan: false,
may_error,
exact: a.exact && b.exact,
})
}
BinaryOp::In => {
let a = self.node(l, fb, false)?;
let Ast::List(items) = r else {
return no("'in' with a non-literal list");
};
if items.is_empty() {
return Ok(Node::new(lit(false), Kind::Bool).certain().finite());
}
let mut exact = a.exact;
let mut may_error = a.may_error;
let mut clauses = Vec::with_capacity(items.len());
for it in items {
let b = self.node(it, fb, false)?;
exact &= b.exact;
may_error |= b.may_error;
clauses.push(self.eq(&a, &b, pred));
}
Ok(Node {
expr: or_all(clauses),
kind: Kind::Bool,
nullable: false,
maybe_nan: false,
may_error,
exact,
})
}
_ => {
let a = self.node(l, fb, false)?;
let b = self.node(r, fb, false)?;
let exact = a.exact && b.exact;
let may_error = a.may_error || b.may_error;
let mut n = match op {
BinaryOp::Eq => Node::new(self.eq(&a, &b, pred), Kind::Bool).certain().finite(),
BinaryOp::Ne => Node::new(DfExpr::Not(Box::new(self.eq(&a, &b, false))), Kind::Bool)
.certain()
.finite(),
BinaryOp::Lt | BinaryOp::Le | BinaryOp::Gt | BinaryOp::Ge => self.ordered(op, &a, &b, pred)?,
BinaryOp::Add => self.add(&a, &b)?,
BinaryOp::Sub | BinaryOp::Mul | BinaryOp::Div | BinaryOp::Rem => self.arith(op, &a, &b)?,
BinaryOp::And | BinaryOp::Or | BinaryOp::In => unreachable!(),
_ => return no("unknown operator"),
};
n.exact = exact;
n.may_error |= may_error;
Ok(n)
}
}
}
fn eq(&self, a: &Node, b: &Node, pred: bool) -> DfExpr {
let both_null = binary(a.expr.clone().is_null(), Operator::And, b.expr.clone().is_null());
match (a.kind, b.kind) {
(Kind::Null, Kind::Null) => lit(true),
(Kind::Null, _) => b.expr.clone().is_null(),
(_, Kind::Null) => a.expr.clone().is_null(),
(x, y) if x != y => {
if a.nullable && b.nullable {
both_null
} else {
lit(false)
}
}
(Kind::Num, Kind::Num) => {
let (x, y) = (a.expr.clone(), b.expr.clone());
let mut clauses = Vec::new();
if !pred {
if a.nullable && b.nullable {
clauses.push((both_null, lit(true)));
}
if let Some(null) = any_null(&[a, b]) {
clauses.push((null, lit(false)));
}
}
if let Some(nan) = any_nan(&[a, b]) {
clauses.push((nan, lit(false)));
}
let plain = if is_nonzero_literal(&x) || is_nonzero_literal(&y) {
binary(x, Operator::Eq, y)
} else {
binary(
binary(x, Operator::Plus, f64_lit(0.0)),
Operator::Eq,
binary(y, Operator::Plus, f64_lit(0.0)),
)
};
if clauses.is_empty() {
plain
} else {
case(clauses, plain)
}
}
_ => {
if pred && (!a.nullable || !b.nullable) {
binary(a.expr.clone(), Operator::Eq, b.expr.clone())
} else {
binary(a.expr.clone(), Operator::IsNotDistinctFrom, b.expr.clone())
}
}
}
}
fn ordered(&self, op: BinaryOp, a: &Node, b: &Node, pred: bool) -> Result<Node, Unsupported> {
let dfop = match op {
BinaryOp::Lt => Operator::Lt,
BinaryOp::Le => Operator::LtEq,
BinaryOp::Gt => Operator::Gt,
_ => Operator::GtEq,
};
match (a.kind, b.kind) {
(Kind::Null, _) | (_, Kind::Null) => Ok(Node::new(lit(false), Kind::Bool).certain().finite()),
(Kind::Num, Kind::Num) | (Kind::Str, Kind::Str) => {
let (x, y) = (a.expr.clone(), b.expr.clone());
let mut clauses = Vec::new();
if !pred {
if let Some(null) = any_null(&[a, b]) {
clauses.push((null, lit(false)));
}
}
if a.kind == Kind::Num {
if let Some(nan) = any_nan(&[a, b]) {
clauses.push((nan, lit(false)));
}
}
let plain = binary(x, dfop, y);
let expr = if clauses.is_empty() {
plain
} else {
case(clauses, plain)
};
Ok(Node::new(expr, Kind::Bool).certain().finite())
}
(x, y) => no(format!("'{}' between a {x:?} and a {y:?}", op.symbol())),
}
}
fn add(&self, a: &Node, b: &Node) -> Result<Node, Unsupported> {
match (a.kind, b.kind) {
(Kind::Null, Kind::Null) => Ok(Node::null(Kind::Null)),
(Kind::Null, k) | (k, Kind::Null) if matches!(k, Kind::Num | Kind::Str) => Ok(Node::null(k)),
(Kind::Num, Kind::Num) => Ok(Node {
expr: binary(a.expr.clone(), Operator::Plus, b.expr.clone()),
kind: Kind::Num,
nullable: a.nullable || b.nullable,
maybe_nan: true,
may_error: false,
exact: true,
}),
(Kind::Str, Kind::Str) => Ok(Node {
expr: binary(a.expr.clone(), Operator::StringConcat, b.expr.clone()),
kind: Kind::Str,
nullable: a.nullable || b.nullable,
maybe_nan: false,
may_error: false,
exact: true,
}),
(x, y) => no(format!("'+' between a {x:?} and a {y:?}")),
}
}
fn arith(&self, op: BinaryOp, a: &Node, b: &Node) -> Result<Node, Unsupported> {
match (a.kind, b.kind) {
(Kind::Null, _) | (_, Kind::Null)
if matches!(a.kind, Kind::Null | Kind::Num) && matches!(b.kind, Kind::Null | Kind::Num) =>
{
Ok(Node::null(Kind::Num))
}
(Kind::Num, Kind::Num) => {
let (x, y) = (a.expr.clone(), b.expr.clone());
let dfop = match op {
BinaryOp::Sub => Operator::Minus,
BinaryOp::Mul => Operator::Multiply,
BinaryOp::Div => Operator::Divide,
_ => Operator::Modulo,
};
let guarded = matches!(op, BinaryOp::Div | BinaryOp::Rem) && !is_nonzero_literal(&b.expr);
let plain = binary(x, dfop, y.clone());
let expr = if guarded {
let mut clauses = Vec::new();
if let Some(null) = any_null(&[a, b]) {
clauses.push((null, null_of(Kind::Num)));
}
clauses.push((
binary(y, Operator::Eq, f64_lit(0.0)),
error(Kind::Num, "division by zero"),
));
case(clauses, plain)
} else {
plain
};
Ok(Node {
expr,
kind: Kind::Num,
nullable: a.nullable || b.nullable,
maybe_nan: true,
may_error: guarded,
exact: true,
})
}
(x, y) => no(format!("'{}' between a {x:?} and a {y:?}", op.symbol())),
}
}
fn call(&self, name: &str, args: Vec<Node>) -> Result<Node, Unsupported> {
let exact = args.iter().all(|a| a.exact);
let nullable = args.iter().any(|a| a.nullable);
let may_error = args.iter().any(|a| a.may_error);
let num = |expr: DfExpr, nullable: bool| Node {
expr,
kind: Kind::Num,
nullable,
maybe_nan: true,
may_error,
exact,
};
let num1 = |f: Udf, a: &Node| -> Result<Node, Unsupported> {
match a.kind {
Kind::Num => Ok(num(f().call(vec![a.expr.clone()]), a.nullable)),
Kind::Null => Ok(Node::null(Kind::Num)),
k => no(format!("{name} on a {k:?}")),
}
};
let str1 = |f: Udf, a: &Node, kind: Kind| -> Result<Node, Unsupported> {
match a.kind {
Kind::Str => Ok(Node {
expr: f().call(vec![a.expr.clone()]),
kind,
nullable: a.nullable,
maybe_nan: false,
may_error,
exact,
}),
Kind::Null => Ok(Node::null(kind)),
k => no(format!("{name} on a {k:?}")),
}
};
let all_num = |args: &[Node]| args.iter().all(|a| a.kind == Kind::Num);
let all_str = |args: &[Node]| args.iter().all(|a| a.kind == Kind::Str);
let any_null_kind = |args: &[Node]| args.iter().any(|a| a.kind == Kind::Null);
let all_num_or_null = |args: &[Node]| args.iter().all(|a| matches!(a.kind, Kind::Num | Kind::Null));
let all_str_or_null = |args: &[Node]| args.iter().all(|a| matches!(a.kind, Kind::Str | Kind::Null));
match (name, args.as_slice()) {
("abs", [a]) => num1(math::abs, a),
("floor", [a]) => num1(math::floor, a),
("ceil", [a]) => num1(math::ceil, a),
("trunc", [a]) => num1(math::trunc, a),
("exp", [a]) => num1(math::exp, a),
("cbrt", [a]) => num1(math::cbrt, a),
("sin", [a]) => num1(math::sin, a),
("cos", [a]) => num1(math::cos, a),
("tan", [a]) => num1(math::tan, a),
("atan", [a]) => num1(math::atan, a),
("sinh", [a]) => num1(math::sinh, a),
("cosh", [a]) => num1(math::cosh, a),
("tanh", [a]) => num1(math::tanh, a),
("degToRad", [a]) => num1(math::radians, a),
("radToDeg", [a]) => num1(math::degrees, a),
("sqrt", [a]) => Ok(guarded(
num1(math::sqrt, a)?,
a,
Guard::NonNegative,
"sqrt: negative argument",
)),
("ln", [a]) => Ok(guarded(
num1(math::ln, a)?,
a,
Guard::Positive,
"ln: argument must be positive",
)),
("log10", [a]) => Ok(guarded(
num1(math::log10, a)?,
a,
Guard::Positive,
"log10: argument must be positive",
)),
("log2", [a]) => Ok(guarded(
num1(math::log2, a)?,
a,
Guard::Positive,
"log2: argument must be positive",
)),
("asin", [a]) => Ok(guarded(
num1(math::asin, a)?,
a,
Guard::UnitInterval,
"asin: argument must be in [-1, 1]",
)),
("acos", [a]) => Ok(guarded(
num1(math::acos, a)?,
a,
Guard::UnitInterval,
"acos: argument must be in [-1, 1]",
)),
("round", [a]) => match a.kind {
Kind::Num => Ok(num(
math::floor().call(vec![binary(a.expr.clone(), Operator::Plus, f64_lit(0.5))]),
a.nullable,
)),
Kind::Null => Ok(Node::null(Kind::Num)),
k => no(format!("round on a {k:?}")),
},
("roundTo" | "pow" | "atan2", _) if any_null_kind(&args) && all_num_or_null(&args) => {
Ok(Node::null(Kind::Num))
}
("roundTo", [a, n]) if all_num(&args) => {
let f = math::power().call(vec![f64_lit(10.0), math::trunc().call(vec![n.expr.clone()])]);
let scaled = binary(
binary(a.expr.clone(), Operator::Multiply, f.clone()),
Operator::Plus,
f64_lit(0.5),
);
Ok(num(
binary(math::floor().call(vec![scaled]), Operator::Divide, f),
nullable,
))
}
("pow", [a, b]) if all_num(&args) => {
Ok(num(math::power().call(vec![a.expr.clone(), b.expr.clone()]), nullable))
}
("atan2", [a, b]) if all_num(&args) => {
Ok(num(math::atan2().call(vec![a.expr.clone(), b.expr.clone()]), nullable))
}
("pi", []) => Ok(Node::new(f64_lit(std::f64::consts::PI), Kind::Num).certain().finite()),
("sign", [a]) => match a.kind {
Kind::Num => {
let x = a.expr.clone();
let mut clauses = Vec::new();
if a.nullable {
clauses.push((x.clone().is_null(), null_of(Kind::Num)));
}
clauses.push((binary(x.clone(), Operator::Gt, f64_lit(0.0)), f64_lit(1.0)));
clauses.push((binary(x, Operator::Lt, f64_lit(0.0)), f64_lit(-1.0)));
Ok(Node {
expr: case(clauses, f64_lit(0.0)),
kind: Kind::Num,
nullable: a.nullable,
maybe_nan: false,
may_error,
exact,
})
}
Kind::Null => Ok(Node::null(Kind::Num)),
k => no(format!("sign on a {k:?}")),
},
("coalesce", _) => {
let kind = common_kind(args.iter().map(|a| Some(a.kind))).ok_or_else(|| Unsupported {
reason: "coalesce over mixed kinds".into(),
})?;
let exprs: Vec<DfExpr> = args.iter().map(|a| retype(a.expr.clone(), a.kind, kind)).collect();
Ok(Node {
expr: core::coalesce().call(exprs),
kind,
nullable: args.iter().all(|a| a.nullable),
maybe_nan: args.iter().any(|a| a.maybe_nan),
may_error,
exact,
})
}
("isNull", [a]) => Ok(Node {
expr: a.expr.clone().is_null(),
kind: Kind::Bool,
nullable: false,
maybe_nan: false,
may_error,
exact,
}),
("isNumber" | "isString" | "isBoolean", [a]) => {
let want = match name {
"isNumber" => Kind::Num,
"isString" => Kind::Str,
_ => Kind::Bool,
};
let expr = if a.kind == want {
a.expr.clone().is_not_null()
} else {
lit(false)
};
Ok(Node {
expr,
kind: Kind::Bool,
nullable: false,
maybe_nan: false,
may_error,
exact,
})
}
("typeOf", [a]) => {
let type_name = match a.kind {
Kind::Num => "number",
Kind::Str => "string",
Kind::Bool => "boolean",
Kind::Null => "null",
};
let expr = if a.nullable && a.kind != Kind::Null {
case(vec![(a.expr.clone().is_null(), lit("null"))], lit(type_name))
} else {
lit(type_name)
};
Ok(Node {
expr,
kind: Kind::Str,
nullable: false,
maybe_nan: false,
may_error,
exact,
})
}
("nullIf", [a, b]) => {
let kind = common_kind([Some(a.kind), Some(b.kind)].into_iter()).ok_or_else(|| Unsupported {
reason: "nullIf over mixed kinds".into(),
})?;
Ok(Node {
expr: case(
vec![(self.eq(a, b, false), null_of(kind))],
retype(a.expr.clone(), a.kind, kind),
),
kind,
nullable: true,
maybe_nan: a.maybe_nan,
may_error,
exact,
})
}
("upper", [a]) => str1(string::upper, a, Kind::Str),
("lower", [a]) => str1(string::lower, a, Kind::Str),
("reverse", [a]) => str1(unicode::reverse, a, Kind::Str),
("length", [a]) => match a.kind {
Kind::Str => Ok(Node {
expr: to_f64(unicode::character_length().call(vec![a.expr.clone()])),
kind: Kind::Num,
nullable: a.nullable,
maybe_nan: false,
may_error,
exact,
}),
Kind::Null => Ok(Node::null(Kind::Num)),
k => no(format!("length on a {k:?}")),
},
("startsWith" | "endsWith" | "contains" | "regexMatch", _)
if any_null_kind(&args) && all_str_or_null(&args) =>
{
Ok(Node::null(Kind::Bool))
}
("startsWith" | "endsWith" | "contains", [a, b]) if all_str(&args) => {
let f = match name {
"startsWith" => string::starts_with(),
"endsWith" => string::ends_with(),
_ => string::contains(),
};
Ok(Node {
expr: f.call(vec![a.expr.clone(), b.expr.clone()]),
kind: Kind::Bool,
nullable,
maybe_nan: false,
may_error,
exact,
})
}
("regexMatch", [a, b]) if all_str(&args) => Ok(Node {
expr: regex::regexp_like().call(vec![a.expr.clone(), b.expr.clone()]),
kind: Kind::Bool,
nullable,
maybe_nan: false,
may_error: true, exact,
}),
("replace" | "indexOf", _) if any_null_kind(&args) && all_str_or_null(&args) => {
Ok(Node::null(if name == "replace" { Kind::Str } else { Kind::Num }))
}
("splitPart", [a, b, c]) if a.kind == Kind::Str && b.kind == Kind::Str && c.kind == Kind::Num => Ok(Node {
expr: string::split_part().call(vec![
a.expr.clone(),
b.expr.clone(),
cast(c.expr.clone(), DataType::Int64),
]),
kind: Kind::Str,
nullable,
maybe_nan: false,
may_error: true, exact,
}),
("left" | "right", [a, b]) if a.kind == Kind::Str && non_negative_literal(&b.expr) => Ok(Node {
expr: (if name == "left" {
unicode::left()
} else {
unicode::right()
})
.call(vec![a.expr.clone(), cast(b.expr.clone(), DataType::Int64)]),
kind: Kind::Str,
nullable,
maybe_nan: false,
may_error,
exact,
}),
("substr", [a, b]) if a.kind == Kind::Str && positive_literal(&b.expr) => Ok(Node {
expr: unicode::substr().call(vec![a.expr.clone(), cast(b.expr.clone(), DataType::Int64)]),
kind: Kind::Str,
nullable,
maybe_nan: false,
may_error,
exact,
}),
("substr", [a, b, c])
if a.kind == Kind::Str && positive_literal(&b.expr) && non_negative_literal(&c.expr) =>
{
Ok(Node {
expr: unicode::substr().call(vec![
a.expr.clone(),
cast(b.expr.clone(), DataType::Int64),
cast(c.expr.clone(), DataType::Int64),
]),
kind: Kind::Str,
nullable,
maybe_nan: false,
may_error,
exact,
})
}
("padStart" | "padEnd", [a, b, c]) if a.kind == Kind::Str && b.kind == Kind::Num && c.kind == Kind::Str => {
let pad = (if name == "padStart" {
unicode::lpad()
} else {
unicode::rpad()
})
.call(vec![
a.expr.clone(),
cast(b.expr.clone(), DataType::Int64),
c.expr.clone(),
]);
let too_short = unicode::character_length()
.call(vec![a.expr.clone()])
.lt(cast(b.expr.clone(), DataType::Int32));
let has_pad = unicode::character_length().call(vec![c.expr.clone()]).gt(lit(0i32));
Ok(Node {
expr: when(too_short.and(has_pad), pad)
.otherwise(a.expr.clone())
.expect("case"),
kind: Kind::Str,
nullable,
maybe_nan: false,
may_error,
exact,
})
}
("regexExtract", [a, b]) if a.kind == Kind::Str && groupless_pattern(&b.expr) => Ok(Node {
expr: array_element(
regex::regexp_match().call(vec![a.expr.clone(), b.expr.clone()]),
lit(1i64),
),
kind: Kind::Str,
nullable: true,
maybe_nan: false,
may_error: true, exact,
}),
("epochToIso", [a]) if a.kind == Kind::Num => Ok(Node {
expr: datetime::expr_fn::to_char(
datetime::expr_fn::to_timestamp_millis(vec![cast(a.expr.clone(), DataType::Int64)]),
lit("%Y-%m-%dT%H:%M:%S%.3fZ"),
),
kind: Kind::Str,
nullable,
maybe_nan: false,
may_error,
exact,
}),
("toString", [a]) if a.kind == Kind::Str => Ok(a.clone()),
("toString", [a]) if a.kind == Kind::Bool => Ok(Node {
expr: when(a.expr.clone(), lit("true")).otherwise(lit("false")).expect("case"),
kind: Kind::Str,
nullable,
maybe_nan: false,
may_error,
exact,
}),
("toString", [a]) if a.kind == Kind::Num => Ok(Node {
expr: when(
math::floor().call(vec![a.expr.clone()]).eq(a.expr.clone()),
cast(cast(a.expr.clone(), DataType::Int64), DataType::Utf8),
)
.otherwise(cast(a.expr.clone(), DataType::Utf8))
.expect("case"),
kind: Kind::Str,
nullable,
maybe_nan: false,
may_error,
exact,
}),
("replace", [a, b, c]) if all_str(&args) => Ok(Node {
expr: string::replace().call(vec![a.expr.clone(), b.expr.clone(), c.expr.clone()]),
kind: Kind::Str,
nullable,
maybe_nan: false,
may_error,
exact,
}),
("indexOf", [a, b]) if all_str(&args) => {
let pos = to_f64(unicode::strpos().call(vec![a.expr.clone(), b.expr.clone()]));
let mut clauses = Vec::new();
if let Some(null) = any_null(&[a, b]) {
clauses.push((null, null_of(Kind::Num)));
}
clauses.push((binary(pos.clone(), Operator::Eq, f64_lit(0.0)), f64_lit(-1.0)));
Ok(Node {
expr: case(clauses, binary(pos, Operator::Minus, f64_lit(1.0))),
kind: Kind::Num,
nullable,
maybe_nan: false,
may_error,
exact,
})
}
("yearOf" | "monthOf" | "dayOf" | "hourOf" | "minuteOf" | "dayOfWeekOf" | "dayOfYear", [a]) => match a.kind
{
Kind::Num => {
let secs = math::floor().call(vec![binary(a.expr.clone(), Operator::Divide, f64_lit(1000.0))]);
let ts = datetime::to_timestamp_seconds(&ConfigOptions::default())
.call(vec![cast(secs, DataType::Int64)]);
let part = |p: &str| to_f64(datetime::date_part().call(vec![lit(p), ts.clone()]));
let expr = match name {
"yearOf" => part("year"),
"monthOf" => part("month"),
"dayOf" => part("day"),
"hourOf" => part("hour"),
"minuteOf" => part("minute"),
"dayOfYear" => part("doy"),
_ => {
let dow = part("dow");
case(
vec![(binary(dow.clone(), Operator::Eq, f64_lit(0.0)), f64_lit(7.0))],
dow,
)
}
};
Ok(Node {
expr,
kind: Kind::Num,
nullable: a.nullable,
maybe_nan: false,
may_error,
exact,
})
}
Kind::Null => Ok(Node::null(Kind::Num)),
k => no(format!("{name} on a {k:?}")),
},
_ => no(format!("function {name}")),
}
}
}